EGamma/test/dbs_discovery.py

0001 #!/usr/bin/env python3
0002
0003 #===================================================================
0004 # This python script is querying https://cmsweb.cern.ch/dbs_discovery/
0005 # so to get the list of input files. It can be called interactively,
0006 # or imported within a cmsRun config file. In the later case, one
0007 # must call :
0008 #   search(), to get the list of primary files
0009 #   search2(), to get the list of eventual secondary files
0010 #
0011 # The selection of files is configured thanks to shell
0012 # environment variables:
0013 #
0014 #   DBS_RELEASE, for example CMSSW_2_2_0_pre1
0015 #   DBS_SAMPLE, for example RelValSingleElectronPt35
0016 #   DBS_RUN, for example Any
0017 #   DBS_COND , for example MC_31X_V2-v1
0018 #   DBS_TIER , for example RECO
0019 #   DBS_TIER_SECONDARY, for eventual secondary files
0020 #
0021 # In the three last variables, one can use wildcard *
0022 #===================================================================
0023
0024
0025 import httplib, urllib, urllib2, types, string, os, sys
0026
0027 if 'DBS_RELEASE' not in os.environ:
0028   os.environ['DBS_RELEASE'] = "Any"
0029 if 'DBS_SAMPLE' not in os.environ:
0030   os.environ['DBS_SAMPLE'] = "Any"
0031 if 'DBS_RUN' not in os.environ:
0032   os.environ['DBS_RUN'] = "Any"
0033
0034 def common_search(dbs_tier):
0035
0036   if os.environ['DBS_RELEASE'] == "LOCAL":
0037     result = []
0038     for line in  open('dbs_discovery.txt').readlines():
0039       line = line.strip()
0040       if line == "": continue
0041       if os.environ['DBS_SAMPLE'] != "Any" and line.find(os.environ['DBS_SAMPLE'])== -1: continue
0042       if line.find(os.environ['DBS_COND'])== -1: continue
0043       if line.find(dbs_tier)== -1: continue
0044       result.append('file:'+line)
0045   else:
0046     input = "find file"
0047     separator = " where "
0048     if os.environ['DBS_RELEASE'] != "Any":
0049       input = input + separator + "release = " + os.environ['DBS_RELEASE']
0050       separator = " and "
0051     if os.environ['DBS_SAMPLE'] != "Any":
0052       input = input + separator + "primds = " + os.environ['DBS_SAMPLE']
0053       separator = " and "
0054     if os.environ['DBS_RUN'] != "Any":
0055       input = input + separator + "run = " + os.environ['DBS_RUN']
0056       separator = " and "
0057     input = input + separator + "dataset like *" + os.environ['DBS_COND'] + "*" + dbs_tier + "*"
0058
0059     url = "https://cmsweb.cern.ch:443/dbs_discovery/aSearch"
0060     final_input = urllib.quote(input) ;
0061
0062     agent   = "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"
0063     ctypes  = "text/plain"
0064     headers = { 'User-Agent':agent, 'Accept':ctypes}
0065     params  = {'dbsInst':'cms_dbs_prod_global',
0066                'html':0,'caseSensitive':'on','_idx':0,'pagerStep':-1,
0067                'userInput':final_input,
0068                'xml':0,'details':0,'cff':0,'method':'dbsapi'}
0069     data    = urllib.urlencode(params,doseq=True)
0070     req     = urllib2.Request(url, data, headers)
0071     data    = ""
0072
0073     try:
0074       response = urllib2.urlopen(req)
0075       data = response.read()
0076     except urllib2.HTTPError as e:
0077       if e.code==201:
0078         print(e.headers)
0079         print(e.msg)
0080         pass
0081       else:
0082         raise e
0083
0084     result = []
0085     for line in data.split("\n"):
0086       if line != "" and line[0] =="/":
0087         result.append(line)
0088
0089   return result
0090
0091 def search():
0092   return common_search(os.environ['DBS_TIER'])
0093
0094 def search2():
0095   return common_search(os.environ['DBS_TIER_SECONDARY'])
0096
0097 if __name__ == "__main__":
0098   if 'DBS_TIER_SECONDARY' not in os.environ:
0099     os.environ['DBS_TIER_SECONDARY'] = ""
0100   if os.environ['DBS_TIER_SECONDARY'] == "":
0101     for file in search():
0102       print(file)
0103   else:
0104     for file in search():
0105       print("primary:"+file)
0106     for file in search2():
0107       print("secondary:"+file)
0108
0109
0110
0111