EGamma/test/dbs_discovery.py

0001 #!/usr/bin/env python3
0002
0003 #===================================================================
0004 # This python script is querying https://cmsweb.cern.ch/dbs_discovery/
0005 # so to get the list of input files. It can be called interactively,
0006 # or imported within a cmsRun config file. In the later case, one
0007 # must call :
0008 #   search(), to get the list of primary files
0009 #   search2(), to get the list of eventual secondary files
0010 #
0011 # The selection of files is configured thanks to shell
0012 # environment variables:
0013 #
0014 #   DBS_RELEASE, for example CMSSW_2_2_0_pre1
0015 #   DBS_SAMPLE, for example RelValSingleElectronPt35
0016 #   DBS_RUN, for example Any
0017 #   DBS_COND , for example MC_31X_V2-v1
0018 #   DBS_TIER , for example RECO
0019 #   DBS_TIER_SECONDARY, for eventual secondary files
0020 #
0021 # In the three last variables, one can use wildcard *
0022 #===================================================================
0023
0024
0025 from __future__ import print_function
0026 import httplib, urllib, urllib2, types, string, os, sys
0027
0028 if 'DBS_RELEASE' not in os.environ:
0029   os.environ['DBS_RELEASE'] = "Any"
0030 if 'DBS_SAMPLE' not in os.environ:
0031   os.environ['DBS_SAMPLE'] = "Any"
0032 if 'DBS_RUN' not in os.environ:
0033   os.environ['DBS_RUN'] = "Any"
0034
0035 def common_search(dbs_tier):
0036
0037   if os.environ['DBS_RELEASE'] == "LOCAL":
0038     result = []
0039     for line in  open('dbs_discovery.txt').readlines():
0040       line = line.strip()
0041       if line == "": continue
0042       if os.environ['DBS_SAMPLE'] != "Any" and line.find(os.environ['DBS_SAMPLE'])== -1: continue
0043       if line.find(os.environ['DBS_COND'])== -1: continue
0044       if line.find(dbs_tier)== -1: continue
0045       result.append('file:'+line)
0046   else:
0047     input = "find file"
0048     separator = " where "
0049     if os.environ['DBS_RELEASE'] != "Any":
0050       input = input + separator + "release = " + os.environ['DBS_RELEASE']
0051       separator = " and "
0052     if os.environ['DBS_SAMPLE'] != "Any":
0053       input = input + separator + "primds = " + os.environ['DBS_SAMPLE']
0054       separator = " and "
0055     if os.environ['DBS_RUN'] != "Any":
0056       input = input + separator + "run = " + os.environ['DBS_RUN']
0057       separator = " and "
0058     input = input + separator + "dataset like *" + os.environ['DBS_COND'] + "*" + dbs_tier + "*"
0059
0060     url = "https://cmsweb.cern.ch:443/dbs_discovery/aSearch"
0061     final_input = urllib.quote(input) ;
0062
0063     agent   = "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"
0064     ctypes  = "text/plain"
0065     headers = { 'User-Agent':agent, 'Accept':ctypes}
0066     params  = {'dbsInst':'cms_dbs_prod_global',
0067                'html':0,'caseSensitive':'on','_idx':0,'pagerStep':-1,
0068                'userInput':final_input,
0069                'xml':0,'details':0,'cff':0,'method':'dbsapi'}
0070     data    = urllib.urlencode(params,doseq=True)
0071     req     = urllib2.Request(url, data, headers)
0072     data    = ""
0073
0074     try:
0075       response = urllib2.urlopen(req)
0076       data = response.read()
0077     except urllib2.HTTPError as e:
0078       if e.code==201:
0079         print(e.headers)
0080         print(e.msg)
0081         pass
0082       else:
0083         raise e
0084
0085     result = []
0086     for line in data.split("\n"):
0087       if line != "" and line[0] =="/":
0088         result.append(line)
0089
0090   return result
0091
0092 def search():
0093   return common_search(os.environ['DBS_TIER'])
0094
0095 def search2():
0096   return common_search(os.environ['DBS_TIER_SECONDARY'])
0097
0098 if __name__ == "__main__":
0099   if 'DBS_TIER_SECONDARY' not in os.environ:
0100     os.environ['DBS_TIER_SECONDARY'] = ""
0101   if os.environ['DBS_TIER_SECONDARY'] == "":
0102     for file in search():
0103       print(file)
0104   else:
0105     for file in search():
0106       print("primary:"+file)
0107     for file in search2():
0108       print("secondary:"+file)
0109
0110
0111
0112