Created
October 9, 2019 12:04
-
-
Save brianv0/a089756fbf15c337896dc00aad7b8de9 to your computer and use it in GitHub Desktop.
L1Proc Singularity+centos7
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <?xml version="1.0" encoding="UTF-8"?> | |
| <pipeline | |
| xmlns="http://glast-ground.slac.stanford.edu/pipeline" | |
| xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | |
| xsi:schemaLocation="http://glast-ground.slac.stanford.edu/pipeline https://glast-ground.slac.stanford.edu/Pipeline-II/schemas/2.1/pipeline.xsd"> | |
| <task name="L1Proc-Singularity" version="5.7" type="Data"> | |
| <variables> | |
| <var name="baseVersion">0</var> | |
| <var name="creator">L1Proc-5.7</var> | |
| <var name="deliveriesToIgnore">""</var> <!-- colon-separated list of deliveries to skip when merging --> | |
| <var name="isocMode">ISOC_PROD</var> | |
| <var name="isocRun">/afs/slac/g/glast/isoc/flightOps/rhel6_gcc44/${isocMode}/bin/isoc run</var> | |
| <var name="L1ProcROOT">/afs/slac.stanford.edu/g/glast/ground/PipelineConfig/Level1/5.7</var> | |
| <var name="L1_TASK_NAME">L1Proc</var> | |
| <var name="L1_TASK_VERSION">5.7</var> | |
| <var name="L1_INSTALL_DIR">/afs/slac.stanford.edu/g/glast/ground/PipelineConfig/Level1</var> | |
| <var name="L1_BUILD_DIR">/afs/slac/g/glast/ground/releases/volume11/L1Proc</var> | |
| <var name="L1TrendVersion">Prompt</var> <!-- override if reprocessing --> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib</var> | |
| <var name="logRoot">/nfs/farm/g/glast/u41/L1/logs/${pipeline.mode}</var> | |
| <var name="nameManglingPrefix">L1</var> | |
| <var name="highPriority">75</var> | |
| <var name="midPriority">60</var> | |
| <var name="standardPriority">50</var> | |
| <var name="reconPriority">74</var> | |
| <var name="frdPriority">76</var> | |
| <var name="standardSelect">centos7</var> | |
| <var name="standardRusage">scratch=1</var> | |
| <var name="bigRusage">scratch=250</var> | |
| </variables> | |
| <prerequisites> | |
| <prerequisite name="DOWNLINK_RAWDIR" type="string"/> | |
| <prerequisite name="DOWNLINK_ID" type="string"/> | |
| </prerequisites> | |
| <process name="findRunDirs" autoRetryMaxAttempts="1"> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/findRunDirs.py" batchOptions=" -q glastdataq -sp ${frdPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <createsSubtasks> | |
| <subtask>doRun</subtask> | |
| <subtask>doLci</subtask> | |
| </createsSubtasks> | |
| </process> | |
| <process name="kludgeAsp" autoRetryMaxAttempts="1"> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/kludgeAsp.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="doRun.registerExtendedFT1" status="SUCCESS"/> | |
| <after process="doRun.registerFT2" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="cleanupDl" autoRetryMaxAttempts="1"> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/deleteDirs.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="doLci.checkRunLci" status="SUCCESS"/> | |
| <after process="doRun.checkRun" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <task name="doRun" version="5.7" type="Data"> | |
| <variables> | |
| <var name="dataCatDir">/Data/Flight/Level1/${DATASOURCE}</var> | |
| </variables> | |
| <prerequisites> | |
| <prerequisite name="DATASOURCE" type="string"/> | |
| <prerequisite name="RUNID" type="string"/> | |
| <prerequisite name="RUNSTATUS" type="string"/> | |
| </prerequisites> | |
| <process name="findChunks" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="l1LockAction">LockAndThrottle</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 --add-env=flightops python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -E "${isocRun} ${L1ProcROOT}/lockFile.py" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <createsSubtasks> | |
| <subtask>doChunk</subtask> | |
| </createsSubtasks> | |
| </process> | |
| <process name="setRunning" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="l1RunStatus">Running</var> | |
| </variables> | |
| <script><![CDATA[ | |
| from java.util import HashMap | |
| def setIfUnset(container, key, value): | |
| oldValue = container.get(key) | |
| if oldValue is None or oldValue == 'Unset': | |
| print 'Setting %s to %s.' % (key, value) | |
| container[key] = value | |
| else: print 'Current value of %s is %s, not changing it.' % (key, oldValue) | |
| return | |
| print 'Setting L1RunStatus for run', runNumber, ' to ', l1RunStatus | |
| runQuality.setL1RunStatus(runNumber, l1RunStatus) | |
| reviewStatus = "Unset" | |
| print 'Setting ReviewStatus for run', runNumber, ' to ', reviewStatus | |
| runQuality.setReviewStatus(runNumber, reviewStatus) | |
| newQuality = 'Good' | |
| quality = runQuality.getRunQuality(runNumber) | |
| print 'Current quality for run %s is %s.' % (runNumber, quality) | |
| if quality is None or quality == 'Unset': | |
| print 'Setting quality for run %s to %s.' % (runNumber, newQuality) | |
| runQuality.setRunQuality(runNumber, newQuality) | |
| quality = newQuality | |
| else: print 'Not changing run quality.' | |
| pipeline.setVariable('runQuality', quality) | |
| fields = ['ACDQuality', 'CALQuality', 'GPSQuality', 'TRACKERQuality', | |
| 'TRIGGERQuality'] | |
| metaData = runQuality.getMetaData(runNumber) | |
| print 'old metaData:', metaData | |
| for key in fields: setIfUnset(metaData, key, newQuality) | |
| print 'new metadata:', metaData | |
| runQuality.setOrUpdateMetaData(runNumber, metaData) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="findChunks" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="copyM7Hp" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="L1_magic7Hp_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MAGIC7HP")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/copyM7.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="findChunks" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerM7Hp" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">copyM7Hp</var> | |
| <var name="fileType">magic7Hp</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="copyM7Hp" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="makeM7L1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="L1_magic7L1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MAGIC7L1")}</var> | |
| <var name="tStart">${pipeline.getCurrentStream().getProcessInstance("findChunks").getVariable("tStart")}</var> | |
| <var name="tStop">${pipeline.getCurrentStream().getProcessInstance("findChunks").getVariable("tStop")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/makeM7.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="findChunks" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerM7L1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">makeM7L1</var> | |
| <var name="fileType">magic7L1</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="makeM7L1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="drawOrbit" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">magic7L1</var> | |
| <var name="L1_magic7L1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MAGIC7L1")}</var> | |
| <var name="outFileTypes">orbitPlot</var> | |
| <var name="L1_orbitPlot_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ORBITPLOT")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="registerM7L1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerOrbitPlot" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">drawOrbit</var> | |
| <var name="fileType">orbitPlot</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="drawOrbit" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanDigi" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">digitization</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.digitization" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeDigi" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanDigi").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">digi</var> | |
| <var name="L1_digi_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGI")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${bigRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="scanDigi" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerDigi" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeDigi</var> | |
| <var name="fileType">digi</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeDigi" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanDigiHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">digiHist</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.digiHist" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeDigiHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanDigiHist").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">digiHist</var> | |
| <var name="L1_digiHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIHIST")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="scanDigiHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerDigiHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeDigiHist</var> | |
| <var name="fileType">digiHist</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeDigiHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="digiHistAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">digiHist</var> | |
| <var name="outFileTypes">digiHistAlarm</var> | |
| <var name="L1_digiHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIHIST")}</var> | |
| <var name="L1_digiHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIHISTALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerDigiHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerDigiHistAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">digiHistAlarm</var> | |
| <var name="fileType">digiHistAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="digiHistAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestDigiHistRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">digiHistAlarm</var> | |
| <var name="L1_digiHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIHISTALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerDigiHistAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="digiHistLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">digiHistAlarm</var> | |
| <var name="L1_digiHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIHISTALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerDigiHistAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="verify" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="L1_digi_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGI")}</var> | |
| <var name="L1_verifyLog_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYLOG")}</var> | |
| <var name="L1_verifyHisto_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYHISTO")}</var> | |
| <var name="completeness">InProgress</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/verify.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerDigi" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerVerifyHisto" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">verify</var> | |
| <var name="fileType">verifyHisto</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="verify" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerVerifyLog" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">verify</var> | |
| <var name="fileType">verifyLog</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="verify" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestVerifyErrorRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">verifyLog</var> | |
| <var name="L1_verifyLog_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYLOG")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyLog" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="verifyErrorAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">verifyLog</var> | |
| <var name="outFileTypes">verifyErrorAlarm</var> | |
| <var name="L1_verifyLog_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYLOG")}</var> | |
| <var name="L1_verifyErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyLog" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerVerifyErrorAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">verifyErrorAlarm</var> | |
| <var name="fileType">verifyErrorAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="verifyErrorAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="verifyErrorLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">verifyErrorAlarm</var> | |
| <var name="L1_verifyErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyErrorAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestVerifyErrorAlarmRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">verifyErrorAlarm</var> | |
| <var name="L1_verifyErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyErrorAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="acdPlots" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">digiHist</var> | |
| <var name="outFileTypes">acdPlots</var> | |
| <var name="L1_digiHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIHIST")}</var> | |
| <var name="L1_acdPlots_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ACDPLOTS")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerDigiHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerAcdPlots" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">acdPlots</var> | |
| <var name="fileType">acdPlots</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="acdPlots" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanDigiTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">digiTrend</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.digiTrend" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeDigiTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanDigiTrend").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">digiTrend</var> | |
| <var name="L1_digiTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGITREND")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="scanDigiTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerDigiTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeDigiTrend</var> | |
| <var name="fileType">digiTrend</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeDigiTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestDigiTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">digiTrend</var> | |
| <var name="L1_digiTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGITREND")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/ingestTrending.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerDigiTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="digiTrendAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">digiTrend</var> | |
| <var name="outFileTypes">digiTrendAlarm</var> | |
| <var name="L1_digiTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGITREND")}</var> | |
| <var name="L1_digiTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGITRENDALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerDigiTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerDigiTrendAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">digiTrendAlarm</var> | |
| <var name="fileType">digiTrendAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="digiTrendAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="digiTrendLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">digiTrendAlarm</var> | |
| <var name="L1_digiTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGITRENDALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerDigiTrendAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestDigiTrendRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">digiTrendAlarm</var> | |
| <var name="L1_digiTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGITRENDALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerDigiTrendAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="calHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">digi</var> | |
| <var name="outFileTypes">calHist</var> | |
| <var name="L1_calHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALHIST")}</var> | |
| <var name="L1_digi_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGI")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${bigRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerDigi" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerCalHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">calHist</var> | |
| <var name="fileType">calHist</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="calHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="calHistAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">calHist</var> | |
| <var name="outFileTypes">calHistAlarm</var> | |
| <var name="L1_calHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALHIST")}</var> | |
| <var name="L1_calHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALHISTALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerCalHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerCalHistAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">calHistAlarm</var> | |
| <var name="fileType">calHistAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="calHistAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="calHistLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">calHistAlarm</var> | |
| <var name="L1_calHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALHISTALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerCalHistAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestCalHistRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">calHistAlarm</var> | |
| <var name="L1_calHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALHISTALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerCalHistAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanCalTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">calTrend</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.calTrend" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeCalTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanCalTrend").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">calTrend</var> | |
| <var name="L1_calTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALTREND")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="scanCalTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerCalTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeCalTrend</var> | |
| <var name="fileType">calTrend</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeCalTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestCalTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">calTrend</var> | |
| <var name="L1_calTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALTREND")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/ingestTrending.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerCalTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="acdPedsAnalyzer" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="fileType">calHist</var> | |
| <var name="reportType">acdPedsAnalyzer</var> | |
| <var name="L1_calHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALHIST")}</var> | |
| <var name="L1_acdPedsAnalyzer_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ACDPEDSANALYZER")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/gainsPedsAnalyzer.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerCalHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerAcdPedsAnalyzer" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">acdPedsAnalyzer</var> | |
| <var name="fileType">acdPedsAnalyzer</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="acdPedsAnalyzer" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="acdPedsAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">acdPedsAnalyzer</var> | |
| <var name="outFileTypes">acdPedsAlarm</var> | |
| <var name="L1_acdPedsAnalyzer_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ACDPEDSANALYZER")}</var> | |
| <var name="L1_acdPedsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ACDPEDSALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerAcdPedsAnalyzer" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerAcdPedsAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">acdPedsAlarm</var> | |
| <var name="fileType">acdPedsAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="acdPedsAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="acdPedsLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">acdPedsAlarm</var> | |
| <var name="L1_acdPedsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ACDPEDSALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerAcdPedsAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestAcdPedsRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">acdPedsAlarm</var> | |
| <var name="L1_acdPedsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ACDPEDSALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerAcdPedsAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="calGainsAnalyzer" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="fileType">calHist</var> | |
| <var name="reportType">calGainsAnalyzer</var> | |
| <var name="L1_calHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALHIST")}</var> | |
| <var name="L1_calGainsAnalyzer_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALGAINSANALYZER")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/gainsPedsAnalyzer.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerCalHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerCalGainsAnalyzer" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">calGainsAnalyzer</var> | |
| <var name="fileType">calGainsAnalyzer</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="calGainsAnalyzer" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="calGainsAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">calGainsAnalyzer</var> | |
| <var name="outFileTypes">calGainsAlarm</var> | |
| <var name="L1_calGainsAnalyzer_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALGAINSANALYZER")}</var> | |
| <var name="L1_calGainsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALGAINSALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerCalGainsAnalyzer" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerCalGainsAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">calGainsAlarm</var> | |
| <var name="fileType">calGainsAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="calGainsAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="calGainsLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">calGainsAlarm</var> | |
| <var name="L1_calGainsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALGAINSALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerCalGainsAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestCalGainsRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">calGainsAlarm</var> | |
| <var name="L1_calGainsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALGAINSALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerCalGainsAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="calPedsAnalyzer" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="fileType">calHist</var> | |
| <var name="reportType">calPedsAnalyzer</var> | |
| <var name="L1_calHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALHIST")}</var> | |
| <var name="L1_calPedsAnalyzer_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALPEDSANALYZER")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/gainsPedsAnalyzer.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerCalHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerCalPedsAnalyzer" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">calPedsAnalyzer</var> | |
| <var name="fileType">calPedsAnalyzer</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="calPedsAnalyzer" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="calPedsAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">calPedsAnalyzer</var> | |
| <var name="outFileTypes">calPedsAlarm</var> | |
| <var name="L1_calPedsAnalyzer_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALPEDSANALYZER")}</var> | |
| <var name="L1_calPedsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALPEDSALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerCalPedsAnalyzer" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerCalPedsAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">calPedsAlarm</var> | |
| <var name="fileType">calPedsAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="calPedsAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="calPedsLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">calPedsAlarm</var> | |
| <var name="L1_calPedsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALPEDSALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerCalPedsAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestCalPedsRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">calPedsAlarm</var> | |
| <var name="L1_calPedsAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CALPEDSALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerCalPedsAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanFastMonTuple" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">fastMonTuple</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.fastMonTuple" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeFastMonTuple" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanFastMonTuple").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">fastMonTuple</var> | |
| <var name="L1_fastMonTuple_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONTUPLE")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="scanFastMonTuple" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFastMonTuple" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeFastMonTuple</var> | |
| <var name="fileType">fastMonTuple</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeFastMonTuple" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="mergeFastMonError" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanFastMonTuple").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">fastMonError</var> | |
| <var name="L1_fastMonError_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONERROR")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="scanFastMonTuple" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFastMonError" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeFastMonError</var> | |
| <var name="fileType">fastMonError</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeFastMonError" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestFastMonErrorRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">fastMonError</var> | |
| <var name="L1_fastMonError_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONERROR")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerFastMonError" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="fastMonErrorAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">fastMonError</var> | |
| <var name="outFileTypes">fastMonErrorAlarm</var> | |
| <var name="L1_fastMonError_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONERROR")}</var> | |
| <var name="L1_fastMonErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerFastMonError" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFastMonErrorAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">fastMonErrorAlarm</var> | |
| <var name="fileType">fastMonErrorAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="fastMonErrorAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="fastMonErrorLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">fastMonErrorAlarm</var> | |
| <var name="L1_fastMonErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerFastMonErrorAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestFastMonErrorAlarmRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">fastMonErrorAlarm</var> | |
| <var name="L1_fastMonErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerFastMonErrorAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanFastMonHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">fastMonHist</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.fastMonHist" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeFastMonHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanFastMonHist").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">fastMonHist</var> | |
| <var name="L1_fastMonHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONHIST")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="scanFastMonHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFastMonHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeFastMonHist</var> | |
| <var name="fileType">fastMonHist</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeFastMonHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="fastMonHistAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">fastMonHist</var> | |
| <var name="outFileTypes">fastMonHistAlarm</var> | |
| <var name="L1_fastMonHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONHIST")}</var> | |
| <var name="L1_fastMonHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONHISTALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerFastMonHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFastMonHistAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">fastMonHistAlarm</var> | |
| <var name="fileType">fastMonHistAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="fastMonHistAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="fastMonHistLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">fastMonHistAlarm</var> | |
| <var name="L1_fastMonHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONHISTALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerFastMonHistAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestFastMonHistRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">fastMonHistAlarm</var> | |
| <var name="L1_fastMonHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONHISTALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerFastMonHistAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanFastMonTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">fastMonTrend</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.fastMonTrend" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeFastMonTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanFastMonTrend").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">fastMonTrend</var> | |
| <var name="L1_fastMonTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONTREND")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="scanFastMonTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFastMonTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeFastMonTrend</var> | |
| <var name="fileType">fastMonTrend</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeFastMonTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestFastMonTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">fastMonTrend</var> | |
| <var name="L1_fastMonTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONTREND")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/ingestTrending.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerFastMonTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="fastMonTrendAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">fastMonTrend</var> | |
| <var name="outFileTypes">fastMonTrendAlarm</var> | |
| <var name="L1_fastMonTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONTREND")}</var> | |
| <var name="L1_fastMonTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONTRENDALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerFastMonTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFastMonTrendAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">fastMonTrendAlarm</var> | |
| <var name="fileType">fastMonTrendAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="fastMonTrendAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestFastMonTrendRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">fastMonTrendAlarm</var> | |
| <var name="L1_fastMonTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONTRENDALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerFastMonTrendAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="fastMonTrendLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">fastMonTrendAlarm</var> | |
| <var name="L1_fastMonTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONTRENDALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerFastMonTrendAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanGcrChunks" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">mergeGcrCrumbs</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.mergeGcrCrumbs" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeGcrChunks" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanGcrChunks").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">gcr</var> | |
| <var name="L1_gcr_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "GCR")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="scanGcrChunks" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerGcr" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeGcrChunks</var> | |
| <var name="fileType">gcr</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeGcrChunks" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanMeritChunks" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">mergeMeritCrumbs</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.mergeMeritCrumbs" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeMeritChunks" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanMeritChunks").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">merit</var> | |
| <var name="L1_merit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERIT")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[scratch=2]" -app centos6 " /> | |
| <depends> | |
| <after process="scanMeritChunks" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerMerit" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeMeritChunks</var> | |
| <var name="fileType">merit</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeMeritChunks" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <!-- | |
| <process name="filterMerit" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">merit</var> | |
| <var name="L1_merit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERIT")}</var> | |
| <var name="outFileTypes">filteredMerit</var> | |
| <var name="L1_filteredMerit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FILTEREDMERIT")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[scratch=2]" -app centos6 " /> | |
| <depends> | |
| <after process="registerMerit" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFilteredMerit" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">filterMerit</var> | |
| <var name="fileType">filteredMerit</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="filterMerit" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| --> | |
| <process name="electronMerit" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">merit</var> | |
| <var name="L1_merit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERIT")}</var> | |
| <var name="outFileTypes">electronMerit</var> | |
| <var name="L1_electronMerit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ELECTRONMERIT")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[scratch=2]" -app centos6 " /> | |
| <depends> | |
| <after process="registerMerit" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerElectronMerit" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">electronMerit</var> | |
| <var name="fileType">electronMerit</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="electronMerit" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="verifyMerit" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="L1_merit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERIT")}</var> | |
| <var name="L1_verifyMeritError_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYMERITERROR")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/verifyMerit.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerMerit" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerVerifyMeritError" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">verifyMerit</var> | |
| <var name="fileType">verifyMeritError</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="verifyMerit" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestVerifyMeritErrorRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">verifyMeritError</var> | |
| <var name="L1_verifyMeritError_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYMERITERROR")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyMeritError" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="verifyMeritErrorAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">verifyMeritError</var> | |
| <var name="outFileTypes">verifyMeritErrorAlarm</var> | |
| <var name="alarmFileType">verifyMeritErrorAlarm</var> | |
| <var name="L1_verifyMeritError_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYMERITERROR")}</var> | |
| <var name="L1_verifyMeritErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYMERITERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyMeritError" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerVerifyMeritErrorAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">verifyMeritErrorAlarm</var> | |
| <var name="fileType">verifyMeritErrorAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="verifyMeritErrorAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="verifyMeritErrorLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">verifyMeritErrorAlarm</var> | |
| <var name="L1_verifyMeritErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYMERITERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyMeritErrorAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestVerifyMeritErrorAlarmRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">verifyMeritErrorAlarm</var> | |
| <var name="L1_verifyMeritErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYMERITERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyMeritErrorAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="makeGaps" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="L1_digi_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGI")}</var> | |
| <var name="L1_digiGap_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIGAP")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/makeGaps.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[scratch=5]" -app centos6 " /> | |
| <depends> | |
| <after process="registerDigi" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerGaps" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">makeGaps</var> | |
| <var name="fileType">digiGap</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="makeGaps" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ft2Runs" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="outFileType">ft2SecondsNoQual</var> | |
| <var name="L1_digi_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGI")}</var> | |
| <var name="L1_ft2SecondsNoQual_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDSNOQUAL")}</var> | |
| <var name="L1_digiGap_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIGAP")}</var> | |
| <var name="L1_magic7L1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MAGIC7L1")}</var> | |
| <var name="L1_merit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERIT")}</var> | |
| <var name="runQuality">${pipeline.getCurrentStream.getProcessInstance("setRunning").getVariable("runQuality")}</var> | |
| <var name="mootAlias">${pipeline.getCurrentStream().getProcessInstance("findChunks").getVariable("mootAlias")}</var> | |
| <!-- <var name="L1_magic7L1_filename">${datacatalog.getDataset(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MAGIC7L1").getLocation("SLAC_XROOT").getPath()}</var> --> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/ft2Entries.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[scratch=15]" -app centos6 " /> | |
| <depends> | |
| <after process="registerGaps" status="SUCCESS"/> | |
| <after process="registerM7L1" status="SUCCESS"/> | |
| <after process="registerMerit" status="SUCCESS"/> | |
| <after process="setRunning" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFt2SecondsNoQual" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">ft2Runs</var> | |
| <var name="fileType">ft2SecondsNoQual</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="ft2Runs" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <!-- | |
| <process name="ingestFT2" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="L1_ft2Txt_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2TXT")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/ft2Db.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="registerFt2Txt" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| --> | |
| <process name="makeFT2" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileType">ft2SecondsNoQual</var> | |
| <var name="outFileType">ft2NoQual</var> | |
| <var name="L1_ft2SecondsNoQual_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDSNOQUAL")}</var> | |
| <var name="L1_ft2NoQual_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2NOQUAL")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/makeFT2File.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <!-- <after process="ingestFT2" status="SUCCESS"/> --> | |
| <after process="registerFt2SecondsNoQual" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFT2NoQual" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">makeFT2</var> | |
| <var name="fileType">ft2NoQual</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="makeFT2" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="setupQuality"> | |
| <script><![CDATA[ | |
| creator = pipeline.getTaskVersionPath() | |
| try: | |
| theRun = runNumber | |
| except NameError: | |
| theRun = int(RUNID[2:]) | |
| pass | |
| theSet = runQuality.getMostRecentSubmittedTimeIntervalSet(theRun, creator) | |
| if theSet: | |
| intervals = theSet.getActiveIntervals() | |
| print intervals | |
| lines = [] | |
| for interval in intervals: | |
| start = interval.getRequestedStartPeriod() | |
| stop = interval.getRequestedEndPeriod() | |
| qual = interval.getDataQual() | |
| print start, stop, qual | |
| lines.append("%s:%s:%s" % (start, stop, qual)) | |
| continue | |
| rangeStr = ",".join(lines) | |
| else: | |
| rangeStr = "x" | |
| pass | |
| print "rangeStr:", rangeStr | |
| pipeline.setVariable("badRanges", rangeStr) | |
| #print "Failing on purpose for debugging." | |
| #raise SystemExit | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="findChunks" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="flagFT2" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="btiProcess">setupQuality</var> | |
| <var name="badRanges">${pipeline.getCurrentStream.getProcessInstance(btiProcess).getVariable("badRanges")}</var> | |
| <var name="L1_ft2_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2")}</var> | |
| <var name="L1_ft2NoQual_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2NOQUAL")}</var> | |
| <var name="L1_ft2Seconds_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDS")}</var> | |
| <var name="L1_ft2SecondsNoQual_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDSNOQUAL")}</var> | |
| <var name="inFileTypes">ft2NoQual/ft2SecondsNoQual</var> | |
| <var name="outFileTypes">ft2/ft2Seconds</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerFT2NoQual" status="SUCCESS"/> | |
| <after process="setupQuality" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFT2" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">flagFT2</var> | |
| <var name="fileType">ft2</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="flagFT2" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="exportFT2" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="fileType">ft2</var> | |
| <var name="L1_ft2_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/exportStuff.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="registerFT2" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFt2Seconds" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">flagFT2</var> | |
| <var name="fileType">ft2Seconds</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="flagFT2" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="exportFT2Seconds" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="fileType">ft2Seconds</var> | |
| <var name="L1_ft2Seconds_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDS")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/exportStuff.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="registerFt2Seconds" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="wrapupQuality"> | |
| <variables> | |
| <var name="btiProcess">flagFT2</var> | |
| </variables> | |
| <script><![CDATA[ | |
| from java.sql import Timestamp | |
| try: | |
| theRun = runNumber | |
| except NameError: | |
| theRun = int(RUNID[2:]) | |
| pass | |
| creator = pipeline.getTaskVersionPath() | |
| theSet = runQuality.getMostRecentSubmittedTimeIntervalSet(theRun, creator) | |
| ivs = [] | |
| if theSet: | |
| intervals = theSet.getActiveIntervals() | |
| for interval in intervals: | |
| range = ((interval.getRequestedStartPeriod(), interval.getRequestedEndPeriod()), interval.getDataQual()) | |
| ivs.append((range, interval)) | |
| continue | |
| ivs.sort() | |
| pass | |
| print 'ivs:', ivs | |
| currentStream = pipeline.getCurrentStream() | |
| btiProcess = currentStream.getProcessInstance(btiProcess) | |
| badRanges = btiProcess.getVariable("badRanges") | |
| ranges = [] | |
| if badRanges == 'x': | |
| pass | |
| else: | |
| lines = badRanges.split(',') | |
| for line in lines: | |
| fields = line.split(":") | |
| start = int(round(float(fields[0]))) | |
| stop = int(round(float(fields[1]))) | |
| qual = int(fields[2]) | |
| ranges.append(((start, stop), qual)) | |
| continue | |
| ranges.sort() | |
| pass | |
| print 'ranges:', ranges | |
| if len(ivs) != len(ranges): | |
| print "Flagged and requested interval lists have different lengths!" | |
| raise SystemExit | |
| stuff = zip(ranges, ivs) | |
| # # This fails if the interval overlaps the start or end of a run | |
| # # or a gap, probably | |
| # for (range, iv) in stuff: | |
| # ((ft2Start, ft2Stop), ft2Qual) = range | |
| # ((reqStart, reqStop), reqQual) = iv[0] | |
| # if not ((ft2Start <= reqStart) and (ft2Stop >= reqStop) and (ft2Qual == reqQual)): | |
| # print "Interval mismatch!" | |
| # raise SystemExit | |
| # continue | |
| for (range, iv) in stuff: | |
| ((start, stop), qual) = range | |
| interval = iv[1] | |
| interval.setFt2StartPeriod(start) | |
| interval.setFt2EndPeriod(stop) | |
| continue | |
| exportProcess = currentStream.getProcessInstance("exportFT2") | |
| exportTime = exportProcess.getVariable("exportTime") | |
| millis = long(float(exportTime) * 1000) | |
| print millis | |
| exportTime = Timestamp(millis) | |
| if theSet: | |
| theSet.setExportTime(exportTime) | |
| theSet.setStreamId(pipeline.getStream()) | |
| theSet.setTaskName(creator) | |
| runQuality.updateBadTimeIntervalSet(theSet) | |
| pass | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="exportFT2Seconds" status="SUCCESS"/> | |
| <after process="exportFT2" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="verifyFt2" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="L1_ft2_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2")}</var> | |
| <var name="L1_verifyFt2Error_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT2ERROR")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/verifyFt2.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerFT2" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerVerifyFt2Error" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">verifyFt2</var> | |
| <var name="fileType">verifyFt2Error</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="verifyFt2" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestVerifyFt2ErrorRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">verifyFt2Error</var> | |
| <var name="L1_verifyFt2Error_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT2ERROR")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyFt2Error" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="verifyFt2ErrorAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">verifyFt2Error</var> | |
| <var name="outFileTypes">verifyFt2ErrorAlarm</var> | |
| <var name="alarmFileType">verifyFt2ErrorAlarm</var> | |
| <var name="L1_verifyFt2Error_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT2ERROR")}</var> | |
| <var name="L1_verifyFt2ErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT2ERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyFt2Error" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerVerifyFt2ErrorAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">verifyFt2ErrorAlarm</var> | |
| <var name="fileType">verifyFt2ErrorAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="verifyFt2ErrorAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="verifyFt2ErrorLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">verifyFt2ErrorAlarm</var> | |
| <var name="L1_verifyFt2ErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT2ERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyFt2ErrorAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestVerifyFt2ErrorAlarmRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">verifyFt2ErrorAlarm</var> | |
| <var name="L1_verifyFt2ErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT2ERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyFt2ErrorAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanFT1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">diffRspFT1</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.diffRspFT1" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeFT1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanFT1").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">ft1BadGti</var> | |
| <var name="L1_ft1BadGti_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT1BADGTI")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="scanFT1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFT1BadGti" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeFT1</var> | |
| <var name="fileType">ft1BadGti</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeFT1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="gtiFT1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">ft1BadGti/ft2Seconds</var> | |
| <var name="outFileTypes">extendedFT1</var> | |
| <var name="L1_ft1BadGti_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT1BADGTI")}</var> | |
| <var name="L1_ft2Seconds_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDS")}</var> | |
| <var name="L1_extendedFT1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "EXTENDEDFT1")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="registerFT1BadGti" status="SUCCESS"/> | |
| <after process="registerFt2Seconds" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerExtendedFT1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">gtiFT1</var> | |
| <var name="fileType">extendedFT1</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="gtiFT1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="filterFT1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">extendedFT1/ft2Seconds</var> | |
| <var name="outFileTypes">ft1</var> | |
| <var name="L1_extendedFT1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "EXTENDEDFT1")}</var> | |
| <var name="L1_ft2Seconds_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDS")}</var> | |
| <var name="L1_ft1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT1")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="registerExtendedFT1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFT1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">filterFT1</var> | |
| <var name="fileType">ft1</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="filterFT1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="exportFT1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="fileType">ft1</var> | |
| <var name="L1_ft1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT1")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/exportStuff.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="registerFT1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="verifyFt1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="L1_extendedFT1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "EXTENDEDFT1")}</var> | |
| <var name="L1_verifyFt1Error_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT1ERROR")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/verifyFt1.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerExtendedFT1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerVerifyFt1Error" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">verifyFt1</var> | |
| <var name="fileType">verifyFt1Error</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="verifyFt1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestVerifyFt1ErrorRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">verifyFt1Error</var> | |
| <var name="L1_verifyFt1Error_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT1ERROR")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyFt1Error" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="verifyFt1ErrorAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">verifyFt1Error</var> | |
| <var name="outFileTypes">verifyFt1ErrorAlarm</var> | |
| <var name="alarmFileType">verifyFt1ErrorAlarm</var> | |
| <var name="L1_verifyFt1Error_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT1ERROR")}</var> | |
| <var name="L1_verifyFt1ErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT1ERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyFt1Error" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerVerifyFt1ErrorAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">verifyFt1ErrorAlarm</var> | |
| <var name="fileType">verifyFt1ErrorAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="verifyFt1ErrorAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="verifyFt1ErrorLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">verifyFt1ErrorAlarm</var> | |
| <var name="L1_verifyFt1ErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT1ERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyFt1ErrorAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestVerifyFt1ErrorAlarmRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">verifyFt1ErrorAlarm</var> | |
| <var name="L1_verifyFt1ErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYFT1ERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyFt1ErrorAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="electronFT1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">merit</var> | |
| <var name="outFileTypes">electronFT1BadGti</var> | |
| <var name="L1_merit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERIT")}</var> | |
| <var name="L1_electronFT1BadGti_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ELECTRONFT1BADGTI")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="registerElectronMerit" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerElectronFT1BadGti" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">electronFT1</var> | |
| <var name="fileType">electronFT1BadGti</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="electronFT1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="gtiElectronFT1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">electronFT1BadGti/ft2Seconds</var> | |
| <var name="outFileTypes">electronFT1</var> | |
| <var name="L1_ft2Seconds_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDS")}</var> | |
| <var name="L1_electronFT1BadGti_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ELECTRONFT1BADGTI")}</var> | |
| <var name="L1_electronFT1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "ELECTRONFT1")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="registerElectronFT1BadGti" status="SUCCESS"/> | |
| <after process="registerFt2Seconds" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerElectronFT1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">gtiElectronFT1</var> | |
| <var name="fileType">electronFT1</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="gtiElectronFT1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="makeLS1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">merit</var> | |
| <var name="outFileTypes">ls1BadGti</var> | |
| <var name="L1_merit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERIT")}</var> | |
| <var name="L1_ls1BadGti_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "LS1BADGTI")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="registerMerit" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerLS1BadGti" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">makeLS1</var> | |
| <var name="fileType">ls1BadGti</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="makeLS1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="gtiLS1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">ls1BadGti/ft2Seconds</var> | |
| <var name="outFileTypes">extendedLS1</var> | |
| <var name="L1_ft2Seconds_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDS")}</var> | |
| <var name="L1_ls1BadGti_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "LS1BADGTI")}</var> | |
| <var name="L1_extendedLS1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "EXTENDEDLS1")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="registerLS1BadGti" status="SUCCESS"/> | |
| <after process="registerFt2Seconds" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerExtendedLS1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">gtiLS1</var> | |
| <var name="fileType">extendedLS1</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="gtiLS1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="filterLS1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">extendedLS1/ft2Seconds</var> | |
| <var name="outFileTypes">ls1</var> | |
| <var name="L1_extendedLS1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "EXTENDEDLS1")}</var> | |
| <var name="L1_ft2Seconds_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FT2SECONDS")}</var> | |
| <var name="L1_ls1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "LS1")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="registerExtendedLS1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerLS1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">filterLS1</var> | |
| <var name="fileType">ls1</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="filterLS1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="exportLS1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="fileType">ls1</var> | |
| <var name="L1_ls1_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "LS1")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/exportStuff.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="registerLS1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanRelationChunks" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">mergeRelationCrumbs</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.mergeRelationCrumbs" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeRelationChunks" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanRelationChunks").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">relation</var> | |
| <var name="L1_relation_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RELATION")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[scratch=250]" -app centos6 "/> | |
| <depends> | |
| <after process="scanRelationChunks" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerRelation" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeRelationChunks</var> | |
| <var name="fileType">relation</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeRelationChunks" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanCalChunks" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">mergeCalCrumbs</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.mergeCalCrumbs" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeCalChunks" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanCalChunks").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">cal</var> | |
| <var name="L1_cal_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "CAL")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[scratch=250]" -app centos6 "/> | |
| <depends> | |
| <after process="scanCalChunks" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerCal" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeCalChunks</var> | |
| <var name="fileType">cal</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeCalChunks" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanMeritTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">meritTrend</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.meritTrend" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeMeritTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanMeritTrend").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">meritTrend</var> | |
| <var name="L1_meritTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITTREND")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="scanMeritTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerMeritTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeMeritTrend</var> | |
| <var name="fileType">meritTrend</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeMeritTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestMeritTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">meritTrend</var> | |
| <var name="L1_meritTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITTREND")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/ingestTrending.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerMeritTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="meritTrendAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">meritTrend</var> | |
| <var name="outFileTypes">meritTrendAlarm</var> | |
| <var name="L1_meritTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITTREND")}</var> | |
| <var name="L1_meritTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITTRENDALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerMeritTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerMeritTrendAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">meritTrendAlarm</var> | |
| <var name="fileType">meritTrendAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="meritTrendAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="meritTrendLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">meritTrendAlarm</var> | |
| <var name="L1_meritTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITTRENDALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerMeritTrendAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestMeritTrendRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">meritTrendAlarm</var> | |
| <var name="L1_meritTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITTRENDALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerMeritTrendAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="meritHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">digi/merit</var> | |
| <var name="outFileTypes">meritHist</var> | |
| <var name="L1_digi_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGI")}</var> | |
| <var name="L1_merit_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERIT")}</var> | |
| <var name="L1_meritHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITHIST")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="doChunk.allRecon" status="SUCCESS"/> | |
| <after process="registerDigi" status="SUCCESS"/> | |
| <after process="registerMerit" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerMeritHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">meritHist</var> | |
| <var name="fileType">meritHist</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="meritHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="meritHistAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">meritHist</var> | |
| <var name="outFileTypes">meritHistAlarm</var> | |
| <var name="L1_meritHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITHIST")}</var> | |
| <var name="L1_meritHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITHISTALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerMeritHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerMeritHistAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">meritHistAlarm</var> | |
| <var name="fileType">meritHistAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="meritHistAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestMeritHistRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">meritHistAlarm</var> | |
| <var name="L1_meritHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITHISTALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerMeritHistAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="meritHistLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">meritHistAlarm</var> | |
| <var name="L1_meritHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITHISTALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerMeritHistAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="findSolarFlares" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="L1_digiTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGITREND")}</var> | |
| <var name="L1_meritTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MERITTREND")}</var> | |
| <var name="L1_solarFlareHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "SOLARFLAREHIST")}</var> | |
| <var name="L1_solarFlareLog_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "SOLARFLARELOG")}</var> | |
| <var name="L1_solarFlarePlot_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "SOLARFLAREPLOT")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/findSolarFlares.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerDigiTrend" status="SUCCESS"/> | |
| <after process="registerMeritTrend" status="SUCCESS"/> | |
| <after process="doChunk.digiTrend" status="SUCCESS"/> | |
| <after process="doChunk.meritTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerSolarFlareHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">findSolarFlares</var> | |
| <var name="fileType">solarFlareHist</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="findSolarFlares" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerSolarFlareLog" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">findSolarFlares</var> | |
| <var name="fileType">solarFlareLog</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="findSolarFlares" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerSolarFlarePlot" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">findSolarFlares</var> | |
| <var name="fileType">solarFlarePlot</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="findSolarFlares" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="suggestQuality" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="btiProcess">findSolarFlares</var> | |
| </variables> | |
| <script><![CDATA[ | |
| creator = pipeline.getTaskVersionPath() | |
| currentStream = pipeline.getCurrentStream() | |
| btiProcess = currentStream.getProcessInstance(btiProcess) | |
| badRanges = btiProcess.getVariable("badRanges") | |
| ranges = [] | |
| if badRanges == 'x': | |
| pass | |
| else: | |
| lines = badRanges.split(',') | |
| for line in lines: | |
| fields = line.split(":") | |
| start = int(round(float(fields[0]))) | |
| stop = int(round(float(fields[1]))) | |
| qual = int(fields[2]) | |
| ranges.append(((start, stop), qual)) | |
| continue | |
| ranges.sort() | |
| pass | |
| print 'ranges:', ranges | |
| if ranges: | |
| theSet = runQuality.createBadTimeIntervalSetForRun(runNumber, creator) | |
| setId = theSet.getId() | |
| for (start, stop), qual in ranges: | |
| iv = runQuality.createBadTimeIntervalForSet(setId, creator) | |
| iv.setSuggestedStartPeriod(start) | |
| iv.setSuggestedEndPeriod(stop) | |
| iv.setDataQual(qual) | |
| runQuality.updateBadTimeInterval(iv) | |
| continue | |
| runQuality.updateBadTimeIntervalSet(theSet) | |
| pass | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="findSolarFlares" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanReconTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">reconTrend</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.reconTrend" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeReconTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanReconTrend").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">reconTrend</var> | |
| <var name="L1_reconTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONTREND")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <!-- <after process="doChunk.mergeCalCrumbs" status="DONE"/> # This is not really necessary. In the situation it was designed to protect against, we'd rather not run at all. But we can't say that. --> | |
| <!-- <after process="doChunk.mergeReconCrumbs" status="DONE"/> # This is not really necessary. In the situation it was designed to protect against, we'd rather not run at all. But we can't say that. --> | |
| <after process="scanReconTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerReconTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeReconTrend</var> | |
| <var name="fileType">reconTrend</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeReconTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestReconTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">reconTrend</var> | |
| <var name="L1_reconTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONTREND")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/ingestTrending.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerReconTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="reconTrendAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">reconTrend</var> | |
| <var name="outFileTypes">reconTrendAlarm</var> | |
| <var name="L1_reconTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONTREND")}</var> | |
| <var name="L1_reconTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONTRENDALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerReconTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerReconTrendAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">reconTrendAlarm</var> | |
| <var name="fileType">reconTrendAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="reconTrendAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestReconTrendRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">reconTrendAlarm</var> | |
| <var name="L1_reconTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONTRENDALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerReconTrendAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="reconTrendLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">reconTrendAlarm</var> | |
| <var name="L1_reconTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONTRENDALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerReconTrendAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanReconHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">reconHist</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.reconHist" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeReconHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanReconHist").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">reconHist</var> | |
| <var name="L1_reconHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONHIST")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <!-- <after process="doChunk.mergeCalCrumbs" status="DONE"/> # This is not really necessary. In the situation it was designed to protect against, we'd rather not run at all. But we can't say that. --> | |
| <!-- <after process="doChunk.mergeReconCrumbs" status="DONE"/> # This is not really necessary. In the situation it was designed to protect against, we'd rather not run at all. But we can't say that. --> | |
| <after process="scanReconHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerReconHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeReconHist</var> | |
| <var name="fileType">reconHist</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeReconHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="reconHistAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">reconHist</var> | |
| <var name="outFileTypes">reconHistAlarm</var> | |
| <var name="L1_reconHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONHIST")}</var> | |
| <var name="L1_reconHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONHISTALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerReconHist" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerReconHistAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">reconHistAlarm</var> | |
| <var name="fileType">reconHistAlarm</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="reconHistAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="reconHistLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">reconHistAlarm</var> | |
| <var name="L1_reconHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONHISTALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerReconHistAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestReconHistRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">reconHistAlarm</var> | |
| <var name="L1_reconHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONHISTALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerReconHistAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="reconHistPostProcessor" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="fileType">reconHistAlarm</var> | |
| <var name="L1_reconHistAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONHISTALARM")}</var> | |
| <var name="L1_reconHistAlarmDist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECONHISTALARMDIST")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/alarmPostProcessor.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerReconHistAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerReconHistAlarmDist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">reconHistPostProcessor</var> | |
| <var name="fileType">reconHistAlarmDist</var> | |
| <var name="timeProcess">findChunks</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="reconHistPostProcessor" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanTkrAnalysis" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">tkrAnalysis</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.tkrAnalysis" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="scanReconChunks" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunk</var> | |
| <var name="subProcess">mergeReconCrumbs</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunk.mergeReconCrumbs" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="checkRun" autoRetryMaxAttempts="1"> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/checkRun.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="acdPedsLogger" status="SUCCESS"/> | |
| <after process="calGainsLogger" status="SUCCESS"/> | |
| <after process="calHistLogger" status="SUCCESS"/> | |
| <after process="calPedsLogger" status="SUCCESS"/> | |
| <after process="doChunk.checkChunk" status="SUCCESS"/> | |
| <after process="digiHistLogger" status="SUCCESS"/> | |
| <after process="digiTrendLogger" status="SUCCESS"/> | |
| <after process="exportFT1" status="SUCCESS"/> | |
| <after process="exportLS1" status="SUCCESS"/> | |
| <after process="fastMonErrorLogger" status="SUCCESS"/> | |
| <after process="fastMonHistLogger" status="SUCCESS"/> | |
| <after process="fastMonTrendLogger" status="SUCCESS"/> | |
| <after process="ingestAcdPedsRun" status="SUCCESS"/> | |
| <after process="ingestCalGainsRun" status="SUCCESS"/> | |
| <after process="ingestCalHistRun" status="SUCCESS"/> | |
| <after process="ingestCalPedsRun" status="SUCCESS"/> | |
| <after process="ingestCalTrend" status="SUCCESS"/> | |
| <after process="ingestDigiTrend" status="SUCCESS"/> | |
| <after process="ingestFastMonTrend" status="SUCCESS"/> | |
| <after process="ingestMeritTrend" status="SUCCESS"/> | |
| <after process="ingestReconTrend" status="SUCCESS"/> | |
| <after process="ingestFastMonTrendRun" status="SUCCESS"/> | |
| <after process="ingestFastMonHistRun" status="SUCCESS"/> | |
| <after process="ingestFastMonErrorRun" status="SUCCESS"/> | |
| <after process="ingestFastMonErrorAlarmRun" status="SUCCESS"/> | |
| <after process="ingestDigiTrendRun" status="SUCCESS"/> | |
| <after process="ingestDigiHistRun" status="SUCCESS"/> | |
| <after process="ingestMeritTrendRun" status="SUCCESS"/> | |
| <after process="ingestMeritHistRun" status="SUCCESS"/> | |
| <after process="ingestReconTrendRun" status="SUCCESS"/> | |
| <after process="ingestReconHistRun" status="SUCCESS"/> | |
| <after process="ingestVerifyErrorRun" status="SUCCESS"/> | |
| <after process="ingestVerifyErrorAlarmRun" status="SUCCESS"/> | |
| <after process="ingestVerifyFt1ErrorRun" status="SUCCESS"/> | |
| <after process="ingestVerifyFt1ErrorAlarmRun" status="SUCCESS"/> | |
| <after process="ingestVerifyFt2ErrorRun" status="SUCCESS"/> | |
| <after process="ingestVerifyFt2ErrorAlarmRun" status="SUCCESS"/> | |
| <after process="ingestVerifyMeritErrorRun" status="SUCCESS"/> | |
| <after process="ingestVerifyMeritErrorAlarmRun" status="SUCCESS"/> | |
| <after process="meritHistLogger" status="SUCCESS"/> | |
| <after process="meritTrendLogger" status="SUCCESS"/> | |
| <after process="reconHistLogger" status="SUCCESS"/> | |
| <after process="reconTrendLogger" status="SUCCESS"/> | |
| <after process="registerAcdPlots" status="SUCCESS"/> | |
| <after process="registerCal" status="SUCCESS"/> | |
| <after process="registerElectronFT1" status="SUCCESS"/> | |
| <after process="registerFastMonTuple" status="SUCCESS"/> | |
| <!-- <after process="registerFilteredMerit" status="SUCCESS"/> --> | |
| <after process="registerGcr" status="SUCCESS"/> | |
| <after process="registerM7Hp" status="SUCCESS"/> | |
| <after process="registerMeritHist" status="SUCCESS"/> | |
| <after process="registerOrbitPlot" status="SUCCESS"/> | |
| <after process="registerReconHistAlarmDist" status="SUCCESS"/> | |
| <after process="registerRelation" status="SUCCESS"/> | |
| <after process="registerSolarFlareHist" status="SUCCESS"/> | |
| <after process="registerSolarFlareLog" status="SUCCESS"/> | |
| <after process="registerSolarFlarePlot" status="SUCCESS"/> | |
| <after process="registerVerifyHisto" status="SUCCESS"/> | |
| <after process="scanReconChunks" status="SUCCESS"/> | |
| <after process="suggestQuality" status="SUCCESS"/> | |
| <after process="scanTkrAnalysis" status="SUCCESS"/> | |
| <after process="verifyErrorLogger" status="SUCCESS"/> | |
| <after process="verifyFt1ErrorLogger" status="SUCCESS"/> | |
| <after process="verifyFt2ErrorLogger" status="SUCCESS"/> | |
| <after process="wrapupQuality" status="SUCCESS"/> | |
| <after process="verifyMeritErrorLogger" status="SUCCESS"/> | |
| </depends> | |
| <createsSubtasks> | |
| <subtask>cleanupCompleteRun</subtask> | |
| </createsSubtasks> | |
| </process> | |
| <process name="setStatus" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">checkRun</var> | |
| </variables> | |
| <script><![CDATA[ | |
| def good(): | |
| l1RunStatus = parentPi.getVariable('l1RunStatus') | |
| print 'Setting L1RunStatus for run', runNumber, ' to ', l1RunStatus | |
| runQuality.setL1RunStatus(runNumber, l1RunStatus) | |
| reviewStatus = "Waiting review" | |
| print 'Setting ReviewStatus for run', runNumber, ' to ', reviewStatus | |
| runQuality.setReviewStatus(runNumber, reviewStatus) | |
| return | |
| def bad(): | |
| l1RunStatus = 'Failed' | |
| print 'Setting L1RunStatus for run', runNumber, ' to ', l1RunStatus | |
| runQuality.setL1RunStatus(runNumber, l1RunStatus) | |
| return | |
| currentStream = pipeline.getCurrentStream() | |
| parentPi = currentStream.getProcessInstance(parentProcess) | |
| parentStatus = parentPi.getStatus() | |
| if parentStatus in ['SUCCESS']: | |
| good() | |
| else: | |
| bad() | |
| pass | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="checkRun" status="DONE"/> | |
| <after process="setRunning" status="DONE"/> | |
| </depends> | |
| </process> | |
| <task name="cleanupCompleteRun" version="5.7" type="Data"> | |
| <process name="getTimes" autoRetryMaxAttempts="0"> <!-- Should use genericScript.py with a cleanup routine here --> | |
| <variables> | |
| <var name="l1LockAction">LockDirOnly</var> | |
| <var name="tStart">${pipeline.getCurrentStream().getParentStream().getProcessInstance("findChunks").getVariable("tStart")}</var> | |
| <var name="tStop">${pipeline.getCurrentStream().getParentStream().getProcessInstance("findChunks").getVariable("tStop")}</var> | |
| <var name="mootKey">${pipeline.getCurrentStream().getParentStream().getProcessInstance("findChunks").getVariable("mootKey")}</var> | |
| <var name="mootAlias">${pipeline.getCurrentStream().getParentStream().getProcessInstance("findChunks").getVariable("mootAlias")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/getTimes.py" batchOptions=" -E "${isocRun} ${L1ProcROOT}/lockFile.py" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| </process> | |
| <process name="mergeReconChunks" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <!-- <var name="goodPis">${goodReconPis}</var> --> | |
| <var name="goodPis">${pipeline.getCurrentStream().getParentStream().getProcessInstance("scanReconChunks").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">recon</var> | |
| <var name="L1_recon_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "RECON")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[scratch=250]" -app centos6 "/> | |
| <depends> | |
| <after process="getTimes" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerRecon" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeReconChunks</var> | |
| <var name="fileType">recon</var> | |
| <var name="timeProcess">getTimes</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeReconChunks" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="verifyComp" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="L1_digi_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGI")}</var> | |
| <var name="L1_verifyLog_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYLOG")}</var> | |
| <var name="L1_verifyHisto_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYHISTO")}</var> | |
| <var name="completeness">Complete</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/verify.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="getTimes" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerVerifyHistoComp" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">verifyComp</var> | |
| <var name="fileType">verifyHisto</var> | |
| <var name="timeProcess">getTimes</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="verifyComp" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerVerifyLogComp" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">verifyComp</var> | |
| <var name="fileType">verifyLog</var> | |
| <var name="timeProcess">getTimes</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="verifyComp" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestVerifyErrorRunComp" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">verifyLog</var> | |
| <var name="L1_verifyLog_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYLOG")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyLogComp" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="verifyErrorAlarmComp" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">verifyLog</var> | |
| <var name="outFileTypes">verifyErrorAlarm</var> | |
| <var name="L1_verifyLog_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYLOG")}</var> | |
| <var name="L1_verifyErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyLogComp" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerVerifyErrorAlarmComp" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">verifyErrorAlarmComp</var> | |
| <var name="fileType">verifyErrorAlarm</var> | |
| <var name="timeProcess">getTimes</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="verifyErrorAlarmComp" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="verifyErrorLoggerComp" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">verifyErrorAlarm</var> | |
| <var name="L1_verifyErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyErrorAlarmComp" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestVerifyErrorAlarmRunComp" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">verifyErrorAlarm</var> | |
| <var name="L1_verifyErrorAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "VERIFYERRORALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerVerifyErrorAlarmComp" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="mergeTkrAnalysis" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream().getParentStream().getProcessInstance("scanTkrAnalysis").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">tkrAnalysis</var> | |
| <var name="L1_tkrAnalysis_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRANALYSIS")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <!-- <after process="doChunk.mergeReconCrumbs" status="DONE"/> # This is not really necessary. In the situation it was designed to protect against, we'd rather not run at all. But we can't say that. --> | |
| <after process="getTimes" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerTkrAnalysis" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeTkrAnalysis</var> | |
| <var name="fileType">tkrAnalysis</var> | |
| <var name="timeProcess">getTimes</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeTkrAnalysis" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="tkrMonitor" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="L1_tkrAnalysis_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRANALYSIS")}</var> | |
| <var name="L1_tkrMonitor_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRMONITOR")}</var> | |
| <var name="L1_tkrReport_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRREPORT")}</var> | |
| <!-- <var name="L1_tkrAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRALARM")}</var> --> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/tkrMonitor.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerTkrAnalysis" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerTkrMonitor" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">tkrMonitor</var> | |
| <var name="fileType">tkrMonitor</var> | |
| <var name="timeProcess">getTimes</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="tkrMonitor" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerTkrReport" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">tkrMonitor</var> | |
| <var name="fileType">tkrReport</var> | |
| <var name="timeProcess">getTimes</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="tkrMonitor" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <!-- We don't have TKR alarms yet | |
| <process name="registerTkrAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">tkrMonitor</var> | |
| <var name="fileType">tkrAlarm</var> | |
| <var name="timeProcess">getTimes</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="tkrMonitor" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| --> | |
| <process name="tkrTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">tkrMonitor</var> | |
| <var name="outFileTypes">tkrTrend</var> | |
| <var name="L1_tkrMonitor_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRMONITOR")}</var> | |
| <var name="L1_tkrTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRTREND")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerTkrMonitor" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerTkrTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">tkrTrend</var> | |
| <var name="fileType">tkrTrend</var> | |
| <var name="timeProcess">getTimes</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="tkrTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestTkrTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">tkrTrend</var> | |
| <var name="L1_tkrTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRTREND")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/ingestTrending.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerTkrTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="tkrTrendAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">tkrTrend</var> | |
| <var name="outFileTypes">tkrTrendAlarm</var> | |
| <var name="L1_tkrTrend_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRTREND")}</var> | |
| <var name="L1_tkrTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRTRENDALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerTkrTrend" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerTkrTrendAlarm" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">tkrTrendAlarm</var> | |
| <var name="fileType">tkrTrendAlarm</var> | |
| <var name="timeProcess">getTimes</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="tkrTrendAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="ingestTkrTrendRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">tkrTrendAlarm</var> | |
| <var name="L1_tkrTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRTRENDALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/ingestRun.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerTkrTrendAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="tkrTrendLogger" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">tkrTrendAlarm</var> | |
| <var name="L1_tkrTrendAlarm_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "TKRTRENDALARM")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerTkrTrendAlarm" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="cleanupChunks" autoRetryMaxAttempts="1"> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/deleteDirs.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="registerRecon" status="SUCCESS"/> | |
| <after process="registerTkrReport" status="SUCCESS"/> | |
| <after process="ingestTkrTrend" status="SUCCESS"/> | |
| <after process="ingestTkrTrendRun" status="SUCCESS"/> | |
| <after process="ingestVerifyErrorRunComp" status="SUCCESS"/> | |
| <after process="ingestVerifyErrorAlarmRunComp" status="SUCCESS"/> | |
| <!-- We don't have TKR alarms (yet?) | |
| <after process="registerTkrAlarm" status="SUCCESS"/> | |
| --> | |
| <after process="registerVerifyHistoComp" status="SUCCESS"/> | |
| <after process="tkrTrendLogger" status="SUCCESS"/> | |
| <after process="verifyErrorLoggerComp" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="unLockRun" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="l1LockAction">UnLock</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/lockFile.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="cleanupChunks" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| </task> <!-- cleanupCompleteRun --> | |
| <task name="doChunk" version="5.7" type="Data"> | |
| <variables> | |
| <var name="EVENTSTOSKIP"></var> | |
| </variables> | |
| <prerequisites> | |
| <prerequisite name="CHUNK_ID" type="string"/> | |
| </prerequisites> | |
| <process name="digitization" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| <var name="inFileTypes">event</var> | |
| <var name="L1_event_ver">${pipeline.getCurrentStream().getParentStream().getProcessInstance("findChunks").getVariable("L1_PI_version")}</var> | |
| <var name="outFileTypes">digi</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| </process> | |
| <process name="fakeFT2" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="outFileTypes">ft2Fake</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| </process> | |
| <process name="fastMonTuple" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">event</var> | |
| <var name="L1_event_ver">${pipeline.getCurrentStream().getParentStream().getProcessInstance("findChunks").getVariable("L1_PI_version")}</var> | |
| <var name="reportType">fastMonTuple</var> | |
| <var name="outFileTypes">fastMonTuple/fastMonError</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| </process> | |
| <process name="fastMonHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">fastMonHist</var> | |
| <var name="inFileTypes">fastMonTuple</var> | |
| <var name="outFileTypes">fastMonHist</var> | |
| <var name="L1_fastMonTuple_ver">${pipeline.getCurrentStream.getProcessInstance("fastMonTuple").getVariable("L1_PI_version")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="fastMonTuple" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="fastMonTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">fastMonTuple</var> | |
| <var name="outFileTypes">fastMonTrend</var> | |
| <var name="L1_fastMonTuple_ver">${pipeline.getCurrentStream.getProcessInstance("fastMonTuple").getVariable("L1_PI_version")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="fastMonTuple" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="digiHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">digi</var> | |
| <var name="outFileTypes">digiHist</var> | |
| <var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitization").getVariable("L1_PI_version")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="digitization" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="digiTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">digi</var> | |
| <var name="outFileTypes">digiTrend</var> | |
| <var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitization").getVariable("L1_PI_version")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="digitization" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="calTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">digi</var> | |
| <var name="outFileTypes">calTrend</var> | |
| <var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitization").getVariable("L1_PI_version")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="digitization" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="setupCrumbs" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">digi</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| <var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitization").getVariable("L1_PI_version")}</var> | |
| <var name="L1_ft2Fake_ver">${pipeline.getCurrentStream.getProcessInstance("fakeFT2").getVariable("L1_PI_version")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="digitization" status="SUCCESS"/> | |
| <after process="fakeFT2" status="SUCCESS"/> | |
| </depends> | |
| <createsSubtasks> | |
| <subtask>doCrumb</subtask> | |
| </createsSubtasks> | |
| </process> | |
| <process name="scanReconCrumbs" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doCrumb</var> | |
| <var name="subProcess">recon</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doCrumb.recon" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="allRecon" autoRetryMaxAttempts="1"> | |
| <script><![CDATA[ | |
| # This script is automagically pasted into the XML at install time | |
| print "Placeholder to aggregate dependency flow." | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doCrumb.recon" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="mergeMeritCrumbs" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="outFileTypes">merit</var> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanReconCrumbs").getVariable("goodPis")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="scanReconCrumbs" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="mergeReconCrumbs" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="outFileTypes">recon</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanReconCrumbs").getVariable("goodPis")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="scanReconCrumbs" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="mergeRelationCrumbs" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="outFileTypes">relation</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanReconCrumbs").getVariable("goodPis")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="scanReconCrumbs" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="mergeCalCrumbs" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="outFileTypes">cal</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanReconCrumbs").getVariable("goodPis")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="scanReconCrumbs" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="mergeGcrCrumbs" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="outFileTypes">gcr</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanReconCrumbs").getVariable("goodPis")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="scanReconCrumbs" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="makeFT1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">merit</var> | |
| <var name="outFileTypes">ft1NoDiffRsp</var> | |
| <var name="L1_merit_ver">${pipeline.getCurrentStream.getProcessInstance("mergeMeritCrumbs").getVariable("L1_PI_version")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="mergeMeritCrumbs" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="diffRspFT1" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">ft1NoDiffRsp/ft2Fake</var> | |
| <var name="outFileTypes">ft1BadGti</var> | |
| <var name="L1_ft1NoDiffRsp_ver">${pipeline.getCurrentStream.getProcessInstance("makeFT1").getVariable("L1_PI_version")}</var> | |
| <var name="L1_ft2Fake_ver">${pipeline.getCurrentStream.getProcessInstance("fakeFT2").getVariable("L1_PI_version")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${highPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="makeFT1" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="meritTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">digi/merit</var> | |
| <var name="outFileTypes">meritTrend</var> | |
| <var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitization").getVariable("L1_PI_version")}</var> | |
| <var name="L1_merit_ver">${pipeline.getCurrentStream.getProcessInstance("mergeMeritCrumbs").getVariable("L1_PI_version")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="allRecon" status="SUCCESS"/> | |
| <after process="mergeMeritCrumbs" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="reconHist" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">cal/digi/recon</var> | |
| <var name="outFileTypes">reconHist</var> | |
| <var name="L1_cal_ver">${pipeline.getCurrentStream.getProcessInstance("mergeCalCrumbs").getVariable("L1_PI_version")}</var> | |
| <var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitization").getVariable("L1_PI_version")}</var> | |
| <var name="L1_recon_ver">${pipeline.getCurrentStream.getProcessInstance("mergeReconCrumbs").getVariable("L1_PI_version")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="allRecon" status="SUCCESS"/> | |
| <after process="mergeCalCrumbs" status="SUCCESS"/> | |
| <after process="mergeReconCrumbs" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="reconTrend" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">cal/digi/recon</var> | |
| <var name="outFileTypes">reconTrend</var> | |
| <var name="L1_cal_ver">${pipeline.getCurrentStream.getProcessInstance("mergeCalCrumbs").getVariable("L1_PI_version")}</var> | |
| <var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitization").getVariable("L1_PI_version")}</var> | |
| <var name="L1_recon_ver">${pipeline.getCurrentStream.getProcessInstance("mergeReconCrumbs").getVariable("L1_PI_version")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="allRecon" status="SUCCESS"/> | |
| <after process="mergeCalCrumbs" status="SUCCESS"/> | |
| <after process="mergeReconCrumbs" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="tkrAnalysis" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">digi/recon</var> | |
| <var name="outFileTypes">tkrAnalysis</var> | |
| <var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitization").getVariable("L1_PI_version")}</var> | |
| <var name="L1_recon_ver">${pipeline.getCurrentStream.getProcessInstance("mergeReconCrumbs").getVariable("L1_PI_version")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="allRecon" status="SUCCESS"/> | |
| <after process="mergeReconCrumbs" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="checkChunk" autoRetryMaxAttempts="1"> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/checkChunk.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="fastMonHist" status="SUCCESS"/> | |
| <after process="fastMonTrend" status="SUCCESS"/> | |
| <after process="digiHist" status="SUCCESS"/> | |
| <after process="digiTrend" status="SUCCESS"/> | |
| <after process="calTrend" status="SUCCESS"/> | |
| <after process="diffRspFT1" status="SUCCESS"/> | |
| <after process="mergeGcrCrumbs" status="SUCCESS"/> | |
| <after process="mergeRelationCrumbs" status="SUCCESS"/> | |
| <after process="meritTrend" status="SUCCESS"/> | |
| <after process="reconHist" status="SUCCESS"/> | |
| <after process="reconTrend" status="SUCCESS"/> | |
| <after process="tkrAnalysis" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <task name="doCrumb" version="5.7" type="Data"> | |
| <prerequisites> | |
| <prerequisite name="CRUMB_ID" type="string"/> | |
| <prerequisite name="crumbEvents" type="string"/> | |
| <prerequisite name="crumbStart" type="string"/> | |
| </prerequisites> | |
| <process name="recon" autoRetryMaxAttempts="1"> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/recon.py" batchOptions=" -q glastdataq -sp ${reconPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| </process> | |
| </task> <!-- doCrumb --> | |
| </task> <!-- doChunk --> | |
| </task> <!-- doRun --> | |
| <task name="doLci" version="5.7" type="Data"> | |
| <variables> | |
| <var name="dataCatDir">/Data/Flight/Level1/${DATASOURCE}</var> | |
| </variables> | |
| <prerequisites> | |
| <prerequisite name="DATASOURCE" type="string"/> | |
| <prerequisite name="RUNID" type="string"/> | |
| <prerequisite name="RUNSTATUS" type="string"/> | |
| </prerequisites> | |
| <process name="findChunksLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="l1LockAction">LockDirOnly</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 --add-env=flightops python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -E "${isocRun} ${L1ProcROOT}/lockFile.py" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <createsSubtasks> | |
| <subtask>doChunkLci</subtask> | |
| </createsSubtasks> | |
| </process> | |
| <process name="setRunningLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="l1RunStatus">Running</var> | |
| </variables> | |
| <script><![CDATA[ | |
| from java.util import HashMap | |
| def setIfUnset(container, key, value): | |
| oldValue = container.get(key) | |
| if oldValue is None or oldValue == 'Unset': | |
| print 'Setting %s to %s.' % (key, value) | |
| container[key] = value | |
| else: print 'Current value of %s is %s, not changing it.' % (key, oldValue) | |
| return | |
| print 'Setting L1RunStatus for run', runNumber, ' to ', l1RunStatus | |
| runQuality.setL1RunStatus(runNumber, l1RunStatus) | |
| reviewStatus = "Unset" | |
| print 'Setting ReviewStatus for run', runNumber, ' to ', reviewStatus | |
| runQuality.setReviewStatus(runNumber, reviewStatus) | |
| newQuality = 'Good' | |
| quality = runQuality.getRunQuality(runNumber) | |
| print 'Current quality for run %s is %s.' % (runNumber, quality) | |
| if quality is None or quality == 'Unset': | |
| print 'Setting quality for run %s to %s.' % (runNumber, newQuality) | |
| runQuality.setRunQuality(runNumber, newQuality) | |
| quality = newQuality | |
| else: print 'Not changing run quality.' | |
| pipeline.setVariable('runQuality', quality) | |
| fields = ['ACDQuality', 'CALQuality', 'GPSQuality', 'TRACKERQuality', | |
| 'TRIGGERQuality'] | |
| metaData = runQuality.getMetaData(runNumber) | |
| print 'old metaData:', metaData | |
| for key in fields: setIfUnset(metaData, key, newQuality) | |
| print 'new metadata:', metaData | |
| runQuality.setOrUpdateMetaData(runNumber, metaData) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="findChunksLci" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="copyM7HpLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="L1_magic7Hp_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "MAGIC7HP")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/copyM7.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 " /> | |
| <depends> | |
| <after process="findChunksLci" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerM7HpLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">copyM7HpLci</var> | |
| <var name="fileType">magic7Hp</var> | |
| <var name="timeProcess">findChunksLci</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="copyM7HpLci" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanDigiLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunkLci</var> | |
| <var name="subProcess">digitizationLci</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunkLci.digitizationLci" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeDigiLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanDigiLci").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">digi</var> | |
| <var name="L1_digi_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGI")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[scratch=250]" -app centos6 "/> | |
| <depends> | |
| <after process="scanDigiLci" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerDigiLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeDigiLci</var> | |
| <var name="fileType">digi</var> | |
| <var name="timeProcess">findChunksLci</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeDigiLci" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanDigiHistLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunkLci</var> | |
| <var name="subProcess">digiHistLci</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunkLci.digiHistLci" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeDigiHistLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanDigiHistLci").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">digiHist</var> | |
| <var name="L1_digiHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "DIGIHIST")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[scratch=10]" -app centos6 "/> | |
| <depends> | |
| <after process="scanDigiHistLci" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerDigiHistLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeDigiHistLci</var> | |
| <var name="fileType">digiHist</var> | |
| <var name="timeProcess">findChunksLci</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeDigiHistLci" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanFastMonTupleLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunkLci</var> | |
| <var name="subProcess">fastMonTupleLci</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunkLci.fastMonTupleLci" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeFastMonErrorLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanFastMonTupleLci").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">fastMonError</var> | |
| <var name="L1_fastMonError_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONERROR")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="scanFastMonTupleLci" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFastMonErrorLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeFastMonErrorLci</var> | |
| <var name="fileType">fastMonError</var> | |
| <var name="timeProcess">findChunksLci</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeFastMonErrorLci" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="mergeFastMonTupleLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanFastMonTupleLci").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">fastMonTuple</var> | |
| <var name="L1_fastMonTuple_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONTUPLE")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[scratch=10]" -app centos6 "/> | |
| <depends> | |
| <after process="scanFastMonTupleLci" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFastMonTupleLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeFastMonTupleLci</var> | |
| <var name="fileType">fastMonTuple</var> | |
| <var name="timeProcess">findChunksLci</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeFastMonTupleLci" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="scanFastMonHistLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="subTask">doChunkLci</var> | |
| <var name="subProcess">fastMonHistLci</var> | |
| </variables> | |
| <script><![CDATA[ | |
| finalStates = ['SUCCESS', 'FAILED', 'TERMINATED', 'CANCELED'] | |
| def getStreams(): | |
| thisStream = pipeline.getCurrentStream() | |
| thisId = thisStream.getId() | |
| thisPath = thisStream.getIdPath() | |
| print 'stream path: %s' % thisPath | |
| thisTask = thisStream.getTask() | |
| print 'this task: %s' % thisTask | |
| delivsToIgnore = str(deliveriesToIgnore) | |
| badDelivs = [int(deliv) for deliv in delivsToIgnore.split(':') if deliv] | |
| print 'Skipping deliveries', badDelivs | |
| rdlStreams = [] | |
| allStreams = thisTask.getStreams(thisId) | |
| for stream in allStreams: | |
| thatPath = stream.getIdPath() | |
| status = stream.getStatus() | |
| print stream, thatPath, status | |
| if (thatPath == thisPath or status in finalStates) and thatPath[0] not in badDelivs: | |
| rdlStreams.append(stream) | |
| pass | |
| continue | |
| return rdlStreams | |
| def getChunkPis(): | |
| """Examine chunk-level PIs to see what files we expect to find for merging | |
| and their versions. | |
| """ | |
| rdlStreams = getStreams() | |
| pis = [] | |
| for rdl in rdlStreams: | |
| rdlChunks = rdl.getSubStreams(subTask) | |
| rdlPis = [(chunk.getId(), chunk.getProcessInstance(subProcess)) for chunk in rdlChunks] | |
| pis.extend(rdlPis) | |
| continue | |
| return pis | |
| def getCrumbPis(): | |
| thisStream = pipeline.getCurrentStream() | |
| subStreams = thisStream.getSubStreams(subTask) | |
| pis = [] | |
| for stream in subStreams: | |
| print 'subStream: %s' % stream | |
| sId = stream.getId() | |
| print 'subId: %s' % sId | |
| sPi = stream.getProcessInstance(subProcess) | |
| print 'subPi: %s' % sPi | |
| pis.append((sId, sPi)) | |
| return pis | |
| getPis = { | |
| 'doChunk': getChunkPis, | |
| 'doChunkLci': getChunkPis, | |
| 'doCrumb': getCrumbPis | |
| }[subTask] | |
| pis = getPis() | |
| pis.sort() | |
| versionTags = [] | |
| print 'Exit codes for subprocesses %s.%s:' % (subTask, subProcess) | |
| for streamId, subPi in pis: | |
| ec = subPi.getExitCode() | |
| statStr = subPi.getStatus() | |
| print streamId, ec, statStr | |
| if ec or statStr != 'SUCCESS': | |
| continue | |
| varNames = ['L1_PI_ID', 'L1_PI_version'] | |
| # tup = tuple([subPi.getVariable(name) for name in varNames]) | |
| tup = [] | |
| varMap = subPi.getVariables() | |
| print 'varMap: %s' % varMap | |
| #raise SystemExit, 'quit here' | |
| for name in varNames: | |
| print 'key: %s' % name | |
| value = subPi.getVariable(name) | |
| print 'value: %s' % value | |
| tup.append(value) | |
| tup = tuple(tup) | |
| print tup | |
| if None in tup: | |
| continue | |
| tag = '%s:%s' % tup | |
| versionTags.append(tag) | |
| continue | |
| if not versionTags: raise SystemExit, "No successful jobs, failing." | |
| goodPis = ','.join(versionTags) | |
| pipeline.setVariable('goodPis', goodPis) | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="doChunkLci.fastMonHistLci" status="DONE"/> | |
| </depends> | |
| </process> | |
| <process name="mergeFastMonHistLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="goodPis">${pipeline.getCurrentStream.getProcessInstance("scanFastMonHistLci").getVariable("goodPis")}</var> | |
| <var name="outFileTypes">fastMonHist</var> | |
| <var name="L1_fastMonHist_ver">${datacatalog.getDatasetLatestVersion(RUNID, "/Data/Flight/Level1/"+DATASOURCE, "FASTMONHIST")}</var> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[scratch=10]" -app centos6 "/> | |
| <depends> | |
| <after process="scanFastMonHistLci" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="registerFastMonHistLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">mergeFastMonHistLci</var> | |
| <var name="fileType">fastMonHist</var> | |
| <var name="timeProcess">findChunksLci</var> | |
| </variables> | |
| <script><![CDATA[ | |
| """@brief Register output data with the data server. | |
| @author W. Focke <focke@slac.stanford.edu> | |
| """ | |
| # This script is automagically pasted into the XML at install time | |
| from java.util import HashMap | |
| def getVar(fileType, name): | |
| mangledName = '_'.join([nameManglingPrefix, fileType, name]) | |
| value = parentPI.getVariable(mangledName) | |
| return value | |
| currentStream = pipeline.getCurrentStream() | |
| parentPI = currentStream.getProcessInstance(parentProcess) | |
| runNumber = int(RUNID[1:]) | |
| dsName = RUNID | |
| fileFormat = getVar(fileType, 'format') | |
| dcType = getVar(fileType, 'dcType') | |
| dcGroup = getVar(fileType, 'group') | |
| site = getVar(fileType, 'site') | |
| fileName = getVar(fileType, 'fileName') | |
| version = getVar(fileType, 'ver') | |
| fcPi = currentStream.getProcessInstance(timeProcess) | |
| lessBrokenTStart = fcPi.getVariable('tStart') | |
| lessBrokenTStop = fcPi.getVariable('tStop') | |
| mootAlias = fcPi.getVariable('mootAlias') | |
| mootKey = fcPi.getVariable('mootKey') | |
| attributes = HashMap() | |
| attributes.put('sCreator', creator) | |
| attributes.put('sDataSource', DATASOURCE) | |
| attributes.put('nDownlink', DOWNLINK_ID) | |
| attributes.put('sIntent', mootAlias) | |
| attributes.put('nMetStart', lessBrokenTStart) | |
| attributes.put('nMetStop', lessBrokenTStop) | |
| attributes.put('nMootKey', mootKey) | |
| attributes.put('nRun', runNumber) | |
| mdRepr = getVar(fileType, 'metadata') | |
| if mdRepr: | |
| metadata = eval(mdRepr) | |
| for key, value in metadata.items(): attributes.put(key, value) | |
| print attributes | |
| dsNew = datacatalog.newDataset(dsName, fileFormat, dcType, dataCatDir, dcGroup, site, fileName) | |
| dsNew.setVersionID(version) | |
| ds = datacatalog.registerDataset(dsNew, attributes); | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="mergeFastMonHistLci" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="checkRunLci" autoRetryMaxAttempts="1"> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=oracle11 python2.6 ${L1ProcROOT}/checkRun.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="doChunkLci.checkChunkLci" status="SUCCESS"/> | |
| <after process="registerDigiLci" status="SUCCESS"/> | |
| <after process="registerDigiHistLci" status="SUCCESS"/> | |
| <after process="registerFastMonErrorLci" status="SUCCESS"/> | |
| <after process="registerFastMonHistLci" status="SUCCESS"/> | |
| <after process="registerFastMonTupleLci" status="SUCCESS"/> | |
| <after process="registerM7HpLci" status="SUCCESS"/> | |
| </depends> | |
| <createsSubtasks> | |
| <subtask>cleanupCompleteRunLci</subtask> | |
| </createsSubtasks> | |
| </process> | |
| <process name="setStatusLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="parentProcess">checkRunLci</var> | |
| </variables> | |
| <script><![CDATA[ | |
| def good(): | |
| l1RunStatus = parentPi.getVariable('l1RunStatus') | |
| print 'Setting L1RunStatus for run', runNumber, ' to ', l1RunStatus | |
| runQuality.setL1RunStatus(runNumber, l1RunStatus) | |
| reviewStatus = "Waiting review" | |
| print 'Setting ReviewStatus for run', runNumber, ' to ', reviewStatus | |
| runQuality.setReviewStatus(runNumber, reviewStatus) | |
| return | |
| def bad(): | |
| l1RunStatus = 'Failed' | |
| print 'Setting L1RunStatus for run', runNumber, ' to ', l1RunStatus | |
| runQuality.setL1RunStatus(runNumber, l1RunStatus) | |
| return | |
| currentStream = pipeline.getCurrentStream() | |
| parentPi = currentStream.getProcessInstance(parentProcess) | |
| parentStatus = parentPi.getStatus() | |
| if parentStatus in ['SUCCESS']: | |
| good() | |
| else: | |
| bad() | |
| pass | |
| ]]> | |
| </script> | |
| <depends> | |
| <after process="checkRunLci" status="DONE"/> | |
| <after process="setRunningLci" status="DONE"/> | |
| </depends> | |
| </process> | |
| <task name="cleanupCompleteRunLci" version="5.7" type="Data"> | |
| <process name="cleanupChunksLci" autoRetryMaxAttempts="1"> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/deleteDirs.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| </process> | |
| </task> <!-- cleanupCompleteRunLci --> | |
| <task name="doChunkLci" version="5.7" type="Data"> | |
| <variables> | |
| <var name="EVENTSTOSKIP"></var> | |
| </variables> | |
| <prerequisites> | |
| <prerequisite name="CHUNK_ID" type="string"/> | |
| </prerequisites> | |
| <process name="digitizationLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="LD_LIBRARY_PATH">/afs/slac/g/glast/ground/releases/volume11/L1Proc/5.7/lib/redhat6-x86_64-64bit-gcc44-Optimized:/afs/slac.stanford.edu/g/glast/ground/GLAST_EXT/redhat6-x86_64-64bit-gcc44/ROOT/v5.34.03-gr01/lib:/afs/slac.stanford.edu/g/glast/ground/releases/volume14/GlastRelease-20-10-04-gr02/lib/redhat6-x86_64-64bit-gcc44-Optimized</var> | |
| <var name="inFileTypes">event</var> | |
| <var name="L1_event_ver">${pipeline.getCurrentStream().getParentStream().getProcessInstance("findChunksLci").getVariable("L1_PI_version")}</var> | |
| <var name="outFileTypes">digi</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| </process> | |
| <process name="fastMonTupleLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">event</var> | |
| <var name="L1_event_ver">${pipeline.getCurrentStream().getParentStream().getProcessInstance("findChunksLci").getVariable("L1_PI_version")}</var> | |
| <var name="reportType">fastMonTuple</var> | |
| <var name="outFileTypes">fastMonTuple/fastMonError</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| </process> | |
| <process name="fastMonHistLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="reportType">fastMonHist</var> | |
| <var name="inFileTypes">fastMonTuple</var> | |
| <var name="outFileTypes">fastMonHist</var> | |
| <var name="L1_fastMonTuple_ver">${pipeline.getCurrentStream.getProcessInstance("fastMonTupleLci").getVariable("L1_PI_version")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} --add-env=flightops python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="fastMonTupleLci" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="digiHistLci" autoRetryMaxAttempts="1"> | |
| <variables> | |
| <var name="inFileTypes">digi</var> | |
| <var name="outFileTypes">digiHist</var> | |
| <var name="L1_digi_ver">${pipeline.getCurrentStream.getProcessInstance("digitizationLci").getVariable("L1_PI_version")}</var> | |
| </variables> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/genericScript.py" batchOptions=" -q glastdataq -sp ${midPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="digitizationLci" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| <process name="checkChunkLci" autoRetryMaxAttempts="1"> | |
| <job executable="${L1ProcROOT}/tools/wrapper ${isocRun} python2.6 ${L1ProcROOT}/checkChunk.py" batchOptions=" -q glastdataq -sp ${standardPriority} -R "select[${standardSelect}] rusage[${standardRusage}]" -app centos6 "/> | |
| <depends> | |
| <after process="fastMonHistLci" status="SUCCESS"/> | |
| <after process="digiHistLci" status="SUCCESS"/> | |
| </depends> | |
| </process> | |
| </task> <!-- doChunkLci --> | |
| </task> <!-- doLci --> | |
| </task> <!-- L1Proc --> | |
| </pipeline> | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment