diff --git a/src/python/WMCore/ReqMgr/CherryPyThreads/HeartbeatMonitor.py b/src/python/WMCore/ReqMgr/CherryPyThreads/HeartbeatMonitor.py index 8f549ae229..d02b081b2c 100644 --- a/src/python/WMCore/ReqMgr/CherryPyThreads/HeartbeatMonitor.py +++ b/src/python/WMCore/ReqMgr/CherryPyThreads/HeartbeatMonitor.py @@ -5,7 +5,7 @@ import time from WMCore.REST.HeartbeatMonitorBase import HeartbeatMonitorBase -from WMCore.ReqMgr.DataStructs.RequestStatus import ACTIVE_STATUS +from WMCore.ReqMgr.DataStructs.RequestStatus import NON_ARCHIVED_STATUS from WMCore.Services.WMStatsServer.WMStatsServer import WMStatsServer @@ -45,7 +45,7 @@ def initMetrics(): "requestsByStatusAndPrio": {}, "requestsByStatusAndNumEvts": {}} - for st in ACTIVE_STATUS: + for st in NON_ARCHIVED_STATUS: results["requestsByStatus"].setdefault(st, 0) results["requestsByStatusAndNumEvts"].setdefault(st, 0) results["requestsByStatusAndCampaign"].setdefault(st, {}) diff --git a/src/python/WMCore/ReqMgr/DataStructs/Request.py b/src/python/WMCore/ReqMgr/DataStructs/Request.py index 692cf124a9..6c49181bc5 100644 --- a/src/python/WMCore/ReqMgr/DataStructs/Request.py +++ b/src/python/WMCore/ReqMgr/DataStructs/Request.py @@ -21,7 +21,7 @@ import time from copy import deepcopy from WMCore.REST.Auth import get_user_info -from WMCore.ReqMgr.DataStructs.RequestStatus import REQUEST_START_STATE, ACTIVE_STATUS_FILTER +from WMCore.ReqMgr.DataStructs.RequestStatus import REQUEST_START_STATE def initialize_request_args(request, config): @@ -159,20 +159,6 @@ def generateRequestName(request): request["RequestName"] += "_%s_%s" % (currentTime, seconds) -def protectedLFNs(requestInfo): - reqData = RequestInfo(requestInfo) - result = [] - if reqData.andFilterCheck(ACTIVE_STATUS_FILTER): - outs = requestInfo.get('OutputDatasets', []) - base = requestInfo.get('UnmergedLFNBase', '/store/unmerged') - for out in outs: - dsn, ps, tier = out.split('/')[1:] - acq, rest = ps.split('-', 1) - dirPath = '/'.join([base, acq, dsn, tier, rest]) - result.append(dirPath) - return result - - class RequestInfo(object): """ Wrapper class for Request data diff --git a/src/python/WMCore/ReqMgr/DataStructs/RequestStatus.py b/src/python/WMCore/ReqMgr/DataStructs/RequestStatus.py index 281dee4e7b..0294f1db3d 100644 --- a/src/python/WMCore/ReqMgr/DataStructs/RequestStatus.py +++ b/src/python/WMCore/ReqMgr/DataStructs/RequestStatus.py @@ -2,7 +2,7 @@ Definition of valid status values for a request and valid status transitions. """ - +from copy import copy from future.utils import viewitems # make this list to ensure insertion order here @@ -68,9 +68,7 @@ "rejected", "aborted"] -ACTIVE_STATUS = ["new", - "assignment-approved", - "assigned", +ACTIVE_STATUS = ["assigned", "staging", "staged", "acquired", @@ -85,6 +83,8 @@ "aborted-completed", "rejected"] +NON_ARCHIVED_STATUS = ["new", "assignment-approved"] + copy(ACTIVE_STATUS) + ### WMSTATS_JOB_INFO + WMSTATS_NO_JOB_INFO is meant to be equal to ACTIVE_STATUS WMSTATS_JOB_INFO = ["running-open", "running-closed", @@ -92,9 +92,7 @@ "completed", "closed-out"] -WMSTATS_NO_JOB_INFO = ["new", - "assignment-approved", - "assigned", +WMSTATS_NO_JOB_INFO = ["assigned", "staging", "staged", "acquired", @@ -164,8 +162,8 @@ # is name of the status REQUEST_STATE_LIST = list(REQUEST_STATE_TRANSITION) -ACTIVE_STATUS_FILTER = {"RequestStatus": ['assignment-approved', 'assigned', 'staging', 'staged', - 'failed', 'acquired', 'running-open', 'running-closed', +ACTIVE_STATUS_FILTER = {"RequestStatus": ['assigned', 'staging', 'staged', 'failed', 'acquired', + 'running-open', 'running-closed', 'force-complete', 'completed', 'closed-out']} diff --git a/src/python/WMCore/WMStats/DataStructs/DataCache.py b/src/python/WMCore/WMStats/DataStructs/DataCache.py index a8c238d249..a6e6deec94 100644 --- a/src/python/WMCore/WMStats/DataStructs/DataCache.py +++ b/src/python/WMCore/WMStats/DataStructs/DataCache.py @@ -2,7 +2,29 @@ from future.utils import viewitems import time -from WMCore.ReqMgr.DataStructs.Request import RequestInfo, protectedLFNs +from WMCore.ReqMgr.DataStructs.Request import RequestInfo +from WMCore.ReqMgr.DataStructs.RequestStatus import ACTIVE_STATUS_FILTER + + +def protectedLFNs(requestInfo): + """ + Parses a workflow description and provides a list of the + final (excluding transient data) output LFNs. + :param requestInfo: a dictionary with the workflow description + :return: a list of strings for the protected lfns + """ + reqData = RequestInfo(requestInfo) + result = [] + if reqData.andFilterCheck(ACTIVE_STATUS_FILTER): + outs = requestInfo.get('OutputDatasets', []) + base = requestInfo.get('UnmergedLFNBase', '/store/unmerged') + for out in outs: + dsn, ps, tier = out.split('/')[1:] + acq, rest = ps.split('-', 1) + dirPath = '/'.join([base, acq, dsn, tier, rest]) + result.append(dirPath) + return result + class DataCache(object): # TODO: need to change to store in db instead of storing in the memory diff --git a/src/python/WMCore/WMStats/Service/ActiveRequestJobInfo.py b/src/python/WMCore/WMStats/Service/ActiveRequestJobInfo.py index 829f387c06..59cb5e889b 100644 --- a/src/python/WMCore/WMStats/Service/ActiveRequestJobInfo.py +++ b/src/python/WMCore/WMStats/Service/ActiveRequestJobInfo.py @@ -74,7 +74,7 @@ def validate(self, apiobj, method, api, param, safe): @restcall(formats=[('text/plain', PrettyJSONFormat()), ('application/json', JSONFormat())]) @tools.expires(secs=-1) def get(self): - # This assumes DataCahe is periodically updated. + # This assumes DataCache is periodically updated. # If data is not updated, need to check, dataCacheUpdate log if DataCache.isEmpty(): raise DataCacheEmpty()