fix negative times and bad bucket sizes

iceprod/prom_utils.py

@@ -17,6 +17,9 @@ class HistogramBuckets:
     #: Timer bucket up to 1 second
     SECOND = [.0001, .0005, .001, .0025, .005, .0075, .01, .025, .05, .075, .1, .25, .5, .75, 1]
 
+    #: Timer bucket up to 10 seconds
+    TENSECOND = [.001, .0025, .005, .0075, .01, .025, .05, .075, .1, .25, .5, .75, 1, 2.5, 5, 10]
+
     #: Timer bucket up to 1 minute
     MINUTE = [.1, .5, 1, 2.5, 5, 7.5, 10, 15, 20, 25, 30, 45, 60]
 

iceprod/server/plugins/condor.py

@@ -957,10 +957,10 @@ async def check_history(self, prom_histogram):
 
                 # do timing manually to also count generator time per job
                 next_time = time.monotonic()
-                prom_histogram.observe(last_time - next_time)
+                prom_histogram.observe(next_time - last_time)
                 last_time = next_time
 
-    @AsyncPromTimer(lambda self: self.prometheus.histogram('iceprod_grid_check_iceprod', 'IceProd grid check calls', buckets=HistogramBuckets.MINUTE))
+    @AsyncPromTimer(lambda self: self.prometheus.histogram('iceprod_grid_check_iceprod', 'IceProd grid check calls', buckets=HistogramBuckets.TENSECOND))
     async def check_iceprod(self):
         """
         Sync with iceprod server status.
@@ -983,7 +983,7 @@ async def check_iceprod(self):
                     )
                     tg.create_task(self.task_reset(job, reason='task missing from HTCondor queue'))
 
-    @PromWrapper(lambda self: self.prometheus.histogram('iceprod_grid_check_submit_dir', 'IceProd grid check calls', buckets=HistogramBuckets.MINUTE))
+    @PromWrapper(lambda self: self.prometheus.histogram('iceprod_grid_check_submit_dir', 'IceProd grid check calls', buckets=HistogramBuckets.TENSECOND))
     async def check_submit_dir(self, prom_histogram):
         """
         Return directory paths that should be cleaned up.