Retry API calls in case of failure

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
         poetry-version: [1.1.0]
         os: [ubuntu-latest, windows-latest]
     runs-on: ${{ matrix.os }}
@@ -29,7 +29,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7"]
+        python-version: ["3.11"]
         poetry-version: [1.1.0]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
@@ -48,7 +48,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
         poetry-version: [1.1.0]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}

apis/phabricator.py

@@ -7,6 +7,7 @@
 import json
 import platform
 
+from components.utilities import retry
 from components.logging import logEntryExit, LogLevel
 from components.providerbase import BaseProvider, INeedsCommandProvider, INeedsLoggingProvider
 
@@ -36,6 +37,7 @@ def __init__(self, config):
             self.url = config['url']
 
     @logEntryExit
+    @retry
     def submit_patches(self, bug_id, has_patches):
         phab_revisions = []
 
@@ -87,6 +89,7 @@ def submit_to_phabricator(rev_id):
         return phab_revisions
 
     @logEntryExit
+    @retry
     def set_reviewer(self, phab_revision, phab_username):
         # We have to call a different API endpoint if this is a review group
         if phab_username[0] == "#":
@@ -120,6 +123,7 @@ def set_reviewer(self, phab_revision, phab_username):
             raise Exception("Got an error from phabricator when trying to set reviewers to %s (%s) for %s: %s" % (phab_username, phid, phab_revision, result))
 
     @logEntryExit
+    @retry
     def abandon(self, phab_revision):
         cmd = "echo " + quote_echo_string("""{"transactions": [{"type":"abandon", "value":true}],"objectIdentifier": "%s"}""" % phab_revision)
         cmd += " | %s call-conduit --conduit-uri=%s differential.revision.edit --""" % (_arc(), self.url)

apis/taskcluster.py

@@ -12,7 +12,7 @@
 from collections import defaultdict
 from urllib.parse import quote_plus
 
-from components.utilities import Struct, merge_dictionaries, PUSH_HEALTH_IGNORED_DICTS, PUSH_HEALTH_IGNORED_KEYS
+from components.utilities import retry, Struct, merge_dictionaries, PUSH_HEALTH_IGNORED_DICTS, PUSH_HEALTH_IGNORED_KEYS
 from components.logging import logEntryExit, logEntryExitNoArgs, LogLevel
 from components.providerbase import BaseProvider, INeedsCommandProvider, INeedsLoggingProvider
 
@@ -149,6 +149,7 @@ def _vcs_setup(self):
         self._vcs_setup_initialized = True
 
     @logEntryExit
+    @retry
     def submit_to_try(self, library, platform_filter, recursed=0):
         self._vcs_setup()
         if not platform_filter:
@@ -171,12 +172,9 @@ def submit_to_try(self, library, platform_filter, recursed=0):
         else:
             try_arguments = ["./mach", "try", "auto"] + platform_filter_args
 
-        ret = self.run(try_arguments, clean_return=False if recursed < 5 else True)
+        ret = self.run(try_arguments, clean_return=True)
         output = ret.stdout.decode()
 
-        if "timed out waiting for lock held by" in output:
-            return self.submit_to_try(library, platform_filter, recursed + 1)
-
         isNext = False
         try_link = None
         for line in output.split("\n"):

components/logging.py

@@ -36,7 +36,7 @@ def func_wrapper(*args, **kwargs):
         obj.logger.log("Beginning %s" % func.__qualname__, level=LogLevel.Info)
         obj.logger.log(" Arguments: %s" % (str(args) + " " + str(kwargs) if print_arg_list else "[Omitted %s args]" % str(len(args) + len(kwargs))), level=LogLevel.Debug)
         ret = func(*args, **kwargs)
-        if type(ret) == list:
+        if ret is list:
             obj.logger.log("Function returned a list of %s objects" % len(ret), level=LogLevel.Debug)
         else:
             return_string = str(ret)

components/utilities.py

@@ -7,6 +7,7 @@
 import copy
 import pickle
 import functools
+import time
 
 from dateutil.parser import parse
 
@@ -138,3 +139,30 @@ def decorate(func):
         return func
 
     return decorate
+
+
+# Retry calling a function `times` times, sleeping between each tries, with an exponential backoff
+# This is to be used on API calls, that are likely to fail
+
+
+def retry(_func=None, *, times=10, sleep_s=1, exp=2):
+    def decorator_retry(func):
+        @functools.wraps(func)
+        def wrapper_retry(*args, **kwargs):
+            retries_try = times
+            sleep_duration = sleep_s
+            while retries_try > 0:
+                try:
+                    return func(*args, **kwargs)
+                except BaseException as e:
+                    retries_try -= 1
+                    time.sleep(sleep_duration)
+                    sleep_duration *= exp
+                    if retries_try == 0:
+                        raise e
+        return wrapper_retry
+
+    if _func is None:
+        return decorator_retry
+    else:
+        return decorator_retry(_func)

tests/functionality_all_platforms.py

@@ -640,6 +640,57 @@ def treeherder(request_type, fullpath):
         finally:
             self._cleanup(u, expected_values)
 
+    # Fail the first time, then work.
+    @logEntryExitHeaderLine
+    def testRetryFunctionality(self):
+        @treeherder_response
+        def treeherder(request_type, fullpath):
+            if request_type == TYPE_HEALTH:
+                return "health_classified_failures.txt"
+            else:  # TYPE_JOBS
+                if treeherder.jobs_calls == 0:
+                    return "jobs_still_running.txt"
+                return "jobs_classified_failures.txt"
+
+        try_fails = 0
+
+        def try_submit(cmd):
+            nonlocal try_fails
+            if try_fails == 0:
+                try_fails += 1
+                raise Exception("No worky!")
+            if "./mach try auto" in cmd:
+                return TRY_OUTPUT(expected_values.try_revision_id())
+            elif "./mach try fuzzy" in cmd:
+                return TRY_OUTPUT(expected_values.try_revision_id(), False)
+
+        library_filter = 'cubeb-query'
+        (u, expected_values, _check_jobs) = self._setup(
+            library_filter,
+            lambda b: ["e152bb86666565ee6619c15f60156cd6c79580a9|2021-02-09 15:30:04 -0500|2021-02-12 17:40:01 +0000"],
+            lambda: 50,  # get_filed_bug_id_func,
+            lambda b: [],  # filed_bug_ids_func
+            treeherder,
+            command_callbacks={'try_submit': try_submit}
+        )
+        try:
+            # Run it
+            u.run(library_filter=library_filter)
+            # Check that we created the job successfully
+            _check_jobs(JOBSTATUS.AWAITING_SECOND_PLATFORMS_TRY_RESULTS, JOBOUTCOME.PENDING)
+
+            # Run it again, this time we'll tell it the jobs are still in process
+            u.run(library_filter=library_filter)
+            # Should still be Awaiting Try Results
+            _check_jobs(JOBSTATUS.AWAITING_SECOND_PLATFORMS_TRY_RESULTS, JOBOUTCOME.PENDING)
+
+            # Run it again, this time we'll tell it the jobs succeeded
+            u.run(library_filter=library_filter)
+            # Should be DONE
+            _check_jobs(JOBSTATUS.DONE, JOBOUTCOME.CLASSIFIED_FAILURES)
+        finally:
+            self._cleanup(u, expected_values)
+
     @logEntryExitHeaderLine
     def testAllNewFuzzyPathJobs(self):
         @treeherder_response