diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b58223a0..c8c65b83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] poetry-version: [1.1.0] os: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.os }} @@ -29,7 +29,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7"] + python-version: ["3.11"] poetry-version: [1.1.0] os: [ubuntu-latest] runs-on: ${{ matrix.os }} @@ -48,7 +48,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] poetry-version: [1.1.0] os: [ubuntu-latest] runs-on: ${{ matrix.os }} diff --git a/apis/phabricator.py b/apis/phabricator.py index a534beb1..de38a84a 100644 --- a/apis/phabricator.py +++ b/apis/phabricator.py @@ -7,6 +7,7 @@ import json import platform +from components.utilities import retry from components.logging import logEntryExit, LogLevel from components.providerbase import BaseProvider, INeedsCommandProvider, INeedsLoggingProvider @@ -36,6 +37,7 @@ def __init__(self, config): self.url = config['url'] @logEntryExit + @retry def submit_patches(self, bug_id, has_patches): phab_revisions = [] @@ -87,6 +89,7 @@ def submit_to_phabricator(rev_id): return phab_revisions @logEntryExit + @retry def set_reviewer(self, phab_revision, phab_username): # We have to call a different API endpoint if this is a review group if phab_username[0] == "#": @@ -120,6 +123,7 @@ def set_reviewer(self, phab_revision, phab_username): raise Exception("Got an error from phabricator when trying to set reviewers to %s (%s) for %s: %s" % (phab_username, phid, phab_revision, result)) @logEntryExit + @retry def abandon(self, phab_revision): cmd = "echo " + quote_echo_string("""{"transactions": [{"type":"abandon", "value":true}],"objectIdentifier": "%s"}""" % phab_revision) cmd += " | %s call-conduit --conduit-uri=%s differential.revision.edit --""" % (_arc(), self.url) diff --git a/apis/taskcluster.py b/apis/taskcluster.py index 087e124c..70b10353 100644 --- a/apis/taskcluster.py +++ b/apis/taskcluster.py @@ -12,7 +12,7 @@ from collections import defaultdict from urllib.parse import quote_plus -from components.utilities import Struct, merge_dictionaries, PUSH_HEALTH_IGNORED_DICTS, PUSH_HEALTH_IGNORED_KEYS +from components.utilities import retry, Struct, merge_dictionaries, PUSH_HEALTH_IGNORED_DICTS, PUSH_HEALTH_IGNORED_KEYS from components.logging import logEntryExit, logEntryExitNoArgs, LogLevel from components.providerbase import BaseProvider, INeedsCommandProvider, INeedsLoggingProvider @@ -149,6 +149,7 @@ def _vcs_setup(self): self._vcs_setup_initialized = True @logEntryExit + @retry def submit_to_try(self, library, platform_filter, recursed=0): self._vcs_setup() if not platform_filter: @@ -171,12 +172,9 @@ def submit_to_try(self, library, platform_filter, recursed=0): else: try_arguments = ["./mach", "try", "auto"] + platform_filter_args - ret = self.run(try_arguments, clean_return=False if recursed < 5 else True) + ret = self.run(try_arguments, clean_return=True) output = ret.stdout.decode() - if "timed out waiting for lock held by" in output: - return self.submit_to_try(library, platform_filter, recursed + 1) - isNext = False try_link = None for line in output.split("\n"): diff --git a/components/logging.py b/components/logging.py index a2275eef..9e9a9fec 100644 --- a/components/logging.py +++ b/components/logging.py @@ -36,7 +36,7 @@ def func_wrapper(*args, **kwargs): obj.logger.log("Beginning %s" % func.__qualname__, level=LogLevel.Info) obj.logger.log(" Arguments: %s" % (str(args) + " " + str(kwargs) if print_arg_list else "[Omitted %s args]" % str(len(args) + len(kwargs))), level=LogLevel.Debug) ret = func(*args, **kwargs) - if type(ret) == list: + if ret is list: obj.logger.log("Function returned a list of %s objects" % len(ret), level=LogLevel.Debug) else: return_string = str(ret) diff --git a/components/utilities.py b/components/utilities.py index c6777b50..5eece93f 100644 --- a/components/utilities.py +++ b/components/utilities.py @@ -7,6 +7,7 @@ import copy import pickle import functools +import time from dateutil.parser import parse @@ -138,3 +139,30 @@ def decorate(func): return func return decorate + + +# Retry calling a function `times` times, sleeping between each tries, with an exponential backoff +# This is to be used on API calls, that are likely to fail + + +def retry(_func=None, *, times=10, sleep_s=1, exp=2): + def decorator_retry(func): + @functools.wraps(func) + def wrapper_retry(*args, **kwargs): + retries_try = times + sleep_duration = sleep_s + while retries_try > 0: + try: + return func(*args, **kwargs) + except BaseException as e: + retries_try -= 1 + time.sleep(sleep_duration) + sleep_duration *= exp + if retries_try == 0: + raise e + return wrapper_retry + + if _func is None: + return decorator_retry + else: + return decorator_retry(_func) diff --git a/tests/functionality_all_platforms.py b/tests/functionality_all_platforms.py index 90d00413..804548cd 100755 --- a/tests/functionality_all_platforms.py +++ b/tests/functionality_all_platforms.py @@ -640,6 +640,57 @@ def treeherder(request_type, fullpath): finally: self._cleanup(u, expected_values) + # Fail the first time, then work. + @logEntryExitHeaderLine + def testRetryFunctionality(self): + @treeherder_response + def treeherder(request_type, fullpath): + if request_type == TYPE_HEALTH: + return "health_classified_failures.txt" + else: # TYPE_JOBS + if treeherder.jobs_calls == 0: + return "jobs_still_running.txt" + return "jobs_classified_failures.txt" + + try_fails = 0 + + def try_submit(cmd): + nonlocal try_fails + if try_fails == 0: + try_fails += 1 + raise Exception("No worky!") + if "./mach try auto" in cmd: + return TRY_OUTPUT(expected_values.try_revision_id()) + elif "./mach try fuzzy" in cmd: + return TRY_OUTPUT(expected_values.try_revision_id(), False) + + library_filter = 'cubeb-query' + (u, expected_values, _check_jobs) = self._setup( + library_filter, + lambda b: ["e152bb86666565ee6619c15f60156cd6c79580a9|2021-02-09 15:30:04 -0500|2021-02-12 17:40:01 +0000"], + lambda: 50, # get_filed_bug_id_func, + lambda b: [], # filed_bug_ids_func + treeherder, + command_callbacks={'try_submit': try_submit} + ) + try: + # Run it + u.run(library_filter=library_filter) + # Check that we created the job successfully + _check_jobs(JOBSTATUS.AWAITING_SECOND_PLATFORMS_TRY_RESULTS, JOBOUTCOME.PENDING) + + # Run it again, this time we'll tell it the jobs are still in process + u.run(library_filter=library_filter) + # Should still be Awaiting Try Results + _check_jobs(JOBSTATUS.AWAITING_SECOND_PLATFORMS_TRY_RESULTS, JOBOUTCOME.PENDING) + + # Run it again, this time we'll tell it the jobs succeeded + u.run(library_filter=library_filter) + # Should be DONE + _check_jobs(JOBSTATUS.DONE, JOBOUTCOME.CLASSIFIED_FAILURES) + finally: + self._cleanup(u, expected_values) + @logEntryExitHeaderLine def testAllNewFuzzyPathJobs(self): @treeherder_response