From 1ea700f2c6464c253dcec8df393c0f856dbcc9fc Mon Sep 17 00:00:00 2001 From: pyoor Date: Fri, 24 Jul 2020 13:40:07 -0400 Subject: [PATCH 1/4] Ensure found build is newer or older than requested based on direction --- src/fuzzfetch/fetch.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/src/fuzzfetch/fetch.py b/src/fuzzfetch/fetch.py index 8e35058b..13486142 100644 --- a/src/fuzzfetch/fetch.py +++ b/src/fuzzfetch/fetch.py @@ -489,6 +489,7 @@ def __init__(self, target, branch, build, flags, platform=None, nearest=None): self._branch = branch self._flags = BuildFlags(*flags) self._platform = platform or Platform() + self._task = None if not isinstance(build, BuildTask): # If build doesn't match the following, assume it's a namespace @@ -554,41 +555,57 @@ def __init__(self, target, branch, build, flags, platform=None, nearest=None): if not nearest: raise - start = None + requested = None asc = nearest == Fetcher.BUILD_ORDER_ASC if 'latest' in build: - start = now + timedelta(days=1) if asc else now - timedelta(days=1) + requested = now + timedelta(days=1) if asc else now - timedelta(days=1) elif BuildTask.RE_DATE.match(build) is not None: date = datetime.strptime(build, '%Y-%m-%d') localized = timezone('UTC').localize(date) - start = localized + timedelta(days=1) if asc else localized - timedelta(days=1) + requested = localized + timedelta(days=1) if asc else localized - timedelta(days=1) elif BuildTask.RE_REV.match(build) is not None: - start = HgRevision(build, branch).pushdate + requested = HgRevision(build, branch).pushdate else: # If no match, assume it's a TaskCluster namespace if re.match(r'.*[0-9]{4}\.[0-9]{2}\.[0-9]{2}.*', build) is not None: match = re.search(r'[0-9]{4}\.[0-9]{2}\.[0-9]{2}', build) date = datetime.strptime(match.group(0), '%Y.%m.%d') - start = timezone('UTC').localize(date) + requested = timezone('UTC').localize(date) elif re.match(r'.*revision.*[0-9[a-f]{40}', build): match = re.search(r'[0-9[a-f]{40}', build) - start = HgRevision(match.group(0), branch).pushdate + requested = HgRevision(match.group(0), branch).pushdate # If start date is outside the range of the newest/oldest available build, adjust it if asc: - start = min(max(start, now - timedelta(days=364)), now) + start = min(max(requested, now - timedelta(days=364)), now) end = now else: end = now - timedelta(days=364) - start = max(min(start, now), end) + start = max(min(requested, now), end) while start <= end if asc else start >= end: + build_str = start.strftime('%Y-%m-%d') try: - self._task = BuildTask(start.strftime('%Y-%m-%d'), branch, self._flags, self._platform) - break + if start.date() == requested.date(): + for task in BuildTask.iterall(build_str, branch, self._flags, self._platform): + task_date = timezone('EST').localize(datetime.fromtimestamp(task.rank)) + if (asc and requested <= task_date) or (requested >= task_date): + self._task = task + break + else: + task = BuildTask(build_str, branch, self._flags, self._platform) + task_date = timezone('EST').localize(datetime.fromtimestamp(task.rank)) + if (asc and requested <= task_date) or (requested >= task_date): + self._task = task except FetcherException: LOG.warning('Unable to find build for %s', start.strftime('%Y-%m-%d')) + + if self._task is None: + # Increment start date start = start + timedelta(days=1) if asc else start - timedelta(days=1) + else: + break + else: raise FetcherException('Failed to find build near %s' % build) From 8eb027645cf8f709208f6659a5991824cadc77bf Mon Sep 17 00:00:00 2001 From: Jesse Schwartzentruber Date: Fri, 31 Jul 2020 11:30:34 -0400 Subject: [PATCH 2/4] Simplify the nearest search. --- src/fuzzfetch/fetch.py | 45 ++-- ...cko.v2.mozilla-central.pushdate.2019.12.01 | 19 ++ ...cko.v2.mozilla-central.pushdate.2020.08.01 | 3 + ...9.11.07.20191107230801.firefox.linux64-opt | 7 + ...date.2019.11.07.latest.firefox.linux64-opt | 7 + ...9.12.01.20191201093732.firefox.linux64-opt | 7 + ...9.12.01.20191201211801.firefox.linux64-opt | 7 + ...date.2019.12.01.latest.firefox.linux64-opt | 7 + .../YiiDJr7_RvW6zEmm56cXXA/artifacts/.get | 238 ++++++++++++++++++ .../artifacts/public/build/target.json | 23 ++ .../public/build/target.mozinfo.json | 37 +++ .../eAxfP98wQnqzkydrMLeV-A/artifacts/.get | 238 ++++++++++++++++++ .../artifacts/public/build/target.json | 23 ++ .../public/build/target.mozinfo.json | 37 +++ tests/test_fetch.py | 16 +- 15 files changed, 689 insertions(+), 25 deletions(-) create mode 100644 tests/mock-firefoxci/api/index/v1/namespaces/gecko.v2.mozilla-central.pushdate.2019.12.01 create mode 100644 tests/mock-firefoxci/api/index/v1/namespaces/gecko.v2.mozilla-central.pushdate.2020.08.01 create mode 100644 tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.11.07.20191107230801.firefox.linux64-opt create mode 100644 tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.11.07.latest.firefox.linux64-opt create mode 100644 tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.12.01.20191201093732.firefox.linux64-opt create mode 100644 tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.12.01.20191201211801.firefox.linux64-opt create mode 100644 tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.12.01.latest.firefox.linux64-opt create mode 100644 tests/mock-firefoxci/api/queue/v1/task/YiiDJr7_RvW6zEmm56cXXA/artifacts/.get create mode 100644 tests/mock-firefoxci/api/queue/v1/task/YiiDJr7_RvW6zEmm56cXXA/artifacts/public/build/target.json create mode 100644 tests/mock-firefoxci/api/queue/v1/task/YiiDJr7_RvW6zEmm56cXXA/artifacts/public/build/target.mozinfo.json create mode 100644 tests/mock-firefoxci/api/queue/v1/task/eAxfP98wQnqzkydrMLeV-A/artifacts/.get create mode 100644 tests/mock-firefoxci/api/queue/v1/task/eAxfP98wQnqzkydrMLeV-A/artifacts/public/build/target.json create mode 100644 tests/mock-firefoxci/api/queue/v1/task/eAxfP98wQnqzkydrMLeV-A/artifacts/public/build/target.mozinfo.json diff --git a/src/fuzzfetch/fetch.py b/src/fuzzfetch/fetch.py index 13486142..2803886a 100644 --- a/src/fuzzfetch/fetch.py +++ b/src/fuzzfetch/fetch.py @@ -558,11 +558,10 @@ def __init__(self, target, branch, build, flags, platform=None, nearest=None): requested = None asc = nearest == Fetcher.BUILD_ORDER_ASC if 'latest' in build: - requested = now + timedelta(days=1) if asc else now - timedelta(days=1) + requested = now elif BuildTask.RE_DATE.match(build) is not None: date = datetime.strptime(build, '%Y-%m-%d') - localized = timezone('UTC').localize(date) - requested = localized + timedelta(days=1) if asc else localized - timedelta(days=1) + requested = timezone('UTC').localize(date) elif BuildTask.RE_REV.match(build) is not None: requested = HgRevision(build, branch).pushdate else: @@ -582,30 +581,36 @@ def __init__(self, target, branch, build, flags, platform=None, nearest=None): else: end = now - timedelta(days=364) start = max(min(requested, now), end) + LOG.debug("searching for nearest build to %s from %s -> %s", requested, start, end) while start <= end if asc else start >= end: - build_str = start.strftime('%Y-%m-%d') - try: - if start.date() == requested.date(): - for task in BuildTask.iterall(build_str, branch, self._flags, self._platform): + search_build = start.strftime('%Y-%m-%d') + + # in the case where a calendar date was specified, we've already tried it directly + if search_build != build: + LOG.debug("trying %s", search_build) + try: + # iterate over all builds for the day, and take the next older/newer build available + build_tasks = BuildTask.iterall(search_build, branch, self._flags, self._platform) + if not asc: + build_tasks = reversed(list(build_tasks)) + + for task in build_tasks: task_date = timezone('EST').localize(datetime.fromtimestamp(task.rank)) - if (asc and requested <= task_date) or (requested >= task_date): + LOG.debug("got %s", task_date) + if (asc and task_date >= requested) or (not asc and task_date <= requested): self._task = task break - else: - task = BuildTask(build_str, branch, self._flags, self._platform) - task_date = timezone('EST').localize(datetime.fromtimestamp(task.rank)) - if (asc and requested <= task_date) or (requested >= task_date): - self._task = task - except FetcherException: - LOG.warning('Unable to find build for %s', start.strftime('%Y-%m-%d')) - - if self._task is None: - # Increment start date - start = start + timedelta(days=1) if asc else start - timedelta(days=1) - else: + except FetcherException: + LOG.warning('Unable to find build for %s', start.strftime('%Y-%m-%d')) + + if self._task is not None: + # a task was found break + # Increment start date + start = start + timedelta(days=1) if asc else start - timedelta(days=1) + else: raise FetcherException('Failed to find build near %s' % build) diff --git a/tests/mock-firefoxci/api/index/v1/namespaces/gecko.v2.mozilla-central.pushdate.2019.12.01 b/tests/mock-firefoxci/api/index/v1/namespaces/gecko.v2.mozilla-central.pushdate.2019.12.01 new file mode 100644 index 00000000..55280b08 --- /dev/null +++ b/tests/mock-firefoxci/api/index/v1/namespaces/gecko.v2.mozilla-central.pushdate.2019.12.01 @@ -0,0 +1,19 @@ +{ + "namespaces": [ + { + "namespace": "gecko.v2.mozilla-central.pushdate.2019.12.01.20191201093732", + "name": "20191201093732", + "expires": "2020-12-03T00:00:00.000Z" + }, + { + "namespace": "gecko.v2.mozilla-central.pushdate.2019.12.01.20191201211801", + "name": "20191201211801", + "expires": "2020-12-03T00:00:00.000Z" + }, + { + "namespace": "gecko.v2.mozilla-central.pushdate.2019.12.01.latest", + "name": "latest", + "expires": "2020-12-03T00:00:00.000Z" + } + ] +} \ No newline at end of file diff --git a/tests/mock-firefoxci/api/index/v1/namespaces/gecko.v2.mozilla-central.pushdate.2020.08.01 b/tests/mock-firefoxci/api/index/v1/namespaces/gecko.v2.mozilla-central.pushdate.2020.08.01 new file mode 100644 index 00000000..eb4f85a7 --- /dev/null +++ b/tests/mock-firefoxci/api/index/v1/namespaces/gecko.v2.mozilla-central.pushdate.2020.08.01 @@ -0,0 +1,3 @@ +{ + "namespaces": [] +} \ No newline at end of file diff --git a/tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.11.07.20191107230801.firefox.linux64-opt b/tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.11.07.20191107230801.firefox.linux64-opt new file mode 100644 index 00000000..c4ca4fbe --- /dev/null +++ b/tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.11.07.20191107230801.firefox.linux64-opt @@ -0,0 +1,7 @@ +{ + "namespace": "gecko.v2.mozilla-central.pushdate.2019.11.07.20191107230801.firefox.linux64-opt", + "taskId": "YiiDJr7_RvW6zEmm56cXXA", + "rank": 1573168081, + "data": {}, + "expires": "2020-11-06T23:16:10.532Z" +} \ No newline at end of file diff --git a/tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.11.07.latest.firefox.linux64-opt b/tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.11.07.latest.firefox.linux64-opt new file mode 100644 index 00000000..80bc24eb --- /dev/null +++ b/tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.11.07.latest.firefox.linux64-opt @@ -0,0 +1,7 @@ +{ + "namespace": "gecko.v2.mozilla-central.pushdate.2019.11.07.latest.firefox.linux64-opt", + "taskId": "YiiDJr7_RvW6zEmm56cXXA", + "rank": 1573168081, + "data": {}, + "expires": "2020-11-06T23:16:10.532Z" +} \ No newline at end of file diff --git a/tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.12.01.20191201093732.firefox.linux64-opt b/tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.12.01.20191201093732.firefox.linux64-opt new file mode 100644 index 00000000..261f3ee2 --- /dev/null +++ b/tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.12.01.20191201093732.firefox.linux64-opt @@ -0,0 +1,7 @@ +{ + "namespace": "gecko.v2.mozilla-central.pushdate.2019.12.01.20191201093732.firefox.linux64-opt", + "taskId": "UHCnYtLpRRi77kbh6YOeUg", + "rank": 1575193052, + "data": {}, + "expires": "2020-11-30T09:39:27.729Z" +} \ No newline at end of file diff --git a/tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.12.01.20191201211801.firefox.linux64-opt b/tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.12.01.20191201211801.firefox.linux64-opt new file mode 100644 index 00000000..304ee898 --- /dev/null +++ b/tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.12.01.20191201211801.firefox.linux64-opt @@ -0,0 +1,7 @@ +{ + "namespace": "gecko.v2.mozilla-central.pushdate.2019.12.01.20191201211801.firefox.linux64-opt", + "taskId": "eAxfP98wQnqzkydrMLeV-A", + "rank": 1575235081, + "data": {}, + "expires": "2020-11-30T21:20:05.284Z" +} \ No newline at end of file diff --git a/tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.12.01.latest.firefox.linux64-opt b/tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.12.01.latest.firefox.linux64-opt new file mode 100644 index 00000000..0c71df45 --- /dev/null +++ b/tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.12.01.latest.firefox.linux64-opt @@ -0,0 +1,7 @@ +{ + "namespace": "gecko.v2.mozilla-central.pushdate.2019.12.01.latest.firefox.linux64-opt", + "taskId": "eAxfP98wQnqzkydrMLeV-A", + "rank": 1575235081, + "data": {}, + "expires": "2020-11-30T21:20:05.284Z" +} \ No newline at end of file diff --git a/tests/mock-firefoxci/api/queue/v1/task/YiiDJr7_RvW6zEmm56cXXA/artifacts/.get b/tests/mock-firefoxci/api/queue/v1/task/YiiDJr7_RvW6zEmm56cXXA/artifacts/.get new file mode 100644 index 00000000..7831f908 --- /dev/null +++ b/tests/mock-firefoxci/api/queue/v1/task/YiiDJr7_RvW6zEmm56cXXA/artifacts/.get @@ -0,0 +1,238 @@ +{ + "artifacts": [ + { + "storageType": "s3", + "name": "public/build/buildhub.json", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/json" + }, + { + "storageType": "s3", + "name": "public/build/build_resources.html", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "text/html" + }, + { + "storageType": "s3", + "name": "public/build/build_resources.json", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/json" + }, + { + "storageType": "s3", + "name": "public/build/config.status", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/octet-stream" + }, + { + "storageType": "s3", + "name": "public/build/host/bin/mar", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/octet-stream" + }, + { + "storageType": "s3", + "name": "public/build/host/bin/mbsdiff", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/octet-stream" + }, + { + "storageType": "s3", + "name": "public/build/mozharness.zip", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/zip" + }, + { + "storageType": "s3", + "name": "public/build/sccache.log", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "text/plain" + }, + { + "storageType": "s3", + "name": "public/build/target.awsy.tests.tar.gz", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.checksums", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/octet-stream" + }, + { + "storageType": "s3", + "name": "public/build/target.common.tests.tar.gz", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.condprof.tests.tar.gz", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.cppunittest.tests.tar.gz", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.crashreporter-symbols-full.zip", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/zip" + }, + { + "storageType": "s3", + "name": "public/build/target.crashreporter-symbols.zip", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/zip" + }, + { + "storageType": "s3", + "name": "public/build/target.generated-files.tar.gz", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.gtest.tests.tar.gz", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target_info.txt", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "text/plain" + }, + { + "storageType": "s3", + "name": "public/build/target.json", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/json" + }, + { + "storageType": "s3", + "name": "public/build/target.jsshell.zip", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/zip" + }, + { + "storageType": "s3", + "name": "public/build/target.langpack.xpi", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/x-xpinstall" + }, + { + "storageType": "s3", + "name": "public/build/target.mochitest.tests.tar.gz", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.mozinfo.json", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/json" + }, + { + "storageType": "s3", + "name": "public/build/target.raptor.tests.tar.gz", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.reftest.tests.tar.gz", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.talos.tests.tar.gz", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.tar.bz2", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/x-bzip2" + }, + { + "storageType": "s3", + "name": "public/build/target.test_packages.json", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/json" + }, + { + "storageType": "s3", + "name": "public/build/target.txt", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "text/plain" + }, + { + "storageType": "s3", + "name": "public/build/target.updater-dep.tests.tar.gz", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.web-platform.tests.tar.gz", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.xpcshell.tests.tar.gz", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/xvfb/xvfb.log", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "text/plain" + }, + { + "storageType": "s3", + "name": "public/chain-of-trust.json", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "text/plain" + }, + { + "storageType": "s3", + "name": "public/chain-of-trust.json.sig", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/octet-stream" + }, + { + "storageType": "s3", + "name": "public/logs/certified.log", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "text/plain" + }, + { + "storageType": "s3", + "name": "public/logs/live_backing.log", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "text/plain; charset=utf-8" + }, + { + "storageType": "reference", + "name": "public/logs/live.log", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "text/plain; charset=utf-8" + }, + { + "storageType": "s3", + "name": "public/logs/localconfig.json", + "expires": "2020-11-06T23:16:10.532Z", + "contentType": "application/json" + } + ] +} \ No newline at end of file diff --git a/tests/mock-firefoxci/api/queue/v1/task/YiiDJr7_RvW6zEmm56cXXA/artifacts/public/build/target.json b/tests/mock-firefoxci/api/queue/v1/task/YiiDJr7_RvW6zEmm56cXXA/artifacts/public/build/target.json new file mode 100644 index 00000000..b8334335 --- /dev/null +++ b/tests/mock-firefoxci/api/queue/v1/task/YiiDJr7_RvW6zEmm56cXXA/artifacts/public/build/target.json @@ -0,0 +1,23 @@ +{ + "as": "/builds/worker/fetches/sccache/sccache /builds/worker/fetches/clang/bin/clang -std=gnu99", + "buildid": "20191107230801", + "cc": "/builds/worker/fetches/sccache/sccache /builds/worker/fetches/clang/bin/clang -std=gnu99", + "cxx": "/builds/worker/fetches/sccache/sccache /builds/worker/fetches/clang/bin/clang++", + "host_alias": "x86_64-pc-linux-gnu", + "host_cpu": "x86_64", + "host_os": "linux-gnu", + "host_vendor": "pc", + "moz_app_id": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}", + "moz_app_maxversion": "72.0a1", + "moz_app_name": "firefox", + "moz_app_vendor": "Mozilla", + "moz_app_version": "72.0a1", + "moz_pkg_platform": "linux-x86_64", + "moz_source_repo": "https://hg.mozilla.org/mozilla-central", + "moz_source_stamp": "ac63c8962183502a4b0ec32222efc67d3841d157", + "moz_update_channel": "default", + "target_alias": "x86_64-pc-linux-gnu", + "target_cpu": "x86_64", + "target_os": "linux-gnu", + "target_vendor": "pc" +} diff --git a/tests/mock-firefoxci/api/queue/v1/task/YiiDJr7_RvW6zEmm56cXXA/artifacts/public/build/target.mozinfo.json b/tests/mock-firefoxci/api/queue/v1/task/YiiDJr7_RvW6zEmm56cXXA/artifacts/public/build/target.mozinfo.json new file mode 100644 index 00000000..51718130 --- /dev/null +++ b/tests/mock-firefoxci/api/queue/v1/task/YiiDJr7_RvW6zEmm56cXXA/artifacts/public/build/target.mozinfo.json @@ -0,0 +1,37 @@ +{ + "allow_legacy_extensions": true, + "appname": "firefox", + "artifact": false, + "asan": false, + "bin_suffix": "", + "bits": 64, + "buildapp": "browser", + "buildtype_guess": "opt", + "cc_type": "clang", + "ccov": false, + "crashreporter": true, + "datareporting": true, + "debug": false, + "devedition": false, + "healthreport": true, + "mozconfig": "/builds/worker/workspace/build/src/.mozconfig", + "nightly_build": true, + "normandy": true, + "official": true, + "os": "linux", + "pgo": false, + "platform_guess": "linux64", + "processor": "x86_64", + "release_or_beta": false, + "require_signing": false, + "stylo": true, + "sync": true, + "telemetry": false, + "tests_enabled": true, + "toolkit": "gtk", + "topsrcdir": "/builds/worker/workspace/build/src", + "tsan": false, + "ubsan": false, + "updater": true, + "xbl": false +} \ No newline at end of file diff --git a/tests/mock-firefoxci/api/queue/v1/task/eAxfP98wQnqzkydrMLeV-A/artifacts/.get b/tests/mock-firefoxci/api/queue/v1/task/eAxfP98wQnqzkydrMLeV-A/artifacts/.get new file mode 100644 index 00000000..e227ffc9 --- /dev/null +++ b/tests/mock-firefoxci/api/queue/v1/task/eAxfP98wQnqzkydrMLeV-A/artifacts/.get @@ -0,0 +1,238 @@ +{ + "artifacts": [ + { + "storageType": "s3", + "name": "public/build/buildhub.json", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/json" + }, + { + "storageType": "s3", + "name": "public/build/build_resources.html", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "text/html" + }, + { + "storageType": "s3", + "name": "public/build/build_resources.json", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/json" + }, + { + "storageType": "s3", + "name": "public/build/config.status", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/octet-stream" + }, + { + "storageType": "s3", + "name": "public/build/host/bin/mar", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/octet-stream" + }, + { + "storageType": "s3", + "name": "public/build/host/bin/mbsdiff", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/octet-stream" + }, + { + "storageType": "s3", + "name": "public/build/mozharness.zip", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/zip" + }, + { + "storageType": "s3", + "name": "public/build/sccache.log", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "text/plain" + }, + { + "storageType": "s3", + "name": "public/build/target.awsy.tests.tar.gz", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.checksums", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/octet-stream" + }, + { + "storageType": "s3", + "name": "public/build/target.common.tests.tar.gz", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.condprof.tests.tar.gz", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.cppunittest.tests.tar.gz", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.crashreporter-symbols-full.zip", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/zip" + }, + { + "storageType": "s3", + "name": "public/build/target.crashreporter-symbols.zip", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/zip" + }, + { + "storageType": "s3", + "name": "public/build/target.generated-files.tar.gz", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.gtest.tests.tar.gz", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target_info.txt", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "text/plain" + }, + { + "storageType": "s3", + "name": "public/build/target.json", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/json" + }, + { + "storageType": "s3", + "name": "public/build/target.jsshell.zip", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/zip" + }, + { + "storageType": "s3", + "name": "public/build/target.langpack.xpi", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/x-xpinstall" + }, + { + "storageType": "s3", + "name": "public/build/target.mochitest.tests.tar.gz", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.mozinfo.json", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/json" + }, + { + "storageType": "s3", + "name": "public/build/target.raptor.tests.tar.gz", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.reftest.tests.tar.gz", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.talos.tests.tar.gz", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.tar.bz2", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/x-bzip2" + }, + { + "storageType": "s3", + "name": "public/build/target.test_packages.json", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/json" + }, + { + "storageType": "s3", + "name": "public/build/target.txt", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "text/plain" + }, + { + "storageType": "s3", + "name": "public/build/target.updater-dep.tests.tar.gz", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.web-platform.tests.tar.gz", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/target.xpcshell.tests.tar.gz", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/gzip" + }, + { + "storageType": "s3", + "name": "public/build/xvfb/xvfb.log", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "text/plain" + }, + { + "storageType": "s3", + "name": "public/chain-of-trust.json", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "text/plain" + }, + { + "storageType": "s3", + "name": "public/chain-of-trust.json.sig", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/octet-stream" + }, + { + "storageType": "s3", + "name": "public/logs/certified.log", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "text/plain" + }, + { + "storageType": "s3", + "name": "public/logs/live_backing.log", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "text/plain; charset=utf-8" + }, + { + "storageType": "reference", + "name": "public/logs/live.log", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "text/plain; charset=utf-8" + }, + { + "storageType": "s3", + "name": "public/logs/localconfig.json", + "expires": "2020-11-30T21:20:05.284Z", + "contentType": "application/json" + } + ] +} \ No newline at end of file diff --git a/tests/mock-firefoxci/api/queue/v1/task/eAxfP98wQnqzkydrMLeV-A/artifacts/public/build/target.json b/tests/mock-firefoxci/api/queue/v1/task/eAxfP98wQnqzkydrMLeV-A/artifacts/public/build/target.json new file mode 100644 index 00000000..adaa1d64 --- /dev/null +++ b/tests/mock-firefoxci/api/queue/v1/task/eAxfP98wQnqzkydrMLeV-A/artifacts/public/build/target.json @@ -0,0 +1,23 @@ +{ + "as": "/builds/worker/fetches/sccache/sccache /builds/worker/fetches/clang/bin/clang -std=gnu99", + "buildid": "20191201211801", + "cc": "/builds/worker/fetches/sccache/sccache /builds/worker/fetches/clang/bin/clang -std=gnu99", + "cxx": "/builds/worker/fetches/sccache/sccache /builds/worker/fetches/clang/bin/clang++", + "host_alias": "x86_64-pc-linux-gnu", + "host_cpu": "x86_64", + "host_os": "linux-gnu", + "host_vendor": "pc", + "moz_app_id": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}", + "moz_app_maxversion": "72.0a1", + "moz_app_name": "firefox", + "moz_app_vendor": "Mozilla", + "moz_app_version": "72.0a1", + "moz_pkg_platform": "linux-x86_64", + "moz_source_repo": "https://hg.mozilla.org/mozilla-central", + "moz_source_stamp": "251480204d10c4bf3731fa625e07624c3cd52b0d", + "moz_update_channel": "default", + "target_alias": "x86_64-pc-linux-gnu", + "target_cpu": "x86_64", + "target_os": "linux-gnu", + "target_vendor": "pc" +} diff --git a/tests/mock-firefoxci/api/queue/v1/task/eAxfP98wQnqzkydrMLeV-A/artifacts/public/build/target.mozinfo.json b/tests/mock-firefoxci/api/queue/v1/task/eAxfP98wQnqzkydrMLeV-A/artifacts/public/build/target.mozinfo.json new file mode 100644 index 00000000..a69544ff --- /dev/null +++ b/tests/mock-firefoxci/api/queue/v1/task/eAxfP98wQnqzkydrMLeV-A/artifacts/public/build/target.mozinfo.json @@ -0,0 +1,37 @@ +{ + "allow_legacy_extensions": true, + "appname": "firefox", + "artifact": false, + "asan": false, + "bin_suffix": "", + "bits": 64, + "buildapp": "browser", + "buildtype_guess": "opt", + "cc_type": "clang", + "ccov": false, + "crashreporter": true, + "datareporting": true, + "debug": false, + "devedition": false, + "early_beta_or_earlier": true, + "healthreport": true, + "mozconfig": "/builds/worker/workspace/build/src/.mozconfig", + "nightly_build": true, + "normandy": true, + "official": true, + "os": "linux", + "pgo": false, + "platform_guess": "linux64", + "processor": "x86_64", + "release_or_beta": false, + "require_signing": false, + "stylo": true, + "sync": true, + "telemetry": false, + "tests_enabled": true, + "toolkit": "gtk", + "topsrcdir": "/builds/worker/workspace/build/src", + "tsan": false, + "ubsan": false, + "updater": true +} \ No newline at end of file diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 60b27089..84fc0c0a 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -148,12 +148,18 @@ def test_metadata(branch, build_flags, os_, cpu): # whenever BUILD_CACHE is set: # - requested should be set to the near future, or the hg hash of a changeset prior to the first build yesterday # - expected should be updated to the value that asserts -@pytest.mark.parametrize('requested, expected', ( - ('2019-11-06', '2019-11-07'), - ('d271c572a9bcd008ed14bf104b2eb81949952e4c', 'e8b7c48d4e7ed1b63aeedff379b51e566ea499d9'))) +@pytest.mark.parametrize('requested, expected, direction', ( + ('2019-11-06', '2019-11-07', fuzzfetch.Fetcher.BUILD_ORDER_ASC), + ('2020-08-01', '2019-12-01', fuzzfetch.Fetcher.BUILD_ORDER_DESC), + ('d271c572a9bcd008ed14bf104b2eb81949952e4c', + 'ac63c8962183502a4b0ec32222efc67d3841d157', + fuzzfetch.Fetcher.BUILD_ORDER_ASC), + ('d271c572a9bcd008ed14bf104b2eb81949952e4c', + 'e8b7c48d4e7ed1b63aeedff379b51e566ea499d9', + fuzzfetch.Fetcher.BUILD_ORDER_DESC))) @pytest.mark.parametrize('is_namespace', [True, False]) @pytest.mark.usefixtures("requests_mock_cache") -def test_nearest_retrieval(requested, expected, is_namespace): +def test_nearest_retrieval(requested, expected, direction, is_namespace): """ Attempt to retrieve a build near the supplied build_id """ @@ -161,8 +167,8 @@ def test_nearest_retrieval(requested, expected, is_namespace): # Set freeze_time to a date ahead of the latest mock build with freeze_time('2019-12-01'): - direction = fuzzfetch.Fetcher.BUILD_ORDER_ASC + LOG.debug("looking for nearest to %s", requested) if is_namespace: if fuzzfetch.BuildTask.RE_DATE.match(requested): date = requested.replace('-', '.') From 1a1acba1c6d4e4e5ef5fe9811f37aa26f7e3e8f9 Mon Sep 17 00:00:00 2001 From: Jesse Schwartzentruber Date: Fri, 31 Jul 2020 11:31:06 -0400 Subject: [PATCH 3/4] Fix logging configuration. --- setup.cfg | 2 +- tests/test_fetch.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index fff49780..fb5e8a93 100644 --- a/setup.cfg +++ b/setup.cfg @@ -83,7 +83,7 @@ passenv = TRAVIS_* CODECOV_* commands = - pytest -v --cache-clear --flake8 --pylint --cov="{toxinidir}" --cov-report term-missing --basetemp="{envtmpdir}" {posargs} + pytest -v --cache-clear --flake8 --pylint --cov="{toxinidir}" --cov-report term-missing --basetemp="{envtmpdir}" --log-level=DEBUG {posargs} [testenv:codecov] deps = diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 84fc0c0a..3ae8c462 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -17,7 +17,6 @@ import fuzzfetch LOG = logging.getLogger("fuzzfetch_test") -logging.basicConfig(level=logging.DEBUG) logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("flake8").setLevel(logging.WARNING) From 02a248bdf468a9fd7caa1849e01d286da1afd3c1 Mon Sep 17 00:00:00 2001 From: Jesse Schwartzentruber Date: Fri, 31 Jul 2020 12:43:11 -0400 Subject: [PATCH 4/4] Take the longest match when searching for platform in a namespace. This was causing intermittent test failures on py27 and py35 where dictionary iteration order is random. --- src/fuzzfetch/fetch.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/fuzzfetch/fetch.py b/src/fuzzfetch/fetch.py index 2803886a..572b093e 100644 --- a/src/fuzzfetch/fetch.py +++ b/src/fuzzfetch/fetch.py @@ -171,10 +171,13 @@ def from_platform_guess(cls, build_string): """ Create a platform object from a namespace build string """ + match = [] for system, platform in cls.SUPPORTED.items(): for machine, platform_guess in platform.items(): - if platform_guess in build_string: - return cls(system, machine) + if platform_guess in build_string and (not match or len(match[2]) < len(platform_guess)): + match = [system, machine, platform_guess] + if match: + return cls(match[0], match[1]) raise FetcherException('Could not extract platform from %s' % (build_string,)) def auto_name_prefix(self):