Merge pull request #73 from MozillaSecurity/check-build-date

Ensure found build is newer or older than requested based on direction

setup.cfg

@@ -83,7 +83,7 @@ passenv =
     TRAVIS_*
     CODECOV_*
 commands =
-    pytest -v --cache-clear --flake8 --pylint --cov="{toxinidir}" --cov-report term-missing --basetemp="{envtmpdir}" {posargs}
+    pytest -v --cache-clear --flake8 --pylint --cov="{toxinidir}" --cov-report term-missing --basetemp="{envtmpdir}" --log-level=DEBUG {posargs}
 
 [testenv:codecov]
 deps =

src/fuzzfetch/fetch.py

@@ -171,10 +171,13 @@ def from_platform_guess(cls, build_string):
         """
         Create a platform object from a namespace build string
         """
+        match = []
         for system, platform in cls.SUPPORTED.items():
             for machine, platform_guess in platform.items():
-                if platform_guess in build_string:
-                    return cls(system, machine)
+                if platform_guess in build_string and (not match or len(match[2]) < len(platform_guess)):
+                    match = [system, machine, platform_guess]
+        if match:
+            return cls(match[0], match[1])
         raise FetcherException('Could not extract platform from %s' % (build_string,))
 
     def auto_name_prefix(self):
@@ -489,6 +492,7 @@ def __init__(self, target, branch, build, flags, platform=None, nearest=None):
         self._branch = branch
         self._flags = BuildFlags(*flags)
         self._platform = platform or Platform()
+        self._task = None
 
         if not isinstance(build, BuildTask):
             # If build doesn't match the following, assume it's a namespace
@@ -554,41 +558,62 @@ def __init__(self, target, branch, build, flags, platform=None, nearest=None):
                 if not nearest:
                     raise
 
-                start = None
+                requested = None
                 asc = nearest == Fetcher.BUILD_ORDER_ASC
                 if 'latest' in build:
-                    start = now + timedelta(days=1) if asc else now - timedelta(days=1)
+                    requested = now
                 elif BuildTask.RE_DATE.match(build) is not None:
                     date = datetime.strptime(build, '%Y-%m-%d')
-                    localized = timezone('UTC').localize(date)
-                    start = localized + timedelta(days=1) if asc else localized - timedelta(days=1)
+                    requested = timezone('UTC').localize(date)
                 elif BuildTask.RE_REV.match(build) is not None:
-                    start = HgRevision(build, branch).pushdate
+                    requested = HgRevision(build, branch).pushdate
                 else:
                     # If no match, assume it's a TaskCluster namespace
                     if re.match(r'.*[0-9]{4}\.[0-9]{2}\.[0-9]{2}.*', build) is not None:
                         match = re.search(r'[0-9]{4}\.[0-9]{2}\.[0-9]{2}', build)
                         date = datetime.strptime(match.group(0), '%Y.%m.%d')
-                        start = timezone('UTC').localize(date)
+                        requested = timezone('UTC').localize(date)
                     elif re.match(r'.*revision.*[0-9[a-f]{40}', build):
                         match = re.search(r'[0-9[a-f]{40}', build)
-                        start = HgRevision(match.group(0), branch).pushdate
+                        requested = HgRevision(match.group(0), branch).pushdate
 
                 # If start date is outside the range of the newest/oldest available build, adjust it
                 if asc:
-                    start = min(max(start, now - timedelta(days=364)), now)
+                    start = min(max(requested, now - timedelta(days=364)), now)
                     end = now
                 else:
                     end = now - timedelta(days=364)
-                    start = max(min(start, now), end)
+                    start = max(min(requested, now), end)
+                LOG.debug("searching for nearest build to %s from %s -> %s", requested, start, end)
 
                 while start <= end if asc else start >= end:
-                    try:
-                        self._task = BuildTask(start.strftime('%Y-%m-%d'), branch, self._flags, self._platform)
+                    search_build = start.strftime('%Y-%m-%d')
+
+                    # in the case where a calendar date was specified, we've already tried it directly
+                    if search_build != build:
+                        LOG.debug("trying %s", search_build)
+                        try:
+                            # iterate over all builds for the day, and take the next older/newer build available
+                            build_tasks = BuildTask.iterall(search_build, branch, self._flags, self._platform)
+                            if not asc:
+                                build_tasks = reversed(list(build_tasks))
+
+                            for task in build_tasks:
+                                task_date = timezone('EST').localize(datetime.fromtimestamp(task.rank))
+                                LOG.debug("got %s", task_date)
+                                if (asc and task_date >= requested) or (not asc and task_date <= requested):
+                                    self._task = task
+                                    break
+                        except FetcherException:
+                            LOG.warning('Unable to find build for %s', start.strftime('%Y-%m-%d'))
+
+                    if self._task is not None:
+                        # a task was found
                         break
-                    except FetcherException:
-                        LOG.warning('Unable to find build for %s', start.strftime('%Y-%m-%d'))
-                        start = start + timedelta(days=1) if asc else start - timedelta(days=1)
+
+                    # Increment start date
+                    start = start + timedelta(days=1) if asc else start - timedelta(days=1)
+
                 else:
                     raise FetcherException('Failed to find build near %s' % build)
 

tests/mock-firefoxci/api/index/v1/namespaces/gecko.v2.mozilla-central.pushdate.2019.12.01

@@ -0,0 +1,19 @@
+{
+  "namespaces": [
+    {
+      "namespace": "gecko.v2.mozilla-central.pushdate.2019.12.01.20191201093732",
+      "name": "20191201093732",
+      "expires": "2020-12-03T00:00:00.000Z"
+    },
+    {
+      "namespace": "gecko.v2.mozilla-central.pushdate.2019.12.01.20191201211801",
+      "name": "20191201211801",
+      "expires": "2020-12-03T00:00:00.000Z"
+    },
+    {
+      "namespace": "gecko.v2.mozilla-central.pushdate.2019.12.01.latest",
+      "name": "latest",
+      "expires": "2020-12-03T00:00:00.000Z"
+    }
+  ]
+}

tests/mock-firefoxci/api/index/v1/namespaces/gecko.v2.mozilla-central.pushdate.2020.08.01

@@ -0,0 +1,3 @@
+{
+  "namespaces": []
+}

tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.11.07.20191107230801.firefox.linux64-opt

@@ -0,0 +1,7 @@
+{
+  "namespace": "gecko.v2.mozilla-central.pushdate.2019.11.07.20191107230801.firefox.linux64-opt",
+  "taskId": "YiiDJr7_RvW6zEmm56cXXA",
+  "rank": 1573168081,
+  "data": {},
+  "expires": "2020-11-06T23:16:10.532Z"
+}

tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.11.07.latest.firefox.linux64-opt

@@ -0,0 +1,7 @@
+{
+  "namespace": "gecko.v2.mozilla-central.pushdate.2019.11.07.latest.firefox.linux64-opt",
+  "taskId": "YiiDJr7_RvW6zEmm56cXXA",
+  "rank": 1573168081,
+  "data": {},
+  "expires": "2020-11-06T23:16:10.532Z"
+}

tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.12.01.20191201093732.firefox.linux64-opt

@@ -0,0 +1,7 @@
+{
+  "namespace": "gecko.v2.mozilla-central.pushdate.2019.12.01.20191201093732.firefox.linux64-opt",
+  "taskId": "UHCnYtLpRRi77kbh6YOeUg",
+  "rank": 1575193052,
+  "data": {},
+  "expires": "2020-11-30T09:39:27.729Z"
+}

tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.12.01.20191201211801.firefox.linux64-opt

@@ -0,0 +1,7 @@
+{
+  "namespace": "gecko.v2.mozilla-central.pushdate.2019.12.01.20191201211801.firefox.linux64-opt",
+  "taskId": "eAxfP98wQnqzkydrMLeV-A",
+  "rank": 1575235081,
+  "data": {},
+  "expires": "2020-11-30T21:20:05.284Z"
+}

tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.pushdate.2019.12.01.latest.firefox.linux64-opt

@@ -0,0 +1,7 @@
+{
+  "namespace": "gecko.v2.mozilla-central.pushdate.2019.12.01.latest.firefox.linux64-opt",
+  "taskId": "eAxfP98wQnqzkydrMLeV-A",
+  "rank": 1575235081,
+  "data": {},
+  "expires": "2020-11-30T21:20:05.284Z"
+}