nightly ci (#71)

* remove the upstream-dev logic

* add a nightly ci that runs on new prs, pushes to existing prs and at midnight

* automatically open a new issue for failures

* import and execute blackdoc before running the test suite

* specify which vm to run the report job on

* add a missing single quote

* update changelog.rst

.github/workflows/ci.yml

@@ -15,12 +15,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.9]
-        include:
-          - python-version: 3.8
-            dependencies: ""
-          - python-version: 3.8
-            dependencies: "upstream-dev"
+        python-version: [3.6, 3.7, 3.8, 3.9]
 
     steps:
     - name: checkout the repository
@@ -36,12 +31,7 @@ jobs:
 
     - name: install dependencies
       run: |
-        if [ -n "${{ matrix.dependencies }}" ]; then
-            dependencies="${{ matrix.dependencies }}"
-        else
-            dependencies="normal"
-        fi
-        python -m pip install -r ci/requirements/$dependencies.txt
+        python -m pip install -r ci/requirements/normal.txt
 
     - name: install blackdoc
       run: python -m pip install .

.github/workflows/nightly.yml

@@ -0,0 +1,159 @@
+name: Nightly CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  schedule:
+    - cron: "0 0 * * *"  # Daily "At 00:00" UTC
+
+jobs:
+  upstream-dev:
+    name: upstream-dev
+    runs-on: ubuntu-latest
+    if: |
+      always()
+      && (github.repository == 'keewis/blackdoc' || github.event_name != 'schedule')
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.9]
+
+    outputs:
+      artifacts_availability: ${{ steps.status.outputs.ARTIFACTS_AVAILABLE }}
+
+    steps:
+    - name: cancel previous runs
+      uses: styfle/[email protected]
+      with:
+        access_token: ${{ github.token }}
+
+    - name: checkout the repository
+      uses: actions/checkout@v2
+
+    - name: setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: upgrade pip
+      run: python -m pip install --upgrade pip
+
+    - name: install dependencies
+      run: |
+        python -m pip install -r ci/requirements/upstream-dev.txt
+
+    - name: install blackdoc
+      run: python -m pip install .
+
+    - name: show versions
+      run: python -m pip list
+
+    - name: import and run blackdoc
+      run: |
+        python -c 'import blackdoc'
+        python -m blackdoc
+
+    - name: run tests
+      if: success()
+      id: status
+      run: |
+        set -euo pipefail
+        python -m pytest -rf | tee output-${{ matrix.python-version }}-log || (
+          echo '::set-output name=ARTIFACTS_AVAILABLE::true' && false
+        )
+
+    - name: Upload artifacts
+      if: |
+        failure()
+        && github.event_name == 'schedule'
+      uses: actions/upload-artifact@v2
+      with:
+        name: output-${{ matrix.python-version }}-log
+        path: output-${{ matrix.python-version }}-log
+        retention-days: 5
+
+  report:
+    name: Report
+    runs-on: ubuntu-latest
+    needs: upstream-dev
+    if: |
+      always()
+      && github.event_name == 'schedule'
+      && github.repository == 'keewis/blackdoc'
+      && needs.upstream-dev.outputs.artifacts_availability == 'true'
+    steps:
+    - name: checkout the repository
+      uses: actions/checkout@v2
+    - name: setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.x"
+    - uses: actions/download-artifact@v2
+      with:
+        path: /tmp/workspace/logs
+    - name: Move all log files into a single directory
+      run: |
+        rsync -a /tmp/workspace/logs/output-*/ ./logs
+        ls -R ./logs
+    - name: Parse logs
+      run: |
+        shopt -s globstar
+        python .github/workflows/parse_logs.py logs/**/*-log
+    - name: Report failures
+      uses: actions/github-script@v3
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          const fs = require('fs');
+          const pytest_logs = fs.readFileSync('pytest-logs.txt', 'utf8');
+          const title = "⚠️ Nightly upstream-dev CI failed ⚠️"
+          const workflow_url = `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
+          const issue_body = `[Workflow Run URL](${workflow_url})\n${pytest_logs}`
+
+          // Run GraphQL query against GitHub API to find the most recent open issue used for reporting failures
+          const query = `query($owner:String!, $name:String!, $creator:String!, $label:String!){
+            repository(owner: $owner, name: $name) {
+              issues(first: 1, states: OPEN, filterBy: {createdBy: $creator, labels: [$label]}, orderBy: {field: CREATED_AT, direction: DESC}) {
+                edges {
+                  node {
+                    body
+                    id
+                    number
+                  }
+                }
+              }
+            }
+          }`;
+          const variables = {
+              owner: context.repo.owner,
+              name: context.repo.repo,
+              label: 'CI',
+              creator: "github-actions[bot]"
+          }
+          const result = await github.graphql(query, variables)
+          const issue_info = result.repository.issues.edges[0].node
+          // If no issue is open, create a new issue, else update the
+          // body of the existing issue.
+          if (typeof issue_info.number === 'undefined') {
+          // If no issue is open, create a new issue,
+          // else update the body of the existing issue.
+          if (result.repository.issues.edges.length === 0) {
+              github.issues.create({
+                  owner: variables.owner,
+                  repo: variables.name,
+                  body: issue_body,
+                  title: title,
+                  labels: [variables.label]
+              })
+          } else {
+              github.issues.update({
+                  owner: variables.owner,
+                  repo: variables.name,
+                  issue_number: issue_info.number,
+                  issue_number: result.repository.issues.edges[0].node.number,
+                  body: issue_body
+              })
+          }

.github/workflows/parse_logs.py

@@ -0,0 +1,51 @@
+# type: ignore
+import argparse
+import itertools
+import pathlib
+import textwrap
+
+parser = argparse.ArgumentParser()
+parser.add_argument("filepaths", nargs="+", type=pathlib.Path)
+args = parser.parse_args()
+
+filepaths = sorted(p for p in args.filepaths if p.is_file())
+
+
+def extract_short_test_summary_info(lines):
+    up_to_start_of_section = itertools.dropwhile(
+        lambda l: "=== short test summary info ===" not in l,
+        lines,
+    )
+    up_to_section_content = itertools.islice(up_to_start_of_section, 1, None)
+    section_content = itertools.takewhile(
+        lambda l: l.startswith("FAILED"), up_to_section_content
+    )
+    content = "\n".join(section_content)
+
+    return content
+
+
+def format_log_message(path):
+    py_version = path.name.split("-")[1]
+    summary = f"Python {py_version} Test Summary Info"
+    with open(path) as f:
+        data = extract_short_test_summary_info(line.rstrip() for line in f)
+    message = textwrap.dedent(
+        f"""\
+        <details><summary>{summary}</summary>
+        ```
+        {data}
+        ```
+        </details>
+        """
+    )
+
+    return message
+
+
+print("Parsing logs ...")
+message = "\n\n".join(format_log_message(path) for path in filepaths)
+
+output_file = pathlib.Path("pytest-logs.txt")
+print(f"Writing output file to: {output_file.absolute()}")
+output_file.write_text(message)

doc/changelog.rst

@@ -5,6 +5,7 @@ v0.4 (*unreleased*)
 -------------------
 - don't detect comments ending with a colon as a block (:issue:`67`, :pull:`68`)
 - don't add color to redirected output and print reports to ``stderr`` (:issue:`66`, :pull:`69`)
+- add a nightly CI which also runs every day at 00:00 UTC (:pull:`71`)
 
 v0.3 (04 November 2020)
 -----------------------