Fetch nightly translations during build (home-assistant#13724)

Co-authored-by: Bram Kragten <[email protected]>

.github/workflows/ci.yaml

@@ -13,6 +13,7 @@ on:
 env:
   NODE_VERSION: 16
   NODE_OPTIONS: --max_old_space_size=6144
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
   lint:

.github/workflows/demo.yaml

@@ -26,6 +26,8 @@ jobs:
           CI: true
       - name: Build Demo
         run: ./node_modules/.bin/gulp build-demo
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Deploy to Netlify
         run: npx netlify-cli deploy --dir=demo/dist --prod
         env:

.github/workflows/nightly.yaml

@@ -49,9 +49,8 @@ jobs:
         run: |
           pip install build
           yarn install
-
+          export SKIP_FETCH_NIGHTLY_TRANSLATIONS=1
           script/build_frontend
-
           rm -rf dist home_assistant_frontend.egg-info
           python3 -m build
 

.github/workflows/release.yaml

@@ -52,7 +52,7 @@ jobs:
           python3 -m pip install twine build
           export TWINE_USERNAME="__token__"
           export TWINE_PASSWORD="${{ secrets.TWINE_TOKEN }}"
-
+          export SKIP_FETCH_NIGHTLY_TRANSLATIONS=1
           script/release
 
       - name: Upload release assets

.gitignore

@@ -5,6 +5,7 @@
 build
 hass_frontend/*
 dist
+translations
 
 # yarn
 .yarn/*

.vscode/tasks.json

@@ -191,7 +191,13 @@
       "runOptions": {
         "instanceLimit": 1
       }
-    }
+    },
+    {
+      "label": "Setup and fetch nightly translations",
+      "type": "gulp",
+      "task": "setup-and-fetch-nightly-translations",
+      "problemMatcher": []
+          }
   ],
   "inputs": [
     {

build-scripts/.eslintrc.json

@@ -1,7 +1,12 @@
 {
   "extends": "../.eslintrc.json",
   "rules": {
-    "import/no-extraneous-dependencies": 0,
-    "global-require": 0
+    "no-console": "off",
+    "import/no-extraneous-dependencies": "off",
+    "import/extensions": "off",
+    "import/no-dynamic-require": "off",
+    "global-require": "off",
+    "@typescript-eslint/no-var-requires": "off",
+    "prefer-arrow-callback": "off"
   }
 }

build-scripts/babel-plugins/inline-constants-plugin.js

@@ -1,4 +1,3 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
 const path = require("path");
 
 // Currently only supports CommonJS modules, as require is synchronous. `import` would need babel running asynchronous.
@@ -29,7 +28,6 @@ module.exports = function inlineConstants(babel, options, cwd) {
       const absolute = module.startsWith(".")
         ? require.resolve(module, { paths: [cwd] })
         : module;
-      // eslint-disable-next-line import/no-dynamic-require
       return [absolute, require(absolute)];
     })
   );

build-scripts/bundle.js

@@ -1,9 +1,9 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
 const path = require("path");
 const env = require("./env.js");
 const paths = require("./paths.js");
 
 // Files from NPM Packages that should not be imported
+// eslint-disable-next-line unused-imports/no-unused-vars
 module.exports.ignorePackages = ({ latestBuild }) => [
   // Part of yaml.js and only used for !!js functions that we don't use
   require.resolve("esprima"),

build-scripts/env.js

@@ -1,4 +1,3 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
 const fs = require("fs");
 const path = require("path");
 const paths = require("./paths.js");

build-scripts/gulp/entry-html.js

@@ -1,6 +1,4 @@
 // Tasks to generate entry HTML
-/* eslint-disable import/no-dynamic-require */
-/* eslint-disable global-require */
 const gulp = require("gulp");
 const fs = require("fs-extra");
 const path = require("path");
@@ -91,7 +89,9 @@ gulp.task("gen-pages-prod", (done) => {
 });
 
 gulp.task("gen-index-app-dev", (done) => {
-  let latestAppJS, latestCoreJS, latestCustomPanelJS;
+  let latestAppJS;
+  let latestCoreJS;
+  let latestCustomPanelJS;
 
   if (env.useWDS()) {
     latestAppJS = "http://localhost:8000/src/entrypoints/app.ts";

build-scripts/gulp/fetch-nightly_translations.js

@@ -0,0 +1,170 @@
+// Task to download the latest Lokalise translations from the nightly workflow artifacts
+
+const fs = require("fs/promises");
+const path = require("path");
+const process = require("process");
+const del = require("del");
+const gulp = require("gulp");
+const jszip = require("jszip");
+const tar = require("tar");
+const { Octokit } = require("@octokit/rest");
+const { createOAuthDeviceAuth } = require("@octokit/auth-oauth-device");
+
+const MAX_AGE = 24; // hours
+const OWNER = "home-assistant";
+const REPO = "frontend";
+const WORKFLOW_NAME = "nightly.yaml";
+const ARTIFACT_NAME = "translations";
+const CLIENT_ID = "Iv1.3914e28cb27834d1";
+const EXTRACT_DIR = "translations";
+const TOKEN_FILE = path.join(EXTRACT_DIR, "token.json");
+const ARTIFACT_FILE = path.join(EXTRACT_DIR, "artifact.json");
+
+let allowTokenSetup = false;
+gulp.task("allow-setup-fetch-nightly-translations", (done) => {
+  allowTokenSetup = true;
+  done();
+});
+
+gulp.task("fetch-nightly-translations", async function () {
+  // Skip all when environment flag is set (assumes translations are already in place)
+  if (process.env?.SKIP_FETCH_NIGHTLY_TRANSLATIONS) {
+    console.log("Skipping fetch due to environment signal");
+    return;
+  }
+
+  // Read current translations artifact info if it exists,
+  // and stop if they are not old enough
+  let currentArtifact;
+  try {
+    currentArtifact = JSON.parse(await fs.readFile(ARTIFACT_FILE, "utf-8"));
+    const currentAge =
+      (Date.now() - Date.parse(currentArtifact.created_at)) / 3600000;
+    if (currentAge < MAX_AGE) {
+      console.log(
+        "Keeping current translations (only %s hours old)",
+        currentAge.toFixed(1)
+      );
+      return;
+    }
+  } catch {
+    currentArtifact = null;
+  }
+
+  // To store file writing promises
+  const createExtractDir = fs.mkdir(EXTRACT_DIR, { recursive: true });
+  const writings = [];
+
+  // Authenticate to GitHub using GitHub action token if it exists,
+  // otherwise look for a saved user token or generate a new one if none
+  let tokenAuth;
+  if (process.env.GITHUB_TOKEN) {
+    tokenAuth = { token: process.env.GITHUB_TOKEN };
+  } else {
+    try {
+      tokenAuth = JSON.parse(await fs.readFile(TOKEN_FILE, "utf-8"));
+    } catch {
+      if (!allowTokenSetup) {
+        console.log("No token found so  build wil continue with English only");
+        return;
+      }
+      const auth = createOAuthDeviceAuth({
+        clientType: "github-app",
+        clientId: CLIENT_ID,
+        onVerification: (verification) => {
+          console.log(
+            "Task needs to authenticate to GitHub to fetch the translations from nightly workflow\n" +
+              "Please go to %s to authorize this task\n" +
+              "\nEnter user code: %s\n\n" +
+              "This code will expire in %s minutes\n" +
+              "Task will automatically continue after authorization and token will be saved for future use",
+            verification.verification_uri,
+            verification.user_code,
+            (verification.expires_in / 60).toFixed(0)
+          );
+        },
+      });
+      tokenAuth = await auth({ type: "oauth" });
+      writings.push(
+        createExtractDir.then(
+          fs.writeFile(TOKEN_FILE, JSON.stringify(tokenAuth, null, 2))
+        )
+      );
+    }
+  }
+
+  // Authenticate with token and request workflow runs from GitHub
+  console.log("Fetching new translations...");
+  const octokit = new Octokit({
+    userAgent: "Fetch Nightly Translations",
+    auth: tokenAuth.token,
+  });
+
+  const workflowRunsResponse = await octokit.rest.actions.listWorkflowRuns({
+    owner: OWNER,
+    repo: REPO,
+    workflow_id: WORKFLOW_NAME,
+    status: "success",
+    event: "schedule",
+    per_page: 1,
+    exclude_pull_requests: true,
+  });
+  if (workflowRunsResponse.data.total_count === 0) {
+    throw Error("No successful nightly workflow runs found");
+  }
+  const latestNightlyRun = workflowRunsResponse.data.workflow_runs[0];
+
+  // Stop if current is already the latest, otherwise Find the translations artifact
+  if (currentArtifact?.workflow_run.id === latestNightlyRun.id) {
+    console.log("Stopping because current translations are still the latest");
+    return;
+  }
+  const latestArtifact = (
+    await octokit.actions.listWorkflowRunArtifacts({
+      owner: OWNER,
+      repo: REPO,
+      run_id: latestNightlyRun.id,
+    })
+  ).data.artifacts.find((artifact) => artifact.name === ARTIFACT_NAME);
+  if (!latestArtifact) {
+    throw Error("Latest nightly workflow run has no translations artifact");
+  }
+  writings.push(
+    createExtractDir.then(
+      fs.writeFile(ARTIFACT_FILE, JSON.stringify(latestArtifact, null, 2))
+    )
+  );
+
+  // Remove the current translations
+  const deleteCurrent = Promise.all(writings).then(
+    del([`${EXTRACT_DIR}/*`, `!${ARTIFACT_FILE}`, `!${TOKEN_FILE}`])
+  );
+
+  // Get the download URL and follow the redirect to download (stored as ArrayBuffer)
+  const downloadResponse = await octokit.actions.downloadArtifact({
+    owner: OWNER,
+    repo: REPO,
+    artifact_id: latestArtifact.id,
+    archive_format: "zip",
+  });
+  if (downloadResponse.status !== 200) {
+    throw Error("Failure downloading translations artifact");
+  }
+
+  // Artifact is a tarball, but GitHub adds it to a zip file
+  console.log("Unpacking downloaded translations...");
+  const zip = await jszip.loadAsync(downloadResponse.data);
+  await deleteCurrent;
+  const extractStream = zip.file(/.*/)[0].nodeStream().pipe(tar.extract());
+  await new Promise((resolve, reject) => {
+    extractStream.on("close", resolve).on("error", reject);
+  });
+});
+
+gulp.task(
+  "setup-and-fetch-nightly-translations",
+  gulp.series(
+    "allow-setup-fetch-nightly-translations",
+    "fetch-nightly-translations"
+  )
+);

build-scripts/gulp/gallery.js

@@ -1,4 +1,3 @@
-/* eslint-disable */
 // Run demo develop mode
 const gulp = require("gulp");
 const fs = require("fs");
@@ -41,7 +40,7 @@ gulp.task("gather-gallery-pages", async function gatherPages() {
     }
     processed.add(pageId);
 
-    const [category, name] = pageId.split("/", 2);
+    const [category] = pageId.split("/", 2);
 
     const demoFile = path.resolve(pageDir, `${pageId}.ts`);
     const descriptionFile = path.resolve(pageDir, `${pageId}.markdown`);

build-scripts/gulp/locale-data.js

@@ -1,5 +1,3 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
-
 const del = require("del");
 const path = require("path");
 const gulp = require("gulp");

build-scripts/gulp/rollup.js

@@ -5,9 +5,9 @@ const rollup = require("rollup");
 const handler = require("serve-handler");
 const http = require("http");
 const log = require("fancy-log");
+const open = require("open");
 const rollupConfig = require("../rollup");
 const paths = require("../paths");
-const open = require("open");
 
 const bothBuilds = (createConfigFunc, params) =>
   gulp.series(
@@ -30,11 +30,11 @@ const bothBuilds = (createConfigFunc, params) =>
   );
 
 function createServer(serveOptions) {
-  const server = http.createServer((request, response) => {
-    return handler(request, response, {
+  const server = http.createServer((request, response) =>
+    handler(request, response, {
       public: serveOptions.root,
-    });
-  });
+    })
+  );
 
   server.listen(
     serveOptions.port,

build-scripts/gulp/service-worker.js

@@ -1,7 +1,5 @@
 // Generate service worker.
 // Based on manifest, create a file with the content as service_worker.js
-/* eslint-disable import/no-dynamic-require */
-/* eslint-disable global-require */
 const gulp = require("gulp");
 const path = require("path");
 const fs = require("fs-extra");