i18n: reorg files, api key check, formatting scripts, documenting rea…

…sons why...

.gitignore

@@ -154,5 +154,5 @@ venv
 
 # frontend/i18n files, which are created during build or updating the languages
 src/packages/frontend/i18n/extracted.json
-src/packages/frontend/i18n/*.compiled.json
+src/packages/frontend/i18n/trans/*.compiled.json
 

src/packages/frontend/account/account-page.tsx

@@ -328,6 +328,7 @@ Thank you for your patience and understanding as we work to make our application
 
     const menu: MenuProps = {
       items,
+      style: { maxHeight: "75vh", overflow: "auto" },
       onClick: ({ key }) => {
         if (key in LOCALIZATIONS) {
           redux

src/packages/frontend/app/render.tsx

@@ -28,6 +28,7 @@ function App({ children }) {
   const other_settings = useTypedRedux("account", "other_settings");
 
   // setting via ?lang=[locale] takes precedece over account settings
+  // additionally ?lang_temp=[locale] temporarily changes it, used by these impersonation admin links
   useAsyncEffect(async () => {
     const lang_set = QueryParams.get("lang");
     // lang_temp sets the language *temporarily*, i.e. without changing the account settings and it is sticky

src/packages/frontend/i18n/bin/common.sh

@@ -1 +1,8 @@
 LANGS="de_DE zh_CN es_ES fr_FR ru_RU it_IT ja_JP pt_PT ko_KR pl_PL tr_TR he_IL"
+
+check_api_key() {
+    if [ -z "${SIMPLELOCALIZE_KEY}" ]; then
+        echo "Error: SIMPLELOCALIZE_KEY is not set or is empty. Please provide a valid API key." >&2
+        exit 1
+    fi
+}

src/packages/frontend/i18n/bin/compile.sh

@@ -1,6 +1,13 @@
 #!/usr/bin/env bash
 . ./i18n/bin/common.sh
+
+# It was necessary to write a custom formatter (see formatter.js) – not clear why, but it works. It's just a trivial mapping.
+# "--ast" this is the main point of compiling: we use ICU messages, which no longer need to be parsed each time.
+# This compile step is called by the `pnpm build` step as well, hence there is no need to keep the compiled files in the sources.
 for L in $LANGS; do
-  pnpm exec formatjs compile --ast --format i18n/formatter.js --out-file ./i18n/$L.compiled.json  ./i18n/$L.json
-  #rm ./i18n/$L.json # we might want to delete it at some point, but for now it is nice to have a copy of the translated messages
+  pnpm exec formatjs compile \
+    --ast \
+    --format i18n/formatter.js \
+    --out-file ./i18n/trans/$L.compiled.json \
+    ./i18n/trans/$L.json
 done

src/packages/frontend/i18n/bin/download.sh

@@ -1,5 +1,13 @@
 #!/usr/bin/env bash
 . ./i18n/bin/common.sh
+
+check_api_key
+
+# Each language is downloaded into a spearate file and compiled – this allows for dynamic imports.
 for L in $LANGS; do
-  simplelocalize download --apiKey $SIMPLELOCALIZE_KEY --downloadPath ./i18n/$L.json --downloadFormat single-language-json --languageKey=$L
+  simplelocalize download \
+    --apiKey $SIMPLELOCALIZE_KEY \
+    --downloadPath ./i18n/trans/$L.json \
+    --downloadFormat single-language-json \
+    --languageKey=$L
 done

src/packages/frontend/i18n/bin/extract.sh

@@ -1,8 +1,11 @@
 #!/usr/bin/env bash
 
-pnpm exec formatjs extract $(git ls-files '**/*.tsx') i18n/*.ts jupyter/commands.ts  \
-                   --ignore='**/*.d.ts' --ignore='node_modules/*' \
-		   --ignore='dist/*' \
-		   --out-file i18n/extracted.json \
-		   --throws \
-		   --id-interpolation-pattern 'UNIQUE_ID_IS_MISSING'
+# The interpolation pattern is a fixed string, because we intentionally trigger ID colissions.
+# There is just one (unused) string without a unique ID – otherwise we always set an explicit hierarchical ID.
+# Read the README in this directory for more information.
+pnpm exec formatjs extract $(git ls-files '**/*.tsx') i18n/*.ts jupyter/commands.ts \
+	--ignore='**/*.d.ts' --ignore='node_modules/*' \
+	--ignore='dist/*' \
+	--out-file i18n/extracted.json \
+	--throws \
+	--id-interpolation-pattern 'UNIQUE_ID_IS_MISSING'

src/packages/frontend/i18n/bin/upload.sh

@@ -1,3 +1,13 @@
 #!/usr/bin/env bash
 
-simplelocalize upload --apiKey $SIMPLELOCALIZE_KEY --languageKey en --uploadFormat simplelocalize-json --overwrite --uploadPath ./i18n/extracted.json
+. ./i18n/bin/common.sh
+check_api_key
+
+# The English language is always used directly from the default strings.
+# During upload, any changes are overwritten as well.
+simplelocalize upload \
+    --apiKey $SIMPLELOCALIZE_KEY \
+    --languageKey en \
+    --uploadFormat simplelocalize-json \
+    --overwrite \
+    --uploadPath ./i18n/extracted.json

src/packages/frontend/i18n/i18.test.ts → src/packages/frontend/i18n/i18n.test.ts

@@ -5,7 +5,8 @@ export type Data = { [key in string]: IntlMessage };
 
 describe("i18n", () => {
   const tests: { data: Data; prefix: string }[] = [
-    { data: labels, prefix: "labels." },
+    // forced typing necessary, because of the "unique_id_is_missing" exception – this is harmless
+    { data: labels as any as Data, prefix: "labels." },
     { data: menu, prefix: "menu." },
     { data: editor, prefix: "editor." },
     { data: jupyter.editor, prefix: "jupyter.editor." },
@@ -16,6 +17,7 @@ describe("i18n", () => {
     expect(prefix.endsWith(".")).toBe(true);
     test(`${prefix} should have correct id prefix`, () => {
       for (const k in data) {
+        if (k === "unique_id_is_missing") continue;
         const v = data[k];
         expect(v.id.startsWith(prefix)).toBe(true);
       }

src/packages/frontend/i18n/index.ts

@@ -49,29 +49,29 @@ export function loadLocaleMessages(locale: Locale): Promise<Messages> {
         // Hence "defaultMessage" messages are used directly.
         return {};
       case "de":
-        return import("@cocalc/frontend/i18n/de_DE.compiled.json");
+        return import("@cocalc/frontend/i18n/trans/de_DE.compiled.json");
       case "zh":
-        return import("@cocalc/frontend/i18n/zh_CN.compiled.json");
+        return import("@cocalc/frontend/i18n/trans/zh_CN.compiled.json");
       case "es":
-        return import("@cocalc/frontend/i18n/es_ES.compiled.json");
+        return import("@cocalc/frontend/i18n/trans/es_ES.compiled.json");
       case "fr":
-        return import("@cocalc/frontend/i18n/fr_FR.compiled.json");
+        return import("@cocalc/frontend/i18n/trans/fr_FR.compiled.json");
       case "it":
-        return import("@cocalc/frontend/i18n/it_IT.compiled.json");
+        return import("@cocalc/frontend/i18n/trans/it_IT.compiled.json");
       case "ru":
-        return import("@cocalc/frontend/i18n/ru_RU.compiled.json");
+        return import("@cocalc/frontend/i18n/trans/ru_RU.compiled.json");
       case "ja":
-        return import("@cocalc/frontend/i18n/ja_JP.compiled.json");
+        return import("@cocalc/frontend/i18n/trans/ja_JP.compiled.json");
       case "pt":
-        return import("@cocalc/frontend/i18n/pt_PT.compiled.json");
+        return import("@cocalc/frontend/i18n/trans/pt_PT.compiled.json");
       case "ko":
-        return import("@cocalc/frontend/i18n/ko_KR.compiled.json");
+        return import("@cocalc/frontend/i18n/trans/ko_KR.compiled.json");
       case "pl":
-        return import("@cocalc/frontend/i18n/pl_PL.compiled.json");
+        return import("@cocalc/frontend/i18n/trans/pl_PL.compiled.json");
       case "tr":
-        return import("@cocalc/frontend/i18n/tr_TR.compiled.json");
+        return import("@cocalc/frontend/i18n/trans/tr_TR.compiled.json");
       case "he":
-        return import("@cocalc/frontend/i18n/he_IL.compiled.json");
+        return import("@cocalc/frontend/i18n/trans/he_IL.compiled.json");
       default:
         unreachable(locale);
         throw new Error(`Unknown locale '${locale}.`);