feat(build)!: Introduce ESM entrypoints (#8091)

* feat(build)!: Introduce ESM entrypoints for chunks

  Introduce an "import" conditional export for each of the chunk
  entrypoints (blockly/core, blockly/blocks, blockly/javascript
  etc.), and point these at wrappers created by build_tasks.js
  that import the corresponding <chunk>_compressed.js file and
  export its named exports.

  BREAKING CHANGE:

  Importing Blockly via

      import Blockly from 'blockly/core';

  (and similarly for the other chunk entrypoints) has worked until
  now because most build tools (including Webpack in particular)
  fuilfil the request for the default export of a CJS module by
  providing the module.exports object, rather than an
  explicitly-named default export as they would for an ES module.

  Since core/blockly.ts (the notional entrypoint for blockly/core)
  does not provide a default export, the wrappers created by this
  PR do not either.

  Code of the above form will therefore break, and should be updated
  to use a wildcard:

      import * as Blockly from 'blockly/core';

* feat(build)!: Introduce main package ESM entrypoint

  Introduce an "import" conditional export for the top-level
  package entrypoint (blockly), and point it at a wrappers
  created by build_tasks.js that imports the existing index.js
  file.

  BREAKING CHANGE:

  Importing Blockly via

      import Blockly from 'blockly';

  has worked until now because most build tools (including Webpack
  in particular) fuilfil the request for the default export of a
  CJS module by providing the module.exports object, rather than an
  explicitly-named default export as they would for an ES module.

  Since core/blockly.ts does not provide a default export, the
  wrapper created by this PR does not either.

  Code of the above form will therefore break, and should be updated
  to use a wildcard:

      import * as Blockly from 'blockly';

* feat(build)!: Introduce ESM entrypoints for langfiles

  Introduce an "import" conditional export for each of the
  langfile entrypoints (msg/en, msg/fr, etc.),, and point them
  at wrappers created by build_tasks.js that import the
  existing <lang>.js file.

  BREAKING CHANGE:

  Importing languages via

      import en from 'blockly/msg/en';

  has worked until now because most build tools (including Webpack
  in particular) fuilfil the request for the default export of a
  CJS module by providing the module.exports object, rather than an
  explicitly-named default export as they would for an ES module.

  Code of the above form will therefore break, and should be updated
  to use a wildcard:

      import * as en from 'blockly/msg/en';

* fix(typings): Remove bogus .d.ts file.

  For some reason we had a typings/msg/yue.d.ts that did not
  correxpond to any msg/json/yue.json.  Delete it.

package.json

@@ -57,40 +57,49 @@
   "exports": {
     ".": {
       "types": "./index.d.ts",
+      "import": "./index.mjs",
       "umd": "./blockly.min.js",
       "default": "./index.js"
     },
     "./core": {
       "types": "./core.d.ts",
       "node": "./core-node.js",
+      "import": "./blockly.mjs",
       "default": "./blockly_compressed.js"
     },
     "./blocks": {
       "types": "./blocks.d.ts",
+      "import": "./blocks.mjs",
       "default": "./blocks_compressed.js"
     },
     "./dart": {
       "types": "./dart.d.ts",
+      "import": "./dart.mjs",
       "default": "./dart_compressed.js"
     },
     "./lua": {
       "types": "./lua.d.ts",
+      "import": "./lua.mjs",
       "default": "./lua_compressed.js"
     },
     "./javascript": {
       "types": "./javascript.d.ts",
+      "import": "./javascript.mjs",
       "default": "./javascript_compressed.js"
     },
     "./php": {
       "types": "./php.d.ts",
+      "import": "./php.mjs",
       "default": "./php_compressed.js"
     },
     "./python": {
       "types": "./python.d.ts",
+      "import": "./python.mjs",
       "default": "./python_compressed.js"
     },
     "./msg/*": {
       "types": "./msg/*.d.ts",
+      "import": "./msg/*.mjs",
       "default": "./msg/*.js"
     }
   },

scripts/gulpfiles/build_tasks.js

@@ -344,6 +344,24 @@ this removal!
   done();
 }
 
+var languages = null;
+
+/**
+ * Get list of languages to build langfiles and/or shims for, based on .json
+ * files in msg/json/, skipping certain entries that do not correspond to an
+ * actual language).  Results are cached as this is called from both
+ * buildLangfiles and buildLangfileShims.
+ */
+function getLanguages() {
+  if (!languages) {
+    const skip = /^(keys|synonyms|qqq|constants)\.json$/;
+    languages = fs.readdirSync(path.join('msg', 'json'))
+        .filter(file => file.endsWith('json') && !skip.test(file))
+        .map(file => file.replace(/\.json$/, ''));
+  }
+  return languages;
+}
+
 /**
  * This task builds Blockly's lang files.
  *     msg/*.js
@@ -353,18 +371,16 @@ function buildLangfiles(done) {
   fs.mkdirSync(LANG_BUILD_DIR, {recursive: true});
 
   // Run create_messages.py.
-  let json_files = fs.readdirSync(path.join('msg', 'json'));
-  json_files = json_files.filter(file => file.endsWith('json') &&
-      !(new RegExp(/(keys|synonyms|qqq|constants)\.json$/).test(file)));
-  json_files = json_files.map(file => path.join('msg', 'json', file));
+  const inputFiles = getLanguages().map(
+      lang => path.join('msg', 'json', `${lang}.json`));
 
   const createMessagesCmd = `${PYTHON} ./scripts/i18n/create_messages.py \
   --source_lang_file ${path.join('msg', 'json', 'en.json')} \
   --source_synonym_file ${path.join('msg', 'json', 'synonyms.json')} \
   --source_constants_file ${path.join('msg', 'json', 'constants.json')} \
   --key_file ${path.join('msg', 'json', 'keys.json')} \
   --output_dir ${LANG_BUILD_DIR} \
-  --quiet ${json_files.join(' ')}`;
+  --quiet ${inputFiles.join(' ')}`;
   execSync(createMessagesCmd, {stdio: 'inherit'});
 
   done();
@@ -565,7 +581,10 @@ function buildCompiled() {
 }
 
 /**
- * This task builds the shims used by the playgrounds and tests to
+ * This task builds the ESM wrappers used by the chunk "import"
+ * entrypoints declared in package.json.
+ *
+ * Also builds the shims used by the playgrounds and tests to
  * load Blockly in either compressed or uncompressed mode, creating
  * build/blockly.loader.mjs, blocks.loader.mjs, javascript.loader.mjs,
  * etc.
@@ -579,26 +598,52 @@ async function buildShims() {
   const TMP_PACKAGE_JSON = path.join(BUILD_DIR, 'package.json');
   await fsPromises.writeFile(TMP_PACKAGE_JSON, '{"type": "module"}');
 
-  // Import each entrypoint module, enumerate its exports, and write
-  // a shim to load the chunk either by importing the entrypoint
-  // module or by loading the compiled script.
   await Promise.all(chunks.map(async (chunk) => {
+    // Import chunk entrypoint to get names of exports for chunk.
     const entryPath = path.posix.join(TSC_OUTPUT_DIR_POSIX, chunk.entry);
+    const exportedNames = Object.keys(await import(`../../${entryPath}`));
+
+    // Write an ESM wrapper that imports the CJS module and re-exports
+    // its named exports.
+    const cjsPath = `./${chunk.name}${COMPILED_SUFFIX}.js`;
+    const wrapperPath = path.join(RELEASE_DIR, `${chunk.name}.mjs`);
+    const importName = chunk.scriptExport.replace(/.*\./, '');
+
+    await fsPromises.writeFile(wrapperPath,
+        `import ${importName} from '${cjsPath}';
+export const {
+${exportedNames.map((name) => `  ${name},`).join('\n')}
+} = ${importName};
+`);
+
+    // For first chunk, write an additional ESM wrapper for 'blockly'
+    // entrypoint since it has the same exports as 'blockly/core'.
+    if (chunk.name === 'blockly') {
+      await fsPromises.writeFile(path.join(RELEASE_DIR, `index.mjs`),
+          `import Blockly from './index.js';
+export const {
+${exportedNames.map((name) => `  ${name},`).join('\n')}
+} = Blockly;
+`);
+    }
+
+    // Write a loading shim that uses loadChunk to either import the
+    // chunk's entrypoint (e.g. build/src/core/blockly.js) or load the
+    // compressed chunk (e.g. dist/blockly_compressed.js) as a script.
     const scriptPath =
         path.posix.join(RELEASE_DIR, `${chunk.name}${COMPILED_SUFFIX}.js`);
     const shimPath = path.join(BUILD_DIR, `${chunk.name}.loader.mjs`);
     const parentImport =
         chunk.parent ?
         `import ${quote(`./${chunk.parent.name}.loader.mjs`)};` :
         '';
-    const exports = await import(`../../${entryPath}`);
 
     await fsPromises.writeFile(shimPath,
         `import {loadChunk} from '../tests/scripts/load.mjs';
 ${parentImport}
 
 export const {
-${Object.keys(exports).map((name) => `  ${name},`).join('\n')}
+${exportedNames.map((name) => `  ${name},`).join('\n')}
 } = await loadChunk(
   ${quote(entryPath)},
   ${quote(scriptPath)},
@@ -610,7 +655,37 @@ ${Object.keys(exports).map((name) => `  ${name},`).join('\n')}
   await fsPromises.rm(TMP_PACKAGE_JSON);
 }
 
-
+/**
+ * This task builds the ESM wrappers used by the langfiles "import"
+ * entrypoints declared in package.json.
+ */
+async function buildLangfileShims() {
+  // Create output directory.
+  fs.mkdirSync(path.join(RELEASE_DIR, 'msg'), {recursive: true});
+
+  // Get the names of the exports from the langfile by require()ing
+  // msg/messages.js and letting it mutate the (global) Blockly.Msg.
+  // (We have to do it this way because messages.js is a script and
+  // not a CJS module with exports.)
+  globalThis.Blockly = {Msg: {}};
+  require('../../msg/messages.js');
+  const exportedNames = Object.keys(globalThis.Blockly.Msg);
+  delete globalThis.Blockly;
+
+  await Promise.all(getLanguages().map(async (lang) => {
+    // Write an ESM wrapper that imports the CJS module and re-exports
+    // its named exports.
+    const cjsPath = `./${lang}.js`;
+    const wrapperPath = path.join(RELEASE_DIR, 'msg', `${lang}.mjs`);
+
+    await fsPromises.writeFile(wrapperPath,
+        `import ${lang} from '${cjsPath}';
+export const {
+${exportedNames.map((name) => `  ${name},`).join('\n')}
+} = ${lang};
+`);
+  }));
+}
 
 /**
  * This task builds Blockly core, blocks and generators together and uses
@@ -662,7 +737,7 @@ function cleanBuildDir() {
 
 // Main sequence targets.  Each should invoke any immediate prerequisite(s).
 exports.cleanBuildDir = cleanBuildDir;
-exports.langfiles = buildLangfiles;  // Build build/msg/*.js from msg/json/*.
+exports.langfiles = gulp.parallel(buildLangfiles, buildLangfileShims);
 exports.tsc = buildJavaScript;
 exports.minify = gulp.series(exports.tsc, buildCompiled, buildShims);
 exports.build = gulp.parallel(exports.minify, exports.langfiles);