New build system to create Noto Sans with all characters (including CJK)

.gitignore

@@ -0,0 +1,2 @@
+_output
+node_modules

README.md

@@ -24,25 +24,26 @@ In order to use the fonts without a Mapbox API key edit your Mapbox GL Style JSO
 
 The following fonts that are available in Mapbox Studio are supported.
 
-* Google Noto (patched by Klokan Technologies)
+* Noto Sans (patched by Klokan Technologies)
 * Open Sans
 * PT Sans
 * Roboto
 * Metropolis
 
 ## Package the Fonts
 
-Install [genfontgl](https://github.com/sabas/genfontgl).
+Install required packages:
 
 ```
-npm install genfontgl
+npm install
 ```
 
-Generate fonts.
+Generate fonts:
 
 ```
-./generate.sh
+node ./generate.js
 ```
+The PBFs will created be in the `_output` directory.
 
 ## Font License
 

generate.js

@@ -0,0 +1,151 @@
+var fs = require('fs'),
+    path = require('path');
+
+var fontnik = require('fontnik'),
+    glyphCompose = require('@mapbox/glyph-pbf-composite');
+
+var DEBUG = false;
+
+var outputDir = '_output';
+
+var sizeSumTotal = 0;
+
+if (!fs.existsSync(outputDir)) {
+  fs.mkdirSync(outputDir);
+}
+
+var doFonts = function(dir, fonts) {
+  var makeGlyphs = function(config) {
+    var sourceFonts = {};
+
+    var folderName = outputDir + '/' + config.name;
+
+    config.sources.forEach(function(sourceName) {
+      if (!sourceFonts[sourceName]) {
+        try {
+          sourceFonts[sourceName] = fs.readFileSync(dir + '/' + sourceName);
+        } catch (e) {}
+      }
+    });
+
+    if (!fs.existsSync(folderName)) {
+      fs.mkdirSync(folderName);
+    }
+
+    var sizeSum = 0;
+    var histogram = new Array(256);
+
+    var doRange = function(start, end) {
+      return Promise.all(config.sources.map(function(sourceName) {
+        var source = sourceFonts[sourceName];
+        if (!source) {
+          console.log('[%s] Source "%s" not found', config.name, sourceName);
+          return Promise.resolve();
+        }
+
+        return new Promise(function(resolve, reject) {
+          fontnik.range({
+            font: source,
+            start: start,
+            end: end
+          }, function(err, data) {
+            if (err) {
+              reject();
+            } else {
+              resolve(data);
+            }
+          });
+        });
+      })).then(function(results) {
+        results = results.filter(function(r) {return !!r;});
+        var combined = glyphCompose.combine(results);
+        var size = combined.length;
+        sizeSum += size;
+        histogram[start / 256] = size;
+        if (DEBUG) {
+          console.log('[%s] Range %s-%s size %s B', config.name, start, end, size);
+        }
+        fs.writeFileSync(folderName + '/' + start + '-' + end + '.pbf', combined);
+      });
+    };
+
+    var ranges = [];
+    for (var i = 0; i < 65536; (i = i + 256)) {
+      ranges.push([i, Math.min(i + 255, 65535)]);
+    }
+
+    console.log('[%s]', config.name);
+    var fontPromise;
+    if (DEBUG) {
+      return ranges.reduce(function(p, range) {
+          return p.then(function() {
+            return doRange(range[0], range[1]);
+          });
+        }, Promise.resolve()
+      );
+    } else {
+      fontPromise = Promise.all(ranges.map(function(range) {
+        return doRange(range[0], range[1]);
+      }));
+    }
+    return fontPromise.then(function() {
+      console.log(' Size histo [kB]: %s', histogram.map(function(v) {
+          return v > 512 ? Math.round(v / 1024) : '';
+        }).join('|'));
+      console.log(' Total size %s B', sizeSum);
+      sizeSumTotal += sizeSum;
+    });
+  };
+
+  // would be much faster in parallel, but this is better for logging
+  return fonts.reduce(function(p, font) {
+      return p.then(function() {
+        return makeGlyphs(font);
+      });
+    }, Promise.resolve()
+  );
+};
+
+var todo = [];
+fs.readdirSync('.').forEach(function(dir) {
+  if (fs.lstatSync(dir).isDirectory()) {
+    var fonts;
+    try {
+      fonts = require(path.resolve(__dirname, dir, 'fonts.json'));
+      fonts.forEach(function(font) {
+        font.sources = font.sources.filter(function(f) {
+          // skip sources starting with '//' -- these are "commented"
+          return f.indexOf('//') === -1;
+        });
+      });
+    } catch (e) {
+      fonts = [];
+      fs.readdirSync(dir).forEach(function(file) {
+        if (path.extname(file) == '.ttf' || path.extname(file) == '.otf') {
+          // compatible font name generation with genfontgl
+          var rex = /([A-Z])([A-Z])([a-z])|([a-z])([A-Z])/g;
+          fonts.push({
+            name: path.basename(file).slice(0, -4).replace('-','').replace(rex, '$1$4 $2$3$5'),
+            sources: [
+              path.basename(file)
+            ]
+          });
+        }
+      });
+    }
+    if (fonts && fonts.length) {
+      todo.push([dir, fonts]);
+    }
+  }
+});
+
+// would be much faster in parallel, but this is better for logging
+todo.reduce(function(p, pair) {
+    return p.then(function() {
+      console.log('Directory [%s]:', pair[0]);
+      return doFonts(pair[0], pair[1]);
+    });
+  }, Promise.resolve()
+).then(function() {
+  console.log('Total size %s B', sizeSumTotal);
+});

noto-sans/fonts.json

@@ -0,0 +1,89 @@
+[
+  {
+    "name": "Noto Sans Regular",
+    "sources": [
+      "NotoSans-Regular.ttf",
+      "NotoNaskhArabic-Regular.ttf",
+      "NotoSansArmenian-Regular.ttf",
+      "NotoSansBalinese-Regular.ttf",
+      "NotoSansBengali-Regular.ttf",
+      "NotoSansCJKtc-Regular.otf",
+      "NotoSansDevanagari-Regular.ttf",
+      "NotoSansEthiopic-Regular.ttf",
+      "NotoSansGeorgian-Regular.ttf",
+      "NotoSansGujarati-Regular.ttf",
+      "NotoSansGurmukhi-Regular.ttf",
+      "NotoSansHebrew-Regular.ttf",
+      "NotoSansJavanese-Regular.ttf",
+      "NotoSansKannada-Regular.ttf",
+      "NotoSansKhmer-Regular.ttf",
+      "NotoSansLao-Regular.ttf",
+      "NotoSansMongolian-Regular.ttf",
+      "NotoSansMyanmar-Regular.ttf",
+      "NotoSansOriya-Regular.ttf",
+      "NotoSansSinhala-Regular.ttf",
+      "NotoSansTamil-Regular.ttf",
+      "NotoSansThai-Regular.ttf",
+      "NotoSansTibetan-Regular.ttf",
+      "//NotoSansTifinagh-Regular.ttf"
+    ]
+  },
+  {
+    "name": "Noto Sans Bold",
+    "sources": [
+      "NotoSans-Bold.ttf",
+      "NotoNaskhArabic-Bold.ttf",
+      "NotoSansArmenian-Bold.ttf",
+      "NotoSansBalinese-Regular.ttf",
+      "NotoSansBengali-Bold.ttf",
+      "NotoSansCJKtc-Bold.otf",
+      "NotoSansDevanagari-Bold.ttf",
+      "NotoSansEthiopic-Bold.ttf",
+      "NotoSansGeorgian-Bold.ttf",
+      "NotoSansGujarati-Bold.ttf",
+      "NotoSansGurmukhi-Bold.ttf",
+      "NotoSansHebrew-Bold.ttf",
+      "NotoSansJavanese-Regular.ttf",
+      "NotoSansKannada-Bold.ttf",
+      "NotoSansKhmer-Bold.ttf",
+      "NotoSansLao-Bold.ttf",
+      "NotoSansMongolian-Regular.ttf",
+      "NotoSansMyanmar-Bold.ttf",
+      "NotoSansOriya-Bold.ttf",
+      "NotoSansSinhala-Bold.ttf",
+      "NotoSansTamil-Bold.ttf",
+      "NotoSansThai-Bold.ttf",
+      "NotoSansTibetan-Bold.ttf",
+      "//NotoSansTifinagh-Regular.ttf"
+    ]
+  },
+  {
+    "name": "Noto Sans Italic",
+    "sources": [
+      "NotoSans-Italic.ttf",
+      "NotoNaskhArabic-Regular.ttf",
+      "NotoSansArmenian-Regular.ttf",
+      "NotoSansBalinese-Regular.ttf",
+      "NotoSansBengali-Regular.ttf",
+      "NotoSansCJKtc-Regular.otf",
+      "NotoSansDevanagari-Regular.ttf",
+      "NotoSansEthiopic-Regular.ttf",
+      "NotoSansGeorgian-Regular.ttf",
+      "NotoSansGujarati-Regular.ttf",
+      "NotoSansGurmukhi-Regular.ttf",
+      "NotoSansHebrew-Regular.ttf",
+      "NotoSansJavanese-Regular.ttf",
+      "NotoSansKannada-Regular.ttf",
+      "NotoSansKhmer-Regular.ttf",
+      "NotoSansLao-Regular.ttf",
+      "NotoSansMongolian-Regular.ttf",
+      "NotoSansMyanmar-Regular.ttf",
+      "NotoSansOriya-Regular.ttf",
+      "NotoSansSinhala-Regular.ttf",
+      "NotoSansTamil-Regular.ttf",
+      "NotoSansThai-Regular.ttf",
+      "NotoSansTibetan-Regular.ttf",
+      "//NotoSansTifinagh-Regular.ttf"
+    ]
+  }
+]

package.json

@@ -0,0 +1,11 @@
+{
+  "name": "klokantech-noto-sans",
+  "version": "0.0.0",
+  "authors": [
+    "Petr Sloup <[email protected]>"
+  ],
+  "dependencies": {
+    "@mapbox/glyph-pbf-composite": "0.0.3",
+    "fontnik": "0.5.0"
+  }
+}