From ea3978042ce145685265ad85dee1152aa41eda81 Mon Sep 17 00:00:00 2001 From: vallyian Date: Tue, 21 Mar 2023 09:35:18 +0200 Subject: [PATCH 1/2] init --- .editorconfig | 16 ++++ .gitattributes | 3 + .github/workflows/publish.yml | 36 ++++++++ .github/workflows/test.yml | 26 ++++++ .gitignore | 6 ++ README.md | 59 ++++++++++++++ env.js | 21 +++++ index.d.ts | 7 ++ index.js | 22 +++++ install.js | 149 ++++++++++++++++++++++++++++++++++ package.json | 44 ++++++++++ test.js | 53 ++++++++++++ 12 files changed, 442 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitattributes create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/test.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 env.js create mode 100644 index.d.ts create mode 100644 index.js create mode 100644 install.js create mode 100644 package.json create mode 100644 test.js diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..2e6796b --- /dev/null +++ b/.editorconfig @@ -0,0 +1,16 @@ +# Editor configuration, see http://editorconfig.org +root = true + +[*] +charset = utf-8 +indent_style = space +indent_size = 4 +insert_final_newline = true +trim_trailing_whitespace = true + +[*.js] +quote_type = double + +[*.md] +max_line_length = off +trim_trailing_whitespace = false diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6795149 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +*.sh text eol=lf +*.bash text eol=lf +*.js text eol=lf diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..5f08dc1 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,36 @@ +name: publish + +on: + release: + types: [created] + +jobs: + test: + strategy: + matrix: + os: [macos-latest, ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2 + with: + node-version: 16 + registry-url: https://registry.npmjs.org/ + + - run: node test + + publish: + needs: [test] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2 + with: + node-version: 16 + registry-url: https://registry.npmjs.org/ + + - env: + NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} + run: npm publish diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..793aaa9 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,26 @@ +name: test + +on: + push: + branches: [main] + + pull_request: + branches: [main] + + workflow_dispatch: + +jobs: + test: + strategy: + matrix: + os: [macos-latest, ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2 + with: + node-version: 16 + registry-url: https://registry.npmjs.org/ + + - run: node test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..24712e2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +node_modules +protoc-gen-js +test +tmp +package-lock.json +*.tgz diff --git a/README.md b/README.md new file mode 100644 index 0000000..9474477 --- /dev/null +++ b/README.md @@ -0,0 +1,59 @@ +# protoc-gen-js-binary + +Downloads Google Protocol Buffers Javascript generator binary wrapped as npm package. +By default, it will download the latest released version. +If a specific version is required, add `"protoc-gen-js-binary": "x.x.x"` at the root of your package.json. + +## Install + +`npm i -D protoc-gen-js-binary` + +To force re-check for latest protoc-gen-js binary releases, simply run `npm ci`. +Alternatively, you can manually invoke the install script `node node_modules/protoc-gen-js-binary/install`. +If working in both Windows and WSL, you can invoke the install script to download binaries for both, +however when switching OS you should run `npm ci`. + +## Usage + +Add `--plugin=protoc-gen-js=node_modules/protoc-gen-js-binary/protoc-gen-js` arg to `protoc` command + +```sh +# download protoc and protoc-gen-js binaries +npm install protoc-gen-js-binary protoc-binary + +# replace `input_dir`, `output_dir` and `my.proto` with actual values +node_modules/.bin/protoc \ + --plugin=protoc-gen-js=node_modules/protoc-gen-js-binary/protoc-gen-js \ + --js_out=import_style=commonjs:input_dir \ + -I=output_dir \ + my.proto +``` + +Alternatively, add the full path of `protoc-gen-js-binary` to PATH +e.g. `/home/user/node_modules/protoc-gen-js-binary` + +## API + +### `binary` + +```js +/* Returns the absolute path to local protoc-gen-js binary */ +require("protoc-gen-js-binary").binary; +``` + +### `version` + +```js +/* Returns version of local protoc-gen-js binary */ +require("protoc-gen-js-binary").version; +``` + +## Supported versions + +See official [protocolbuffers/protobuf-javascript](https://github.com/protocolbuffers/protobuf-javascript/releases) download page. + +* osx-x86_64.zip +* linux-x86_32.zip +* linux-x86_64.zip +* win32.zip +* win64.zip diff --git a/env.js b/env.js new file mode 100644 index 0000000..5075d6a --- /dev/null +++ b/env.js @@ -0,0 +1,21 @@ +const path = require("path"); + +const binaryZip = { + "darwin-x64": "osx-x86_64.zip", + "linux-x32": "linux-x86_32.zip", + "linux-x64": "linux-x86_64.zip", + "win32-x32": "win32.zip", + "win32-x64": "win64.zip" +}[process.platform + "-" + process.arch]; + +module.exports = Object.freeze({ + binary: path.join(__dirname, "protoc-gen-js"), + binaryZip, + downloadUrlTemplate: `https://github.com/protocolbuffers/protobuf-javascript/releases/download/v{version}/protobuf-javascript-{version}-${binaryZip}`, + latestReleaseUrl: "https://api.github.com/repos/protocolbuffers/protobuf-javascript/releases/latest", + safeUnzip: Object.freeze({ + MAX_FILES: 1_000, + MAX_SIZE: 10_000_000, // 10 MB + MAX_RATIO: 20 + }) +}); diff --git a/index.d.ts b/index.d.ts new file mode 100644 index 0000000..bee51b5 --- /dev/null +++ b/index.d.ts @@ -0,0 +1,7 @@ +declare module "protoc-gen-js-binary" { + /** Absolute path to local protoc-gen-js binary */ + export const binary: string; + + /** Version of local protoc-gen-js binary */ + export const version: string | ""; +} diff --git a/index.js b/index.js new file mode 100644 index 0000000..3dfb6c3 --- /dev/null +++ b/index.js @@ -0,0 +1,22 @@ +#!/usr/bin/env node +const fs = require("node:fs"); +const evn = require("./env"); + +module.exports = Object.freeze({ + /** Absolute path to local protoc-gen-js binary */ + binary: evn.binary, + + /** Version of local protoc-gen-js binary, or empty string */ + get version() { return getBinaryVersion(); }, +}); + +function getBinaryVersion() { + if (!fs.existsSync(evn.binary)) return ""; + let ret; + try { + ret = require("./package.json")["protoc-gen-js-version"]; + } catch (ex) { + ret = ""; + } + return /[0-9.]/.test(ret) ? ret : ""; +} diff --git a/install.js b/install.js new file mode 100644 index 0000000..5b76585 --- /dev/null +++ b/install.js @@ -0,0 +1,149 @@ +const fs = require("node:fs"); +const path = require("node:path"); +const https = require("node:https"); +const extract = require("extract-zip"); +const env = require("./env"); +const index = require("./index"); + +const binaryZipRx = new RegExp(`\/protobuf-javascript-(.*)-${env.binaryZip}`); + +install(); + +function install() { + let version = ""; + let downloadUrl = ""; + const tmpDir = path.join(__dirname, "tmp"); + const tmpZipFile = path.join(tmpDir, "protoc-gen-js.zip"); + const tmpBinFile = path.join(tmpDir, "bin", `protoc-gen-js${process.platform === "win32" ? ".exe" : ""}`); + + const validatePlatform = () => env.binaryZip + || Promise.reject(`${process.platform}-${process.arch} unsupported`); + const checkBinaryVersion = () => getVersionInfo().then(bin => { + version = bin.version; + downloadUrl = bin.downloadUrl; + if (bin.version === index.version) return Promise.reject("ERR_BIN_EEXIST"); + }); + const removeBinary = () => fs.promises.rm(env.binary, { recursive: true, force: true }); + const removeTmpDir = () => fs.promises.rm(tmpDir, { recursive: true, force: true }); + const cleanDir = () => Promise.all([removeBinary(), removeTmpDir()]); + const createTmpDir = () => fs.promises.mkdir(tmpDir, { recursive: true }); + const downloadZip = () => download(downloadUrl, tmpZipFile).then(() => fs.existsSync(tmpZipFile) + || Promise.reject(`binary "${env.binaryZip} not downloaded"`)); + const deflateZip = () => unzip(tmpZipFile, tmpDir).then(() => fs.existsSync(tmpBinFile) + || Promise.reject(`binary "${tmpBinFile} not unzipped"`)); + const moveBinary = () => fs.promises.rename(tmpBinFile, env.binary).then(() => fs.existsSync(env.binary) + || Promise.reject(`binary "${env.binary} not available"`)); + const makeBinaryExecutable = () => fs.promises.access(env.binary, fs.constants.X_OK).catch(({ code }) => code === "EACCES" + ? fs.promises.chmod(env.binary, 0o775) + : Promise.reject(`binary "${env.binary} not executable"`)); + const storeBinaryVersion = () => fs.promises.writeFile( + "package.json", + JSON.stringify({ ...require("./package.json"), "protoc-gen-js-version": version }, null, 4), + "utf8"); + + return Promise.resolve() + .then(validatePlatform) + .then(checkBinaryVersion) + .then(cleanDir) + .then(createTmpDir) + .then(() => console.log(`downloading protoc-gen-js v${version} from ${downloadUrl}`)) + .then(downloadZip) + .then(deflateZip) + .then(moveBinary) + .then(makeBinaryExecutable) + .then(storeBinaryVersion) + .then(() => console.log(`downloaded protoc-gen-js v${version}`)) + .catch(err => err === "ERR_BIN_EEXIST" + ? console.log(`latest protoc-gen-js v${version} already exists, skipping download`) + : Promise.reject(err)) + .finally(removeTmpDir); +} + +async function unzip(zip, dir) { + let fileCount = 0; + let totalSize = 0; + + await extract(zip, { + dir, + onEntry: entry => { + fileCount++; + if (fileCount > env.safeUnzip.MAX_FILES) + throw Error('Reached max. number of files'); + + let entrySize = entry.uncompressedSize; + totalSize += entrySize; + if (totalSize > env.safeUnzip.MAX_SIZE) + throw Error('Reached max. size'); + + if (entry.compressedSize > 0) { + let compressionRatio = entrySize / entry.compressedSize; + if (compressionRatio > env.safeUnzip.MAX_RATIO) + throw Error('Reached max. compression ratio'); + } + } + }); +} + +async function getVersionInfo() { + let version = getRequestedVersion(); + let downloadUrl = ""; + + if (version) { + downloadUrl = env.downloadUrlTemplate.replace(/\{version\}/g, version); + } else { + process.stdout.write(`querying latest protoc-gen-js version...`); + downloadUrl = await getLatestReleaseLink().catch(e => { + process.stdout.write('\n'); + throw e; + }); + version = downloadUrl.match(binaryZipRx)[1]; + process.stdout.write(`v${version} found\n`); + } + + return { version, downloadUrl }; +} + +function getRequestedVersion() { + let dir = process.cwd(); + let requestedVersion = undefined; + while (!requestedVersion) { + const packageJsonPath = path.join(dir, "package.json"); + if (fs.existsSync(packageJsonPath)) + requestedVersion = require(packageJsonPath)["protoc-gen-js-binary"]; + if (requestedVersion || !dir.includes("node_modules")) + break; + dir = path.normalize(path.join(dir, "..")); + } + return requestedVersion || ""; +} + +function getLatestReleaseLink() { + return new Promise((resolve, reject) => https.get(env.latestReleaseUrl, { headers: { "User-Agent": `Nodejs/${process.version}` } }, response => { + let data = ""; + response.on("data", chunk => data += chunk); + response.on("end", () => { + const link = JSON.parse(data).assets.find(a => binaryZipRx.test(a.browser_download_url)); + link + ? resolve(link.browser_download_url) + : reject(`binary ${env.binaryZip} not available`); + }); + }).on("error", reject)); +} + +function download(uri, filename) { + return new Promise((resolve, reject) => https.get(uri, response => { + if (response.statusCode === 200) + response.pipe( + fs.createWriteStream(filename) + .on("error", reject) + .on("close", resolve) + ); + + else if (response.headers.location) + resolve(download(response.headers.location, filename)); + + else + reject(Error(`${response.statusCode} ${response.statusMessage}`)); + + }).on("error", reject)); +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..03f8a64 --- /dev/null +++ b/package.json @@ -0,0 +1,44 @@ +{ + "name": "protoc-gen-js-binary", + "homepage": "https://github.com/vallyian/protoc-gen-js-binary#readme", + "description": "Downloads latest Google Protocol Buffers Javascript generator binary wrapped as npm package", + "keywords": [ + "download", + "latest", + "google", + "protocol", + "buffers", + "javascript", + "generator", + "protoc-gen-js", + "binary" + ], + "version": "1.0.0", + "author": "vallyian@gmail.com", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/vallyian/protoc-gen-js-binary.git" + }, + "bugs": { + "url": "https://github.com/vallyian/protoc-gen-js-binary/issues" + }, + "engines": { + "node": ">=14", + "npm": ">=6" + }, + "main": "index.js", + "types": "./index.d.ts", + "scripts": { + "postinstall": "node install" + }, + "dependencies": { + "extract-zip": "latest" + }, + "files": [ + "env.js", + "index.d.ts", + "index.js", + "install.js" + ] +} diff --git a/test.js b/test.js new file mode 100644 index 0000000..b5175d9 --- /dev/null +++ b/test.js @@ -0,0 +1,53 @@ +const fs = require("node:fs"); +const child_process = require("node:child_process"); + +test().catch(err => { + console.error(`\x1b[31m${typeof err === "string" ? err : JSON.stringify(err)}\x1b[0m`); + process.exit(1); +}).finally(clean); + +async function test() { + clean(); + await execP(cmd("npm"), ["pack"]); + fs.mkdirSync("test", { recursive: true }); + fs.renameSync(tgz()[0], "test/protoc-gen-js-binary.tgz"); + await execP(cmd("npm"), ["init", "-y"], "test"); + await execP(cmd("npm"), ["i", "-D", "./protoc-gen-js-binary.tgz"], "test"); + await execP(cmd("npm"), ["i", "-D", "protoc-binary"], "test"); + console.log(require("./test/package.json").devDependencies); + fs.writeFileSync("test/test.proto", ` + message EchoRequest { required string message = 1; } + message EchoResponse { required string message = 1; } + service EchoService { rpc Echo(EchoRequest) returns (EchoResponse); } + `, "utf8"); + await execP(cmd("test/node_modules/.bin/protoc"), [ + "--plugin=protoc-gen-js=test/node_modules/protoc-gen-js-binary/protoc-gen-js", + "--js_out=import_style=commonjs:test", + "-I=test", + "test/test.proto", + ]); + if (!fs.existsSync("test/test_pb.js")) + throw Error("js not generated"); +} + +function clean() { + fs.rmSync("test", { recursive: true, force: true }); + tgz().forEach(f => fs.rmSync(f, { force: true })); +} + +function tgz() { return fs.readdirSync(".").filter(f => f.endsWith(".tgz")); } + +function cmd(exe) { return process.platform === "win32" ? `${exe.replace(/\//gmi, "\\")}.cmd` : exe; } + +function execP(cmd, args, cwd = process.cwd()) { + return new Promise((ok, reject) => { + const child = child_process.spawn( + cmd, + args.filter(c => c && !!(String(c).trim())), + { shell: false, cwd } + ); + child.stdout.pipe(process.stdout); + child.stderr.pipe(process.stderr); + child.on("exit", code => code ? reject(code) : ok()) + }); +} From 1259d5e4cf5127ccbd169916c8b2c65acba44324 Mon Sep 17 00:00:00 2001 From: vallyian Date: Tue, 21 Mar 2023 09:49:20 +0200 Subject: [PATCH 2/2] remove world permission for binary --- install.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.js b/install.js index 5b76585..d28f564 100644 --- a/install.js +++ b/install.js @@ -34,7 +34,7 @@ function install() { const moveBinary = () => fs.promises.rename(tmpBinFile, env.binary).then(() => fs.existsSync(env.binary) || Promise.reject(`binary "${env.binary} not available"`)); const makeBinaryExecutable = () => fs.promises.access(env.binary, fs.constants.X_OK).catch(({ code }) => code === "EACCES" - ? fs.promises.chmod(env.binary, 0o775) + ? fs.promises.chmod(env.binary, 0o770) : Promise.reject(`binary "${env.binary} not executable"`)); const storeBinaryVersion = () => fs.promises.writeFile( "package.json",