diff --git a/.github/workflows/codesee-arch-diagram.yml b/.github/workflows/codesee-arch-diagram.yml new file mode 100644 index 0000000000..fe56ca3f89 --- /dev/null +++ b/.github/workflows/codesee-arch-diagram.yml @@ -0,0 +1,87 @@ +on: + push: + branches: + - master + pull_request_target: + types: [opened, synchronize, reopened] + +name: CodeSee Map + +jobs: + test_map_action: + runs-on: ubuntu-latest + continue-on-error: true + name: Run CodeSee Map Analysis + steps: + - name: checkout + id: checkout + uses: actions/checkout@v2 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + fetch-depth: 0 + + # codesee-detect-languages has an output with id languages. + - name: Detect Languages + id: detect-languages + uses: Codesee-io/codesee-detect-languages-action@latest + + - name: Configure JDK 16 + uses: actions/setup-java@v2 + if: ${{ fromJSON(steps.detect-languages.outputs.languages).java }} + with: + java-version: '16' + distribution: 'zulu' + + # CodeSee Maps Go support uses a static binary so there's no setup step required. + + - name: Configure Node.js 14 + uses: actions/setup-node@v2 + if: ${{ fromJSON(steps.detect-languages.outputs.languages).javascript }} + with: + node-version: '14' + + - name: Configure Python 3.x + uses: actions/setup-python@v2 + if: ${{ fromJSON(steps.detect-languages.outputs.languages).python }} + with: + python-version: '3.10' + architecture: 'x64' + + - name: Configure Ruby '3.x' + uses: ruby/setup-ruby@v1 + if: ${{ fromJSON(steps.detect-languages.outputs.languages).ruby }} + with: + ruby-version: '3.0' + + # We need the rust toolchain because it uses rustc and cargo to inspect the package + - name: Configure Rust 1.x stable + uses: actions-rs/toolchain@v1 + if: ${{ fromJSON(steps.detect-languages.outputs.languages).rust }} + with: + toolchain: stable + + - name: Generate Map + id: generate-map + uses: Codesee-io/codesee-map-action@latest + with: + step: map + api_token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }} + github_ref: ${{ github.ref }} + languages: ${{ steps.detect-languages.outputs.languages }} + + - name: Upload Map + id: upload-map + uses: Codesee-io/codesee-map-action@latest + with: + step: mapUpload + api_token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }} + github_ref: ${{ github.ref }} + + - name: Insights + id: insights + uses: Codesee-io/codesee-map-action@latest + with: + step: insights + api_token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }} + github_ref: ${{ github.ref }} diff --git a/.gitignore b/.gitignore index cd32161659..fd28fab785 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ !.env.example /result* /builds +/build +/prebuilds # Logs logs diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 11a1eb8257..9742adbcab 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,85 +1,253 @@ +workflow: + rules: + # Disable merge request pipelines + - if: $CI_MERGE_REQUEST_ID + when: never + - when: always + +default: + interruptible: true + variables: - GIT_SUBMODULE_STRATEGY: "recursive" + GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}" + GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git" + GIT_SUBMODULE_STRATEGY: recursive # Cache .npm - NPM_CONFIG_CACHE: "./tmp/npm" + NPM_CONFIG_CACHE: "${CI_PROJECT_DIR}/tmp/npm" # Prefer offline node module installation NPM_CONFIG_PREFER_OFFLINE: "true" # `ts-node` has its own cache - # It must use an absolute path, otherwise ts-node calls will CWD TS_CACHED_TRANSPILE_CACHE: "${CI_PROJECT_DIR}/tmp/ts-node-cache" TS_CACHED_TRANSPILE_PORTABLE: "true" + # Homebrew cache only used by macos runner + HOMEBREW_CACHE: "${CI_PROJECT_DIR}/tmp/Homebrew" -# Cached directories shared between jobs & pipelines per-branch +# Cached directories shared between jobs & pipelines per-branch per-runner cache: key: $CI_COMMIT_REF_SLUG paths: - ./tmp/npm/ - ./tmp/ts-node-cache/ + # Homebrew cache is only used by the macos runner + - ./tmp/Homebrew # `jest` cache is configured in jest.config.js - ./tmp/jest/ stages: - - check - - test - - build - - quality - - release - -lint: - image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + - check # Linting, unit tests + - build # Cross-platform library compilation, unit tests + - integration # Cross-platform application bundling, integration tests, and pre-release + - release # Cross-platform distribution and deployment + +image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + +check:lint: stage: check - interruptible: true + needs: [] script: - > - nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' - npm ci; + nix-shell --run ' npm run lint; ' + rules: + # Runs on feature and staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual -nix-dry: +check:nix-dry: stage: check - image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + needs: [] script: - - nix-build -v -v --dry-run ./release.nix --attr application - - nix-build -v -v --dry-run ./release.nix --attr docker - - nix-build -v -v --dry-run ./release.nix --attr package.linux.x64.elf - - nix-build -v -v --dry-run ./release.nix --attr package.windows.x64.exe - - nix-build -v -v --dry-run ./release.nix --attr package.macos.x64.macho + - nix-build -v -v --dry-run ./release.nix + rules: + # Runs on feature and staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual -test-generate: - image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner +check:test-generate: stage: check interruptible: true script: - mkdir -p ./tmp - > - nix-shell -I nixpkgs=./pkgs.nix --packages bash --run ' + nix-shell --run ' ./scripts/test-pipelines.sh > ./tmp/test-pipelines.yml ' artifacts: + when: always paths: - ./tmp/test-pipelines.yml + rules: + # Runs on feature and staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual -test: - stage: test - # Don't implicitly inherit top-level variables in child pipeline - # All inherited variables should be explicitly defined here - # Note that variables defined here override any variables defined in the child pipeline - # This causes a bug with $CI_PROJECT_DIR, which is expanded into an empty string +check:test: + stage: check + needs: + - check:test-generate inherit: variables: false trigger: include: - artifact: tmp/test-pipelines.yml - job: test-generate + job: check:test-generate strategy: depend + rules: + # Runs on feature and staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual + +build:merge: + stage: build + needs: [] + allow_failure: true + script: + # Required for `gh pr create` + - git remote add upstream "$GH_PROJECT_URL" + - > + nix-shell -I nixpkgs=./pkgs.nix --packages gitAndTools.gh --run ' + gh pr create \ + --head staging \ + --base master \ + --title "ci: merge staging to master" \ + --body "This is an automatic PR generated by the pipeline CI/CD. This will be automatically fast-forward merged if successful." \ + --assignee "@me" \ + --no-maintainer-edit \ + --repo "$GH_PROJECT_PATH" || true; + printf "Pipeline Attempt on ${CI_PIPELINE_ID} for ${CI_COMMIT_SHA}\n\n${CI_PIPELINE_URL}" \ + | gh pr comment staging \ + --body-file - \ + --repo "$GH_PROJECT_PATH"; + ' + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +build:linux: + stage: build + needs: [] + script: + - > + nix-shell --run ' + npm run build --verbose; + ' + artifacts: + when: always + paths: + # Only the build:linux preserves the dist + - ./dist + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +build:windows: + stage: build + needs: [] + tags: + - windows + before_script: + - choco install nodejs --version=16.14.2 -y + - refreshenv + script: + - npm config set msvs_version 2019 + - npm install --ignore-scripts + - $env:Path = "$(npm bin);" + $env:Path + - npm run build --verbose + # - npm test -- --ci + # artifacts: + # when: always + # reports: + # junit: + # - ./tmp/junit/junit.xml + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +build:macos: + stage: build + needs: [] + tags: + - shared-macos-amd64 + image: macos-11-xcode-12 + variables: + HOMEBREW_NO_INSTALL_UPGRADE: "true" + HOMEBREW_NO_INSTALL_CLEANUP: "true" + before_script: + - brew install node@16 + - brew link --overwrite node@16 + - hash -r + script: + - npm install --ignore-scripts + - export PATH="$(npm bin):$PATH" + - npm run build --verbose + # - npm test -- --ci + # artifacts: + # when: always + # reports: + # junit: + # - ./tmp/junit/junit.xml + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ -nix: +build:prerelease: stage: build - image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + needs: + - build:linux + - build:windows + - build:macos + # Don't interrupt publishing job + interruptible: false + before_script: + - echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc + script: + - echo 'Publishing library prerelease' + - > + nix-shell --run ' + npm publish --tag prerelease --access public; + ' + after_script: + - rm -f ./.npmrc + rules: + # Only runs on tag pipeline where the tag is a prerelease version + # This requires dependencies to also run on tag pipeline + # However version tag comes with a version commit + # Dependencies must not run on the version commit + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+-.*[0-9]+$/ + +integration:builds: + stage: integration + needs: + - build:linux + - build:windows + - build:macos script: - mkdir -p ./builds - # nix-specific application target - > build_application="$(nix-build \ --max-jobs "$(nproc)" --cores "$(nproc)" \ @@ -98,19 +266,40 @@ nix: --attr docker \ --attr package.linux.x64.elf \ --attr package.windows.x64.exe \ - --attr package.macos.x64.macho)" + --attr package.macos.x64.macho \ + --attr package.macos.arm64.macho)" - cp -r $builds ./builds/ - only: - - master artifacts: paths: - ./builds/ + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +integration:deployment: + stage: integration + needs: + - integration:builds + # Don't interrupt deploying job + interruptible: false + # Requires mutual exclusion + resource_group: integration:deployment + script: + - echo 'Perform service deployment for integration testing' + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ -application run: - stage: quality - image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner - dependencies: - - nix +integration:nix: + stage: integration + needs: + - integration:builds + - job: integration:deployment + optional: true script: - > build_application="$( \ @@ -119,14 +308,19 @@ application run: tail -1 \ )" - $build_application/bin/polykey - only: - - master + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ -docker run: - stage: quality +integration:docker: + stage: integration + needs: + - integration:builds + - job: integration:deployment + optional: true image: docker:20.10.11 - dependencies: - - nix services: - docker:20.10.11-dind variables: @@ -136,61 +330,233 @@ docker run: script: - image="$(docker load --input ./builds/*docker* | cut -d' ' -f3)" - docker run "$image" - only: - - master + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ -linux run: - stage: quality +integration:linux: + stage: integration + needs: + - integration:builds + - job: integration:deployment + optional: true image: ubuntu:latest - dependencies: - - nix script: - for f in ./builds/*-linux-*; do "$f"; done - only: - - master + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ -windows run: - stage: quality - dependencies: - - nix - script: - - Get-ChildItem -File ./builds/*-win32-* | ForEach {& $_.FullName} +integration:windows: + stage: integration + needs: + - integration:builds + - job: integration:deployment + optional: true tags: - windows - only: - - master - -macos run: - stage: quality - image: macos-11-xcode-12 - dependencies: - - nix - script: - - for f in ./builds/*-macos-*; do "$f"; done - only: - - master - tags: - - shared-macos-amd64 - -packages: + script: + - Get-ChildItem -File ./builds/*-win-* | ForEach {& $_.FullName} + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +integration:macos: + stage: integration + needs: + - integration:builds + - job: integration:deployment + optional: true + tags: + - shared-macos-amd64 + image: macos-11-xcode-12 + script: + - for f in ./builds/*-macos-x64*; do "$f"; done + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +integration:prerelease: + stage: integration + needs: + - integration:builds + - job: build:prerelease + optional: true + - job: integration:nix + optional: true + - job: integration:docker + optional: true + - job: integration:linux + optional: true + - job: integration:windows + optional: true + - job: integration:macos + optional: true + # Don't interrupt publishing job + interruptible: false + script: + - echo 'Publishing application prerelease' + - > + nix-shell -I nixpkgs=./pkgs.nix --packages gitAndTools.gh --run ' + if gh release view "$CI_COMMIT_TAG" --repo "$GH_PROJECT_PATH" >/dev/null; then \ + gh release \ + upload "$CI_COMMIT_TAG" \ + builds/*.closure.gz \ + builds/*-docker-* \ + builds/*-linux-* \ + builds/*-win-* \ + builds/*-macos-* \ + --clobber \ + --repo "$GH_PROJECT_PATH"; \ + else \ + gh release \ + create "$CI_COMMIT_TAG" \ + builds/*.closure.gz \ + builds/*-docker-* \ + builds/*-linux-* \ + builds/*-win-* \ + builds/*-macos-* \ + --title "${CI_COMMIT_TAG}-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --notes "" \ + --prerelease \ + --target staging \ + --repo "$GH_PROJECT_PATH"; \ + fi; + ' + rules: + # Only runs on tag pipeline where the tag is a prerelease version + # This requires dependencies to also run on tag pipeline + # However version tag comes with a version commit + # Dependencies must not run on the version commit + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+-.*[0-9]+$/ + +integration:merge: + stage: integration + needs: + - build:merge + - job: build:linux + optional: true + - job: build:windows + optional: true + - job: build:macos + optional: true + - job: integration:nix + optional: true + - job: integration:docker + optional: true + - job: integration:linux + optional: true + - job: integration:windows + optional: true + - job: integration:macos + optional: true + # Requires mutual exclusion + resource_group: integration:merge + allow_failure: true + variables: + # Ensure that CI/CD is fetching all commits + # this is necessary to checkout origin/master + # and to also merge origin/staging + GIT_DEPTH: 0 + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages gitAndTools.gh --run ' + printf "Pipeline Succeeded on ${CI_PIPELINE_ID} for ${CI_COMMIT_SHA}\n\n${CI_PIPELINE_URL}" \ + | gh pr comment staging \ + --body-file - \ + --repo "$GH_PROJECT_PATH"; + ' + - git remote add upstream "$GH_PROJECT_URL" + - git checkout origin/master + # Merge up to the current commit (not the latest commit) + - git merge --ff-only "$CI_COMMIT_SHA" + - git push upstream HEAD:master + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +release:deployment:branch: stage: release - image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner - dependencies: - - nix + # Only needs integration:builds from the staging branch pipeline + needs: + - project: $CI_PROJECT_PATH + job: integration:builds + ref: staging + artifacts: true + # Don't interrupt deploying job + interruptible: false + # Requires mutual exclusion (also with release:deployment:tag) + resource_group: release:deployment script: + - echo 'Perform service deployment for production' + rules: + # Runs on master commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +release:deployment:tag: + stage: release + # Tag pipelines run independently + needs: + - integration:builds + - integration:merge + # Don't interrupt deploying job + interruptible: false + # Requires mutual exclusion (also with release:deployment:branch) + resource_group: release:deployment + script: + - echo 'Perform service deployment for production' + rules: + # Runs on tag pipeline where the tag is a release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ + +release:distribution: + stage: release + needs: + - build:linux + - build:windows + - build:macos + - integration:builds + - integration:merge + - release:deployment:tag + # Don't interrupt publishing job + interruptible: false + before_script: + - echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc + script: + - echo 'Publishing library & application release' + - > + nix-shell --run ' + npm publish --access public; + ' - > - nix-shell -I nixpkgs=./pkgs.nix --packages git gitAndTools.gh --run ' - commit="$(git rev-parse --short HEAD)"; + nix-shell -I nixpkgs=./pkgs.nix --packages gitAndTools.gh --run ' gh release \ - create "$commit" \ + create "$CI_COMMIT_TAG" \ builds/*.closure.gz \ + builds/*-docker-* \ builds/*-linux-* \ - builds/*-win32-* \ + builds/*-win-* \ builds/*-macos-* \ - --title "Build-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ - --prerelease \ + --title "${CI_COMMIT_TAG}-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ --notes "" \ - --repo MatrixAI/js-polykey; + --target master \ + --repo "$GH_PROJECT_PATH"; ' - only: - - master + after_script: + - rm -f ./.npmrc + rules: + # Only runs on tag pipeline where the tag is a release version + # This requires dependencies to also run on tag pipeline + # However version tag comes with a version commit + # Dependencies must not run on the version commit + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ diff --git a/.npmignore b/.npmignore index 3e70e24c79..6bb02a31f7 100644 --- a/.npmignore +++ b/.npmignore @@ -1,17 +1,16 @@ .* +/*.nix /nix -/pkgs.nix -/default.nix -/shell.nix -/release.nix /tsconfig.json /tsconfig.build.json /babel.config.js /jest.config.js /src +/scripts /tests /tmp /docs /benches +/build /builds /dist/tsbuildinfo diff --git a/README.md b/README.md index 04e68a90f1..73da94d71e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Polykey -[![pipeline status](https://gitlab.com/MatrixAI/open-source/js-polykey/badges/master/pipeline.svg)](https://gitlab.com/MatrixAI/open-source/js-polykey/commits/master) +staging:[![pipeline status](https://gitlab.com/MatrixAI/open-source/js-polykey/badges/staging/pipeline.svg)](https://gitlab.com/MatrixAI/open-source/js-polykey/commits/staging) +master:[![pipeline status](https://gitlab.com/MatrixAI/open-source/js-polykey/badges/master/pipeline.svg)](https://gitlab.com/MatrixAI/open-source/js-polykey/commits/master) This is the core library for running PolyKey. It provides a CLI `polykey` or `pk` for interacting with the PolyKey system. diff --git a/default.nix b/default.nix index 183bc8b27a..283de7665f 100644 --- a/default.nix +++ b/default.nix @@ -12,32 +12,44 @@ let packageName = utils.node2nixDev.packageName; } '' - mkdir -p $out/lib/node_modules/${utils.node2nixDev.packageName} + mkdir -p "$out/lib/node_modules/$packageName" # copy the package.json - cp ${utils.node2nixDev}/lib/node_modules/${utils.node2nixDev.packageName}/package.json $out/lib/node_modules/${utils.node2nixDev.packageName}/ + cp \ + "${utils.node2nixDev}/lib/node_modules/$packageName/package.json" \ + "$out/lib/node_modules/$packageName/" + # copy the native addons + if [ -d "${utils.node2nixDev}/lib/node_modules/$packageName/prebuilds" ]; then + cp -r \ + "${utils.node2nixDev}/lib/node_modules/$packageName/prebuilds" \ + "$out/lib/node_modules/$packageName/" + fi # copy the dist - cp -r ${utils.node2nixDev}/lib/node_modules/${utils.node2nixDev.packageName}/dist $out/lib/node_modules/${utils.node2nixDev.packageName}/ + cp -r \ + "${utils.node2nixDev}/lib/node_modules/$packageName/dist" \ + "$out/lib/node_modules/$packageName/" # copy over the production dependencies if [ -d "${utils.node2nixProd}/lib/node_modules" ]; then - cp -r ${utils.node2nixProd}/lib/node_modules $out/lib/node_modules/${utils.node2nixDev.packageName}/ + cp -r \ + "${utils.node2nixProd}/lib/node_modules" \ + "$out/lib/node_modules/$packageName/" fi # symlink bin executables if [ \ - "$(${jq}/bin/jq 'has("bin")' "$out/lib/node_modules/${utils.node2nixDev.packageName}/package.json")" \ + "$(${jq}/bin/jq 'has("bin")' "$out/lib/node_modules/$packageName/package.json")" \ == \ "true" \ ]; then mkdir -p "$out/bin" while IFS= read -r bin_name && IFS= read -r bin_path; do # make files executable - chmod a+x "$out/lib/node_modules/${utils.node2nixDev.packageName}/$bin_path" + chmod a+x "$out/lib/node_modules/$packageName/$bin_path" # create the symlink ln -s \ - "../lib/node_modules/${utils.node2nixDev.packageName}/$bin_path" \ + "../lib/node_modules/$packageName/$bin_path" \ "$out/bin/$bin_name" done < <( ${jq}/bin/jq -r 'select(.bin != null) | .bin | to_entries[] | (.key, .value)' \ - "$out/lib/node_modules/${utils.node2nixDev.packageName}/package.json" + "$out/lib/node_modules/$packageName/package.json" ) fi ''; diff --git a/jest.config.js b/jest.config.js index f811716a83..b03be119a9 100644 --- a/jest.config.js +++ b/jest.config.js @@ -5,10 +5,9 @@ const process = require('process'); const { pathsToModuleNameMapper } = require('ts-jest'); const { compilerOptions } = require('./tsconfig'); -const moduleNameMapper = pathsToModuleNameMapper( - compilerOptions.paths, - { prefix: "/src/" } -); +const moduleNameMapper = pathsToModuleNameMapper(compilerOptions.paths, { + prefix: '/src/', +}); // using panva/jose with jest requires subpath exports // https://github.com/panva/jose/discussions/105 @@ -39,34 +38,34 @@ const globals = { process.env['GLOBAL_DATA_DIR'] = globals.dataDir; module.exports = { - testEnvironment: "node", - cacheDirectory: '/tmp/jest', + testEnvironment: 'node', verbose: true, - roots: [ - "/tests" - ], - testMatch: [ - "**/?(*.)+(spec|test|unit.test).+(ts|tsx|js)" - ], + collectCoverage: false, + cacheDirectory: '/tmp/jest', + coverageDirectory: '/tmp/coverage', + roots: ['/tests'], + testMatch: ['**/?(*.)+(spec|test|unit.test).+(ts|tsx|js|jsx)'], transform: { - "^.+\\.tsx?$": "ts-jest", - "^.+\\.jsx?$": "babel-jest" + '^.+\\.tsx?$': 'ts-jest', + '^.+\\.jsx?$': 'babel-jest', }, + reporters: [ + 'default', + ['jest-junit', { outputDirectory: '/tmp/junit' }], + ], + collectCoverageFrom: ['src/**/*.{ts,tsx,js,jsx}', '!src/**/*.d.ts'], + coverageReporters: ['text', 'cobertura'], globals, // Global setup script executed once before all test files - globalSetup: "/tests/globalSetup.ts", + globalSetup: '/tests/globalSetup.ts', // Global teardown script executed once after all test files - globalTeardown: "/tests/globalTeardown.ts", + globalTeardown: '/tests/globalTeardown.ts', // Setup files are executed before each test file // Can access globals - setupFiles: [ - "/tests/setup.ts" - ], + setupFiles: ['/tests/setup.ts'], // Setup files after env are executed before each test file // after the jest test environment is installed // Can access globals - setupFilesAfterEnv: [ - "/tests/setupAfterEnv.ts" - ], - moduleNameMapper: moduleNameMapper + setupFilesAfterEnv: ['/tests/setupAfterEnv.ts'], + moduleNameMapper: moduleNameMapper, }; diff --git a/nix/leveldown.patch b/nix/leveldown.patch deleted file mode 100644 index 0d35bab418..0000000000 --- a/nix/leveldown.patch +++ /dev/null @@ -1,14 +0,0 @@ ---- dictionary/leveldown.js 2021-08-04 15:43:31.836337623 +1000 -+++ dictionary/leveldown_.js 2021-08-04 15:43:11.977266316 +1000 -@@ -1,10 +1,3 @@ - 'use strict'; - --module.exports = { -- pkg: { -- patches: { -- 'binding.js': ['__dirname', "require('path').dirname(process.execPath)"], -- }, -- deployFiles: [['prebuilds', 'prebuilds', 'directory']], -- }, --}; -+module.exports = {}; diff --git a/package-lock.json b/package-lock.json index 97cd45f6ab..e7078de287 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,6 +31,7 @@ "google-protobuf": "^3.14.0", "ip-num": "^1.3.3-0", "isomorphic-git": "^1.8.1", + "jest-junit": "^13.2.0", "jose": "^4.3.6", "lexicographic-integer": "^1.1.0", "multiformats": "^9.4.8", @@ -75,6 +76,7 @@ "node-gyp-build": "4.4.0", "pkg": "5.6.0", "prettier": "^2.6.2", + "shx": "^0.3.4", "ts-jest": "^27.0.5", "ts-node": "^10.4.0", "tsconfig-paths": "^3.9.0", @@ -3373,7 +3375,6 @@ }, "node_modules/balanced-match": { "version": "1.0.2", - "dev": true, "license": "MIT" }, "node_modules/base64-js": { @@ -3454,7 +3455,6 @@ }, "node_modules/brace-expansion": { "version": "1.1.11", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -3775,7 +3775,6 @@ }, "node_modules/concat-map": { "version": "0.0.1", - "dev": true, "license": "MIT" }, "node_modules/console-control-strings": { @@ -5176,7 +5175,6 @@ }, "node_modules/fs.realpath": { "version": "1.0.0", - "dev": true, "license": "ISC" }, "node_modules/function-bind": { @@ -5312,7 +5310,6 @@ }, "node_modules/glob": { "version": "7.2.0", - "dev": true, "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", @@ -5649,7 +5646,6 @@ }, "node_modules/inflight": { "version": "1.0.6", - "dev": true, "license": "ISC", "dependencies": { "once": "^1.3.0", @@ -5677,6 +5673,15 @@ "node": ">= 0.4" } }, + "node_modules/interpret": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", + "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", + "dev": true, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/into-stream": { "version": "6.0.0", "dev": true, @@ -6710,6 +6715,20 @@ "node": ">=8" } }, + "node_modules/jest-junit": { + "version": "13.2.0", + "resolved": "https://registry.npmjs.org/jest-junit/-/jest-junit-13.2.0.tgz", + "integrity": "sha512-B0XNlotl1rdsvFZkFfoa19mc634+rrd8E4Sskb92Bb8MmSXeWV9XJGUyctunZS1W410uAxcyYuPUGVnbcOH8cg==", + "dependencies": { + "mkdirp": "^1.0.4", + "strip-ansi": "^6.0.1", + "uuid": "^8.3.2", + "xml": "^1.0.1" + }, + "engines": { + "node": ">=10.12.0" + } + }, "node_modules/jest-leak-detector": { "version": "27.5.1", "dev": true, @@ -8085,7 +8104,6 @@ }, "node_modules/minimatch": { "version": "3.1.2", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -8105,6 +8123,17 @@ "minimist": "^1.2.5" } }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/mkdirp-classic": { "version": "0.5.3", "dev": true, @@ -8600,7 +8629,6 @@ }, "node_modules/path-is-absolute": { "version": "1.0.1", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -9160,6 +9188,18 @@ "node": ">= 6" } }, + "node_modules/rechoir": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", + "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==", + "dev": true, + "dependencies": { + "resolve": "^1.1.6" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/regenerate": { "version": "1.4.2", "dev": true, @@ -9333,7 +9373,6 @@ }, "node_modules/rimraf": { "version": "3.0.2", - "dev": true, "license": "ISC", "dependencies": { "glob": "^7.1.3" @@ -9471,6 +9510,23 @@ "node": ">=8" } }, + "node_modules/shelljs": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", + "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", + "dev": true, + "dependencies": { + "glob": "^7.0.0", + "interpret": "^1.0.0", + "rechoir": "^0.6.2" + }, + "bin": { + "shjs": "bin/shjs" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/shiki": { "version": "0.10.1", "dev": true, @@ -9481,6 +9537,22 @@ "vscode-textmate": "5.2.0" } }, + "node_modules/shx": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/shx/-/shx-0.3.4.tgz", + "integrity": "sha512-N6A9MLVqjxZYcVn8hLmtneQWIJtp8IKzMP4eMnx+nqkvXoqinUPCbUFLp2UcWTEIUONhlk0ewxr/jaVGlc+J+g==", + "dev": true, + "dependencies": { + "minimist": "^1.2.3", + "shelljs": "^0.8.5" + }, + "bin": { + "shx": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/side-channel": { "version": "1.0.4", "license": "MIT", @@ -10648,6 +10720,11 @@ } } }, + "node_modules/xml": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", + "integrity": "sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=" + }, "node_modules/xml-name-validator": { "version": "3.0.0", "dev": true, @@ -12844,8 +12921,7 @@ } }, "balanced-match": { - "version": "1.0.2", - "dev": true + "version": "1.0.2" }, "base64-js": { "version": "1.5.1" @@ -12891,7 +12967,6 @@ }, "brace-expansion": { "version": "1.1.11", - "dev": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -13084,8 +13159,7 @@ "version": "8.3.0" }, "concat-map": { - "version": "0.0.1", - "dev": true + "version": "0.0.1" }, "console-control-strings": { "version": "1.1.0", @@ -14015,8 +14089,7 @@ } }, "fs.realpath": { - "version": "1.0.0", - "dev": true + "version": "1.0.0" }, "function-bind": { "version": "1.1.1" @@ -14100,7 +14173,6 @@ }, "glob": { "version": "7.2.0", - "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -14294,7 +14366,6 @@ }, "inflight": { "version": "1.0.6", - "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -14315,6 +14386,12 @@ "side-channel": "^1.0.4" } }, + "interpret": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", + "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", + "dev": true + }, "into-stream": { "version": "6.0.0", "dev": true, @@ -14944,6 +15021,17 @@ } } }, + "jest-junit": { + "version": "13.2.0", + "resolved": "https://registry.npmjs.org/jest-junit/-/jest-junit-13.2.0.tgz", + "integrity": "sha512-B0XNlotl1rdsvFZkFfoa19mc634+rrd8E4Sskb92Bb8MmSXeWV9XJGUyctunZS1W410uAxcyYuPUGVnbcOH8cg==", + "requires": { + "mkdirp": "^1.0.4", + "strip-ansi": "^6.0.1", + "uuid": "^8.3.2", + "xml": "^1.0.1" + } + }, "jest-leak-detector": { "version": "27.5.1", "dev": true, @@ -15827,7 +15915,6 @@ }, "minimatch": { "version": "3.1.2", - "dev": true, "requires": { "brace-expansion": "^1.1.7" } @@ -15841,6 +15928,11 @@ "minimist": "^1.2.5" } }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==" + }, "mkdirp-classic": { "version": "0.5.3", "dev": true @@ -16146,8 +16238,7 @@ "dev": true }, "path-is-absolute": { - "version": "1.0.1", - "dev": true + "version": "1.0.1" }, "path-key": { "version": "3.1.1" @@ -16491,6 +16582,15 @@ "util-deprecate": "^1.0.1" } }, + "rechoir": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", + "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==", + "dev": true, + "requires": { + "resolve": "^1.1.6" + } + }, "regenerate": { "version": "1.4.2", "dev": true @@ -16597,7 +16697,6 @@ }, "rimraf": { "version": "3.0.2", - "dev": true, "requires": { "glob": "^7.1.3" } @@ -16660,6 +16759,17 @@ "shebang-regex": { "version": "3.0.0" }, + "shelljs": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", + "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", + "dev": true, + "requires": { + "glob": "^7.0.0", + "interpret": "^1.0.0", + "rechoir": "^0.6.2" + } + }, "shiki": { "version": "0.10.1", "dev": true, @@ -16669,6 +16779,16 @@ "vscode-textmate": "5.2.0" } }, + "shx": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/shx/-/shx-0.3.4.tgz", + "integrity": "sha512-N6A9MLVqjxZYcVn8hLmtneQWIJtp8IKzMP4eMnx+nqkvXoqinUPCbUFLp2UcWTEIUONhlk0ewxr/jaVGlc+J+g==", + "dev": true, + "requires": { + "minimist": "^1.2.3", + "shelljs": "^0.8.5" + } + }, "side-channel": { "version": "1.0.4", "requires": { @@ -17416,6 +17536,11 @@ "dev": true, "requires": {} }, + "xml": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", + "integrity": "sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=" + }, "xml-name-validator": { "version": "3.0.0", "dev": true diff --git a/package.json b/package.json index 96e74b3f64..83a5806a29 100644 --- a/package.json +++ b/package.json @@ -51,24 +51,26 @@ "pkg": { "assets": [ "node_modules/jose/**/*", - "node_modules/utp-native/**/*", - "node_modules/leveldown/**/*", - "node_modules/fd-lock/**/*", "dist/**/*.json" ], - "scripts": "dist/workers/polykeyWorker.js" + "scripts": [ + "dist/workers/polykeyWorker.js", + "dist/bin/polykey-agent.js" + ] }, "scripts": { "prepare": "tsc -p ./tsconfig.build.json", - "build": "rm -r ./dist || true; tsc -p ./tsconfig.build.json", - "postbuild": "cp -fR src/proto dist; cp src/notifications/*.json dist/notifications/; cp src/claims/*.json dist/claims/; cp src/status/*.json dist/status/;", + "build": "shx rm -rf ./dist && tsc -p ./tsconfig.build.json", + "postbuild": "shx cp -fR src/proto dist && shx cp src/notifications/*.json dist/notifications/ && shx cp src/claims/*.json dist/claims/ && shx cp src/status/*.json dist/status/", + "postversion": "npm install --package-lock-only --ignore-scripts --silent", "ts-node": "ts-node --require tsconfig-paths/register", "test": "jest", "lint": "eslint '{src,tests}/**/*.{js,ts}'", "lintfix": "eslint '{src,tests}/**/*.{js,ts}' --fix", - "docs": "rm -r ./docs || true; typedoc --gitRevision master --tsconfig ./tsconfig.build.json --out ./docs src", - "bench": "rm -r ./benches/results || true; ts-node --require tsconfig-paths/register --compiler typescript-cached-transpile --transpile-only ./benches", + "docs": "shx rm -rf ./docs && typedoc --gitRevision master --tsconfig ./tsconfig.build.json --out ./docs src", + "bench": "shx rm -rf ./benches/results && ts-node --require tsconfig-paths/register --compiler typescript-cached-transpile --transpile-only ./benches", "proto-generate": "scripts/proto-generate.sh", + "pkg": "./scripts/pkg.js --no-dict=leveldown.js", "polykey": "ts-node --require tsconfig-paths/register --compiler typescript-cached-transpile --transpile-only src/bin/polykey.ts" }, "dependencies": { @@ -94,6 +96,7 @@ "google-protobuf": "^3.14.0", "ip-num": "^1.3.3-0", "isomorphic-git": "^1.8.1", + "jest-junit": "^13.2.0", "jose": "^4.3.6", "lexicographic-integer": "^1.1.0", "multiformats": "^9.4.8", @@ -134,6 +137,7 @@ "node-gyp-build": "4.4.0", "pkg": "5.6.0", "prettier": "^2.6.2", + "shx": "^0.3.4", "ts-jest": "^27.0.5", "ts-node": "^10.4.0", "tsconfig-paths": "^3.9.0", diff --git a/release.nix b/release.nix index e778a0073f..2544eb4037 100644 --- a/release.nix +++ b/release.nix @@ -8,21 +8,16 @@ let name = "${utils.basename}-${version}-linux-${arch}"; version = utils.node2nixDev.version; src = "${utils.node2nixDev}/lib/node_modules/${utils.node2nixDev.packageName}"; - buildInputs = [ - utils.pkg - ]; + nativeBuildInputs = [ nodejs ]; PKG_CACHE_PATH = utils.pkgCachePath; PKG_IGNORE_TAG = 1; - # ensure that native modules are built from source - npm_config_build_from_source = "true"; buildPhase = '' - cp ${./package.json} package.json - pkg . \ - --targets node${utils.nodeVersion}-linux-${arch} \ - --no-bytecode \ - --public \ - --public-packages "*" \ - --output out + npm run pkg -- \ + --output=out \ + --bin=polykey \ + --node-version=${utils.nodeVersion} \ + --platform=linux \ + --arch=${arch} ''; installPhase = '' cp out $out @@ -31,24 +26,19 @@ let }; buildExe = arch: stdenv.mkDerivation rec { - name = "${utils.basename}-${version}-win32-${arch}.exe"; + name = "${utils.basename}-${version}-win-${arch}.exe"; version = utils.node2nixDev.version; src = "${utils.node2nixDev}/lib/node_modules/${utils.node2nixDev.packageName}"; - buildInputs = [ - utils.pkg - ]; + nativeBuildInputs = [ nodejs ]; PKG_CACHE_PATH = utils.pkgCachePath; PKG_IGNORE_TAG = 1; - # ensure that native modules are built from source - npm_config_build_from_source = "true"; buildPhase = '' - cp ${./package.json} package.json - pkg . \ - --targets node${utils.nodeVersion}-win-${arch} \ - --no-bytecode \ - --public \ - --public-packages "*" \ - --output out.exe + npm run pkg -- \ + --output=out.exe \ + --bin=polykey \ + --node-version=${utils.nodeVersion} \ + --platform=win32 \ + --arch=${arch} ''; installPhase = '' cp out.exe $out @@ -60,44 +50,34 @@ let name = "${utils.basename}-${version}-macos-${arch}"; version = utils.node2nixDev.version; src = "${utils.node2nixDev}/lib/node_modules/${utils.node2nixDev.packageName}"; - buildInputs = [ - utils.pkg - ]; + nativeBuildInputs = [ nodejs ]; PKG_CACHE_PATH = utils.pkgCachePath; PKG_IGNORE_TAG = 1; - # ensure that native modules are built from source - npm_config_build_from_source = "true"; buildPhase = '' - cp ${./package.json} package.json - pkg . \ - --targets node${utils.nodeVersion}-macos-${arch} \ - --no-bytecode \ - --public \ - --public-packages "*" \ - --output out + npm run pkg -- \ + --output=out \ + --bin=polykey \ + --node-version=${utils.nodeVersion} \ + --platform=darwin \ + --arch=${arch} ''; installPhase = '' cp out $out ''; dontFixup = true; }; - # allow resolution of localhost - nsswitch = writeTextDir "etc/nsswitch.conf" - '' - hosts: files dns myhostname - ''; in rec { application = callPackage ./default.nix {}; docker = dockerTools.buildImage { name = application.name; - contents = [ application cacert nsswitch ]; + contents = [ application ]; keepContentsDirlinks = true; extraCommands = '' mkdir -m 1777 tmp ''; config = { - Entrypoint = [ "/bin/polykey" ]; + Cmd = [ "/bin/polykey" ]; }; }; package = { @@ -115,6 +95,9 @@ in x64 = { macho = buildMacho "x64"; }; + arm64 = { + macho = buildMacho "arm64"; + }; }; }; } diff --git a/scripts/pkg.js b/scripts/pkg.js new file mode 100755 index 0000000000..d5e55c10be --- /dev/null +++ b/scripts/pkg.js @@ -0,0 +1,174 @@ +#!/usr/bin/env node + +const os = require('os'); +const fs = require('fs'); +const path = require('path'); +const process = require('process'); +const crypto = require('crypto'); +const child_process = require('child_process'); +const packageJSON = require('../package.json'); + +/** + * Supported platforms + * Maps os.platform() to pkg platform + */ +const platforms = { + 'linux': 'linux', + 'win32': 'win', + 'darwin': 'macos', +}; + +/** + * Supported architectures + * Maps os.arch() to pkg arch + */ +const archs = { + 'x64': 'x64', + 'arm64': 'arm64', +}; + +function randomString(l) { + return crypto + .randomBytes(l) + .toString('base64') + .replace(/\//, '_'); +} + +async function find(dirPath, pattern) { + const found = []; + let entries; + try { + entries = await fs.promises.readdir(dirPath); + } catch (e) { + if (e.code === 'ENOENT') { + return found ; + } + throw e; + } + for (const entry of entries) { + const entryPath = path.join(dirPath, entry); + const stat = await fs.promises.lstat(entryPath); + if (stat.isDirectory()) { + found.push(...(await find(entryPath, pattern))); + } else if (pattern.test(entryPath)) { + found.push(entryPath); + } + } + return found; +}; + +async function main(argv = process.argv) { + argv = argv.slice(2); + let outPath; + let binTarget; + let nodeVersion = process.versions.node.match(/\d+/)[0]; + let platform = os.platform(); + let arch = os.arch(); + const restArgs = []; + while (argv.length > 0) { + const option = argv.shift(); + let match; + if (match = option.match(/--output(?:=(.+)|$)/)) { + outPath = match[1] ?? argv.shift(); + } else if (match = option.match(/--bin(?:=(.+)|$)/)) { + binTarget = match[1] ?? argv.shift(); + } else if (match = option.match(/--node-version(?:=(.+)|$)/)) { + nodeVersion = match[1] ?? argv.shift(); + } else if (match = option.match(/--platform(?:=(.+)|$)/)) { + platform = match[1] ?? argv.shift(); + } else if (match = option.match(/--arch(?:=(.+)|$)/)) { + arch = match[1] ?? argv.shift(); + } else { + restArgs.push(option); + } + } + let entryPoint; + if (binTarget == null) { + entryPoint = Object.values(packageJSON.bin ?? {})[0]; + } else { + entryPoint = packageJSON.bin[binTarget]; + } + if (entryPoint == null) { + throw new Error('Bin executable is required'); + } + if (typeof outPath !== 'string') { + throw new Error('Output path is required'); + } + if (entryPoint == null) { + throw new Error(`Unknown bin target: ${binTarget}`); + } + if (isNaN(parseInt(nodeVersion))) { + throw new Error(`Unsupported node version: ${nodeVersion}`); + } + if (!(platform in platforms)) { + throw new Error(`Unsupported platform: ${platform}`); + } + if (!(arch in archs)) { + throw new Error(`Unsupported architecture: ${arch}`); + } + // Monkey patch the os.platform and os.arch for node-gyp-build + os.platform = () => platform; + os.arch = () => arch; + const nodeGypBuild = require('node-gyp-build'); + const pkgConfig = packageJSON.pkg ?? {}; + pkgConfig.assets = pkgConfig.assets ?? {}; + const npmLsOut = child_process.execFileSync( + 'npm', + ['ls', '--all', '--prod', '--parseable'], + { + windowsHide: true, + encoding: 'utf-8' + } + ); + const nodePackages = npmLsOut.trim().split('\n'); + const projectRoot = path.join(__dirname, '..'); + for (const nodePackage of nodePackages) { + // If `build` or `prebuilds` directory exists with a `.node` file + // then we expect to find the appropriate native addon + // The `node-gyp-build` will look in these 2 directories + const buildPath = path.join(nodePackage, 'build'); + const prebuildsPath = path.join(nodePackage, 'prebuilds'); + const buildFinds = await find(buildPath, /.node$/); + const prebuildsFinds = await find(prebuildsPath, /.node$/); + if (buildFinds.length > 0 || prebuildsFinds.length > 0) { + let nativeAddonPath = nodeGypBuild.path(nodePackage); + // Must use relative paths + // so that assets are added relative to the project + nativeAddonPath = path.relative(projectRoot, nativeAddonPath); + pkgConfig.assets.push(nativeAddonPath); + } + } + console.error('Configured pkg with:'); + console.error(pkgConfig); + // The pkg config must be in the same directory as the `package.json` + // otherwise the relative paths won't work + const pkgConfigPath = path.join(projectRoot, 'pkg.json'); + await fs.promises.writeFile(pkgConfigPath, JSON.stringify(pkgConfig)); + const pkgPlatform = platforms[platform]; + const pkgArch = archs[arch]; + const pkgArgs = [ + entryPoint, + `--config=${pkgConfigPath}`, + `--targets=node${nodeVersion}-${pkgPlatform}-${pkgArch}`, + '--no-bytecode', + '--no-native-build', + '--public', + '--public-packages=\'*\'', + `--output=${outPath}`, + ...restArgs + ]; + console.error('Running pkg:') + console.error(['pkg', ...pkgArgs].join(' ')); + child_process.execFileSync( + 'pkg', + pkgArgs, + { + stdio: ['inherit', 'inherit', 'inherit'], + windowsHide: true, + encoding: 'utf-8' + } + ); + await fs.promises.rm(pkgConfigPath); +} + +void main(); diff --git a/scripts/test-pipelines.sh b/scripts/test-pipelines.sh index 323850fdd3..4864af4fb0 100755 --- a/scripts/test-pipelines.sh +++ b/scripts/test-pipelines.sh @@ -5,7 +5,19 @@ shopt -s nullglob # Quote the heredoc to prevent shell expansion cat << "EOF" +workflow: + rules: + # Disable merge request pipelines + - if: $CI_MERGE_REQUEST_ID + when: never + - when: always + +default: + interruptible: true + variables: + GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}" + GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git" GIT_SUBMODULE_STRATEGY: "recursive" # Cache .npm NPM_CONFIG_CACHE: "./tmp/npm" @@ -16,7 +28,7 @@ variables: TS_CACHED_TRANSPILE_CACHE: "${CI_PROJECT_DIR}/tmp/ts-node-cache" TS_CACHED_TRANSPILE_PORTABLE: "true" -# Cached directories shared between jobs & pipelines per-branch +# Cached directories shared between jobs & pipelines per-branch per-runner cache: key: $CI_COMMIT_REF_SLUG paths: @@ -24,24 +36,12 @@ cache: - ./tmp/ts-node-cache/ # `jest` cache is configured in jest.config.js - ./tmp/jest/ + +image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner EOF printf "\n" -# # SPECIAL CASE -# cat << EOF -# test binagent: -# image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner -# stage: test -# interruptible: true -# script: -# - > -# nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' -# npm ci; -# npm test -- ./tests/bin/agent; -# ' -# EOF - # Each test directory has its own job for test_dir in tests/**/*/; do test_files=("$test_dir"*.test.ts) @@ -53,16 +53,20 @@ for test_dir in tests/**/*/; do # Remove `tests/` prefix test_dir="${test_dir#*/}" cat << EOF -test $test_dir: - image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner +check:test $test_dir: stage: test - interruptible: true + needs: [] script: - > - nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' - npm ci; - npm test -- ${test_files[@]}; + nix-shell --run ' + npm run build --verbose; + npm test -- --ci ${test_files[@]}; ' + artifacts: + when: always + reports: + junit: + - ./tmp/junit/junit.xml EOF printf "\n" done @@ -70,14 +74,18 @@ done # All top-level test files are accumulated into 1 job test_files=(tests/*.test.ts) cat << EOF -test index: - image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner +check:test index: stage: test - interruptible: true + needs: [] script: - > - nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' - npm ci; - npm test -- ${test_files[@]}; + nix-shell --run ' + npm run build --verbose; + npm test -- --ci ${test_files[@]}; ' + artifacts: + when: always + reports: + junit: + - ./tmp/junit/junit.xml EOF diff --git a/shell.nix b/shell.nix index 3b4bd595f6..961a13c8ad 100644 --- a/shell.nix +++ b/shell.nix @@ -4,13 +4,12 @@ with pkgs; let utils = callPackage ./utils.nix {}; in - pkgs.mkShell { + mkShell { nativeBuildInputs = [ nodejs utils.node2nix grpc-tools grpcurl - utils.pkg ]; PKG_CACHE_PATH = utils.pkgCachePath; PKG_IGNORE_TAG = 1; @@ -26,18 +25,10 @@ in # Built executables and NPM executables export PATH="$(pwd)/dist/bin:$(npm bin):$PATH" - # pkg is installed in package.json - # this ensures that in nix-shell we are using the nix packaged versions - export PATH="${lib.makeBinPath - [ - utils.pkg - ] - }:$PATH" - # Enables npm link to work export npm_config_prefix=~/.npm - npm install + npm install --ignore-scripts set +v ''; diff --git a/utils.nix b/utils.nix index e63e463dfe..31f0b10559 100644 --- a/utils.nix +++ b/utils.nix @@ -2,7 +2,6 @@ , linkFarm , nix-gitignore , nodejs -, nodePackages , pkgs , lib , fetchurl @@ -12,7 +11,7 @@ rec { # this removes the org scoping basename = builtins.baseNameOf node2nixDev.packageName; - src = nix-gitignore.gitignoreSource [".git"] ./.; + src = nix-gitignore.gitignoreSource [".git" "/*.nix"] ./.; nodeVersion = builtins.elemAt (lib.versions.splitVersion nodejs.version) 0; # custom node2nix directly from GitHub node2nixSrc = fetchFromGitHub { @@ -33,21 +32,30 @@ rec { --composition $out/default.nix \ --nodejs-${nodeVersion} ''; - # the shell attribute has the nodeDependencies, whereas the package does not - node2nixProd = ( - (import (node2nixDrv false) { inherit pkgs nodejs; }).shell.override (attrs: { - buildInputs = attrs.buildInputs ++ [ nodePackages.node-gyp-build ]; - dontNpmInstall = true; - }) - ).nodeDependencies; - node2nixDev = (import (node2nixDrv true) { inherit pkgs nodejs; }).package.override (attrs: { + node2nixProd = (import (node2nixDrv false) { inherit pkgs nodejs; }).nodeDependencies.override (attrs: { + # Use filtered source src = src; - buildInputs = attrs.buildInputs ++ [ nodePackages.node-gyp-build ]; + # Do not run build scripts during npm rebuild and npm install + npmFlags = "--ignore-scripts"; + # Do not run npm install, dependencies are installed by nix dontNpmInstall = true; + }); + node2nixDev = (import (node2nixDrv true) { inherit pkgs nodejs; }).package.override (attrs: { + # Use filtered source + src = src; + # Do not run build scripts during npm rebuild and npm install + # They will be executed in the postInstall hook + npmFlags = "--ignore-scripts"; + # Show full compilation flags + NIX_DEBUG = 1; + # Don't set rpath for native addons + # Native addons do not require their own runtime search path + # because they dynamically loaded by the nodejs runtime + NIX_DONT_SET_RPATH = true; + NIX_NO_SELF_RPATH = true; postInstall = '' - # The dependencies were prepared in the installphase - # See `node2nix` generated `node-env.nix` for details - npm run build + # This will setup the typescript build + npm --nodedir=${nodejs} run build ''; }); pkgBuilds = { @@ -64,6 +72,10 @@ rec { url = "https://github.com/vercel/pkg-fetch/releases/download/v3.3/node-v16.14.2-macos-x64"; sha256 = "1hq7v40vzc2bfr29y71lm0snaxcc8rys5w0da7pi5nmx4pyybc2v"; }; + "macos-arm64" = fetchurl { + url = "https://github.com/vercel/pkg-fetch/releases/download/v3.3/node-v16.14.2-macos-arm64"; + sha256 = "05q350aw7fhirmlqg6ckyi5hg9pwcvs0w5r047r8mf3ivy1hxra4"; + }; }; }; pkgCachePath = @@ -85,10 +97,9 @@ rec { name = fetchedName pkgBuild.macos-x64.name; path = pkgBuild.macos-x64; } + { + name = fetchedName pkgBuild.macos-arm64.name; + path = pkgBuild.macos-arm64; + } ]; - pkg = nodePackages.pkg.override { - postFixup = '' - patch -p0 < ${./nix/leveldown.patch} - ''; - }; }