security: add detect-secrets (#2983)

* add detect-secrets pre-commit hook

* update baseline

.circleci/config.yml

@@ -6,6 +6,15 @@ orbs:
   yarn: artsy/[email protected]
   horizon: artsy/[email protected]
 
+jobs:
+  detect-secrets:
+    docker:
+      - image: lirantal/detect-secrets
+    working_directory: /usr/src/app
+    steps:
+      - checkout
+      - run: detect-secrets-hook --baseline .secrets.baseline $(git ls-files)
+
 not_staging_or_release: &not_staging_or_release
   filters:
     branches:
@@ -25,9 +34,20 @@ only_release: &only_release
     branches:
       only: release
 
+only_dev: &only_dev
+  filters:
+    branches:
+      ignore:
+        - master
+        - staging
+        - release
+
 workflows:
   build-deploy:
     jobs:
+      - detect-secrets:
+          <<: *only_dev
+
       - yarn/type-check:
           <<: *not_staging_or_release
 

.secrets.baseline

@@ -0,0 +1,150 @@
+{
+  "version": "1.1.0",
+  "plugins_used": [
+    {
+      "name": "ArtifactoryDetector"
+    },
+    {
+      "name": "AWSKeyDetector"
+    },
+    {
+      "name": "AzureStorageKeyDetector"
+    },
+    {
+      "name": "Base64HighEntropyString",
+      "limit": 4.5
+    },
+    {
+      "name": "BasicAuthDetector"
+    },
+    {
+      "name": "CloudantDetector"
+    },
+    {
+      "name": "HexHighEntropyString",
+      "limit": 3.0
+    },
+    {
+      "name": "IbmCloudIamDetector"
+    },
+    {
+      "name": "IbmCosHmacDetector"
+    },
+    {
+      "name": "JwtTokenDetector"
+    },
+    {
+      "name": "KeywordDetector",
+      "keyword_exclude": ""
+    },
+    {
+      "name": "MailchimpDetector"
+    },
+    {
+      "name": "NpmDetector"
+    },
+    {
+      "name": "PrivateKeyDetector"
+    },
+    {
+      "name": "SlackDetector"
+    },
+    {
+      "name": "SoftlayerDetector"
+    },
+    {
+      "name": "SquareOAuthDetector"
+    },
+    {
+      "name": "StripeDetector"
+    },
+    {
+      "name": "TwilioKeyDetector"
+    }
+  ],
+  "filters_used": [
+    {
+      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_baseline_file",
+      "filename": ".secrets.baseline"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
+      "min_level": 2
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_lock_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_sequential_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_swagger_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_templated_secret"
+    },
+    {
+      "path": "detect_secrets.filters.regex.should_exclude_secret",
+      "pattern": [
+        "[a-fA-F0-9]{24}"
+      ]
+    }
+  ],
+  "results": {
+    ".env.oss": [
+      {
+        "type": "Secret Keyword",
+        "filename": ".env.oss",
+        "hashed_secret": "65910e06e1afc820523c9af5d194e7a85218c6d1",
+        "is_verified": false,
+        "line_number": 16
+      }
+    ],
+    "src/api/apps/articles/test/integration.spec.ts": [
+      {
+        "type": "JSON Web Token",
+        "filename": "src/api/apps/articles/test/integration.spec.ts",
+        "hashed_secret": "aa180e831a95a19a94ed6d6ce03a2297734925b5",
+        "is_verified": false,
+        "line_number": 65
+      }
+    ],
+    "src/api/test/lib/joi.test.coffee": [
+      {
+        "type": "Hex High Entropy String",
+        "filename": "src/api/test/lib/joi.test.coffee",
+        "hashed_secret": "be2679f8e288fadc9396194951cddcd8068651e1",
+        "is_verified": false,
+        "line_number": 7
+      }
+    ],
+    "src/test/helpers/fixtures.coffee": [
+      {
+        "type": "JSON Web Token",
+        "filename": "src/test/helpers/fixtures.coffee",
+        "hashed_secret": "d8d87635ba315978be363a2c6356e454f9caa201",
+        "is_verified": false,
+        "line_number": 140
+      }
+    ]
+  },
+  "generated_at": "2021-10-01T12:18:36Z"
+}

package.json

@@ -13,7 +13,7 @@
     "jest": "sh scripts/jest.sh",
     "lint": "eslint src --cache --cache-location '.cache/eslint/' --ext ts,tsx,js,jsx ",
     "mocha": "sh scripts/mocha.sh",
-    "precommit": "lint-staged",
+    "precommit": "lint-staged && scripts/detect-secrets.sh",
     "prepush": "yarn run type-check",
     "prettier-project": "yarn run prettier-write 'src/**/*.{ts,tsx,js,jsx}'",
     "prettier-write": "yarn run prettier --write",

scripts/detect-secrets.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+RED=$(tput setaf 1)
+GREEN=$(tput setaf 2)
+NO_COLOR=$(tput sgr0)
+
+HELP="${RED}command not found: detect-secrets${NO_COLOR}
+
+To install the command line tool re-run: ${GREEN}https://github.com/artsy/potential/blob/main/scripts/setup${NO_COLOR}
+To learn more about this tool: ${GREEN}https://www.notion.so/artsy/Detect-Secrets-cd11d994dabf45f6a3c18e07acb5431c${NO_COLOR}
+
+You can bypass this hook using --no-verify option. ${RED}USE AT YOUR OWN RISK!${NO_COLOR}"
+
+echo 'Executing detect-secrets...'
+if which detect-secrets > /dev/null; test $? != 0; then
+  echo "${HELP}"
+  exit 1
+else
+  detect-secrets-hook --baseline .secrets.baseline $(git diff --staged --name-only)
+fi
+echo "${GREEN}No secrets detected!${NO_COLOR}"