feat: op-conductor-mon (#1)

* feat: op-conductor-mon

* Update CODEOWNERS

.circleci/config.yml

@@ -0,0 +1,27 @@
+version: 2.1
+
+# this allows you to use CircleCI's dynamic configuration feature
+setup: true
+
+orbs:
+  path-filtering: circleci/[email protected]
+
+workflows:
+  check-updated-files:
+    jobs:
+      - path-filtering/filter:
+          name: check-updated-files
+          # 3-column, whitespace-delimited mapping. One mapping per
+          # line:
+          # <regex path-to-test> <parameter-to-set> <value-of-pipeline-parameter>
+          mapping: |
+            op-conductor-mon/.* run-build-op-conductor-mon true
+            .circleci/.* run-all true
+            .github/.* run-all true
+
+          base-revision: main
+          # this is the path of the configuration we should trigger once
+          # path filtering and pipeline parameter value updates are
+          # complete. In this case, we are using the parent dynamic
+          # configuration itself.
+          config-path: .circleci/continue_config.yml

.circleci/continue_config.yml

@@ -0,0 +1,359 @@
+version: 2.1
+
+orbs:
+  go: circleci/[email protected]
+  gcp-cli: circleci/[email protected]
+  shellcheck: circleci/[email protected]
+  path-filtering: circleci/[email protected]
+
+parameters:
+  run-build-op-conductor-mon:
+    type: boolean
+    default: false
+  run-all:
+    type: boolean
+    default: false
+
+commands:
+  gcp-oidc-authenticate:
+    description: "Authenticate with GCP using a CircleCI OIDC token."
+    parameters:
+      project_id:
+        type: env_var_name
+        default: GCP_PROJECT_ID
+      workload_identity_pool_id:
+        type: env_var_name
+        default: GCP_WIP_ID
+      workload_identity_pool_provider_id:
+        type: env_var_name
+        default: GCP_WIP_PROVIDER_ID
+      service_account_email:
+        type: env_var_name
+        default: GCP_SERVICE_ACCOUNT_EMAIL
+      gcp_cred_config_file_path:
+        type: string
+        default: /home/circleci/gcp_cred_config.json
+      oidc_token_file_path:
+        type: string
+        default: /home/circleci/oidc_token.json
+    steps:
+      - run:
+          name: "Create OIDC credential configuration"
+          command: |
+            # Store OIDC token in temp file
+            echo $CIRCLE_OIDC_TOKEN > << parameters.oidc_token_file_path >>
+            # Create a credential configuration for the generated OIDC ID Token
+            gcloud iam workload-identity-pools create-cred-config \
+                "projects/${<< parameters.project_id >>}/locations/global/workloadIdentityPools/${<< parameters.workload_identity_pool_id >>}/providers/${<< parameters.workload_identity_pool_provider_id >>}"\
+                --output-file="<< parameters.gcp_cred_config_file_path >>" \
+                --service-account="${<< parameters.service_account_email >>}" \
+                --credential-source-file=<< parameters.oidc_token_file_path >>
+      - run:
+          name: "Authenticate with GCP using OIDC"
+          command: |
+            # Configure gcloud to leverage the generated credential configuration
+            gcloud auth login --brief --cred-file "<< parameters.gcp_cred_config_file_path >>"
+            # Configure ADC
+            echo "export GOOGLE_APPLICATION_CREDENTIALS='<< parameters.gcp_cred_config_file_path >>'" | tee -a "$BASH_ENV"
+
+
+jobs:
+  docker-build:
+    environment:
+      DOCKER_BUILDKIT: 1
+    parameters:
+      docker_name:
+        description: Docker image name
+        type: string
+      docker_tags:
+        description: Docker image tags as csv
+        type: string
+      docker_file:
+        description: Path to Dockerfile
+        type: string
+      docker_context:
+        description: Docker build context
+        type: string
+      registry:
+        description: Docker registry
+        type: string
+        default: "us-docker.pkg.dev"
+      repo:
+        description: Docker repo
+        type: string
+        default: "oplabs-tools-artifacts/images"
+    machine:
+      image: default
+    steps:
+      - checkout
+      - run:
+          command: mkdir -p /tmp/docker_images
+      - run:
+          name: Build
+          command: |
+            # Check to see if DOCKER_HUB_READ_ONLY_TOKEN is set (i.e. we are in repo) before attempting to use secrets.
+            # Building should work without this read only login, but may get rate limited.
+            if [[ -v DOCKER_HUB_READ_ONLY_TOKEN ]]; then
+              echo "$DOCKER_HUB_READ_ONLY_TOKEN" | docker login -u "$DOCKER_HUB_READ_ONLY_USER" --password-stdin
+            fi
+            IMAGE_BASE="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>"
+            DOCKER_TAGS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|-t ${IMAGE_BASE}:|")
+            docker build \
+            $(echo -ne $DOCKER_TAGS | tr '\n' ' ') \
+            -f <<parameters.docker_file>> \
+            <<parameters.docker_context>>
+      - run:
+          name: Save
+          command: |
+            IMAGE_BASE="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>"
+            DOCKER_LABELS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g")
+            echo -ne $DOCKER_LABELS | tr ' ' '\n' | xargs -I {} docker save -o /tmp/docker_images/<<parameters.docker_name>>_{}.tar $IMAGE_BASE:{}
+      - persist_to_workspace:
+          root: /tmp/docker_images
+          paths:
+            - "."
+
+  docker-publish:
+    parameters:
+      docker_name:
+        description: Docker image name
+        type: string
+      docker_tags:
+        description: Docker image tags as csv
+        type: string
+      registry:
+        description: Docker registry
+        type: string
+        default: "us-docker.pkg.dev"
+      repo:
+        description: Docker repo
+        type: string
+        default: "oplabs-tools-artifacts/images"
+    machine:
+      image: default
+    steps:
+      - attach_workspace:
+          at: /tmp/docker_images
+      - run:
+          name: Docker load
+          command: |
+            DOCKER_LABELS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g")
+            echo -ne $DOCKER_LABELS | tr ' ' '\n' | xargs -I {} docker load -i /tmp/docker_images/<<parameters.docker_name>>_{}.tar
+      - gcp-oidc-authenticate
+      # Below is CircleCI recommended way of specifying nameservers on an Ubuntu box:
+      # https://support.circleci.com/hc/en-us/articles/7323511028251-How-to-set-custom-DNS-on-Ubuntu-based-images-using-netplan
+      - run: sudo sed -i '13 i \ \ \ \ \ \ \ \ \ \ \ \ nameservers:' /etc/netplan/50-cloud-init.yaml
+      - run: sudo sed -i '14 i \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ addresses:' /etc/netplan/50-cloud-init.yaml
+      - run: sudo sed -i "s/addresses:/ addresses":" [8.8.8.8, 8.8.4.4] /g" /etc/netplan/50-cloud-init.yaml
+      - run: cat /etc/netplan/50-cloud-init.yaml
+      - run: sudo netplan apply
+      - run:
+          name: Publish
+          command: |
+            gcloud auth configure-docker <<parameters.registry>>
+            IMAGE_BASE="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>"
+            DOCKER_TAGS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|${IMAGE_BASE}:|")
+            echo -ne $DOCKER_TAGS | tr ' ' '\n' | xargs -L1 docker push
+
+      - when:
+          condition:
+            equal: ['main', <<pipeline.git.branch>>]
+          steps:
+            - gcp-oidc-authenticate:
+                service_account_email: GCP_SERVICE_ATTESTOR_ACCOUNT_EMAIL
+            - run:
+                name: Sign
+                command: |
+                  git clone --branch v1.0.3 --depth 1 https://github.com/ethereum-optimism/binary_signer
+                  cd binary_signer/signer
+
+                  IMAGE_PATH="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>:<<pipeline.git.revision>>"
+                  echo $IMAGE_PATH
+                  pip3 install -r requirements.txt
+
+                  python3 ./sign_image.py --command="sign"\
+                      --attestor-project-name="$ATTESTOR_PROJECT_NAME"\
+                      --attestor-name="$ATTESTOR_NAME"\
+                      --image-path="$IMAGE_PATH"\
+                      --signer-logging-level="INFO"\
+                      --attestor-key-id="//cloudkms.googleapis.com/v1/projects/$ATTESTOR_PROJECT_NAME/locations/global/keyRings/$ATTESTOR_NAME-key-ring/cryptoKeys/$ATTESTOR_NAME-key/cryptoKeyVersions/1"
+
+
+  docker-tag-op-stack-release:
+    parameters:
+      registry:
+        description: Docker registry
+        type: string
+        default: "us-docker.pkg.dev"
+      repo:
+        description: Docker repo
+        type: string
+        default: "oplabs-tools-artifacts/images"
+    docker:
+      - image: cimg/python:3.7
+    resource_class: small
+    steps:
+      - gcp-cli/install
+      - gcp-oidc-authenticate
+      - checkout
+      - run:
+          name: Tag
+          command: |
+            gcloud auth configure-docker <<parameters.registry>>
+            ./ops/scripts/ci-docker-tag-op-stack-release.sh <<parameters.registry>>/<<parameters.repo>> $CIRCLE_TAG $CIRCLE_SHA1
+
+  go-lint:
+    parameters:
+      module:
+        description: Go Module Name
+        type: string
+    docker:
+      - image: cimg/go:1.21
+    steps:
+      - checkout
+      - run:
+          name: run generate
+          command: |
+            make generate || go generate ./...
+          working_directory: <<parameters.module>>
+      - run:
+          name: run tidy
+          command: |
+            go mod tidy && git diff --exit-code
+          working_directory: <<parameters.module>>
+      - run:
+          name: run lint
+          command: |
+            golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" --timeout "3m0s" ./...
+          working_directory: <<parameters.module>>
+
+  go-test:
+    parameters:
+      module:
+        description: Go Module Name
+        type: string
+    docker:
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest # only used to enable codecov.
+      - image: cimg/postgres:14.6
+        environment:
+          POSTGRES_USER: opc
+          POSTGRES_HOST_AUTH_METHOD: trust
+    resource_class: small
+    steps:
+      - checkout
+      - run:
+          name: go version
+          command: go version
+      - run:
+          name: prep results dir
+          command: mkdir -p /tmp/test-results
+      - run:
+          name: run generate
+          command: |
+            make generate || go generate ./...
+          working_directory: <<parameters.module>>
+      - run:
+          name: run tests
+          command: |
+            gotestsum --format=standard-verbose --junitfile=/tmp/test-results/<<parameters.module>>.xml \
+            -- -coverpkg=github.com/ethereum-optimism/infrastructure-services/... -coverprofile=coverage.out ./...
+          working_directory: <<parameters.module>>
+      - run:
+          name: upload coverage
+          command: codecov --verbose --clean --flags <<parameters.module>>
+      - store_test_results:
+          path: /tmp/test-results
+
+  py-presubmit:
+    parameters:
+      poetry_root:
+        description: Root of the Poetry project directory.
+        type: string
+    docker:
+      - image: cimg/python:3.11
+    resource_class: small
+    steps:
+      - checkout
+      - run:
+          name: prep results dir
+          command: mkdir -p /tmp/test-results
+      - run:
+          name: run presubmit
+          command: |
+            poetry install
+            poetry run presubmit
+          working_directory: <<parameters.poetry_root>>
+
+  build-release:
+    parameters:
+      package_name:
+        description: Package to build
+        type: string
+      artifact_path:
+        description: Path to build artifact
+        type: string
+        default: ./bin
+      release_env:
+        description: Release environment
+        type: string
+        default: prod
+    docker:
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
+    steps:
+      - checkout
+      - run:
+          name: Build
+          command: |
+            VERSION=$(echo "$CIRCLE_TAG" | grep -Eow 'v.*' || true)
+            make build-release VERSION=$VERSION RELEASE_ENV=<<parameters.release_env>>
+          working_directory: <<parameters.package_name>>
+      - persist_to_workspace:
+          root: <<parameters.package_name>>/<<parameters.artifact_path>>
+          paths:
+            - "."
+
+  publish-release:
+    parameters:
+      package_name:
+        description: Package to publish
+        type: string
+      artifact_path:
+        description: Path to build artifact
+        type: string
+        default: ./bin
+    docker:
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
+    steps:
+      - attach_workspace:
+          at: <<parameters.package_name>>/<<parameters.artifact_path>>
+      - run:
+          name: "Publish Release on GitHub"
+          command: |
+            go install github.com/tcnksm/[email protected]
+            ghr -t "$GITHUB_TOKEN" -u "$CIRCLE_PROJECT_USERNAME" -r "$CIRCLE_PROJECT_REPONAME" -c "$CIRCLE_SHA1" -delete "$CIRCLE_TAG" <<parameters.package_name>>/<<parameters.artifact_path>>
+
+workflows:
+  op-conductor-mon:
+    when:
+      or: [<< pipeline.parameters.run-build-op-conductor-mon >>, << pipeline.parameters.run-all >>]
+    jobs:
+      - go-lint:
+          name: op-conductor-mon-lint
+          module: op-conductor-mon
+      - go-test:
+          name: op-conductor-mon-tests
+          module: op-conductor-mon
+      - docker-build:
+          name: op-conductor-mon-docker-build
+          docker_file: op-conductor-mon/Dockerfile
+          docker_name: op-conductor-mon
+          docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
+          docker_context: .
+      - docker-publish:
+          name: op-conductor-mon-docker-publish
+          docker_name: op-conductor-mon
+          docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
+          context:
+            - oplabs-gcr
+          requires:
+            - op-conductor-mon-docker-build