From 66f0ed20dba26fef36c7202ada94cc3be9f30052 Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Thu, 7 Mar 2024 16:53:03 -0800 Subject: [PATCH] feat: add init code --- .devcontainer/Dockerfile | 20 + .devcontainer/devcontainer.json | 21 + .dockerignore | 1 + .github/workflows/build.yml | 34 + .github/workflows/kind.yml | 36 + .gitignore | 22 + .vscode/settings.json | 10 + CODE_OF_CONDUCT.md | 9 + CONTRIBUTING.md | 14 + LICENSE | 201 ++++ Makefile | 116 +++ README.md | 230 +++++ SECURITY.md | 41 + SUPPORT.md | 11 + api/docs.go | 139 +++ api/swagger.yaml | 70 ++ assets/images/cluster.drawio | 85 ++ assets/images/cluster.png | Bin 0 -> 35167 bytes build/ci/Makefile | 40 + .../certs.d/mcr.microsoft.com/hosts.toml | 5 + build/ci/configs/containerd.toml | 48 + build/ci/k8s/app.yml | 78 ++ build/ci/k8s/kind-cluster.yml | 53 ++ build/ci/k8s/rbac.yml | 77 ++ build/ci/k8s/test-random.yml | 30 + build/ci/scripts/install-deps.sh | 88 ++ build/ci/scripts/kind.sh | 391 ++++++++ build/package/Dockerfile | 28 + cmd/proxy/cmd.go | 16 + cmd/proxy/main.go | 166 ++++ go.mod | 210 +++++ go.sum | 872 ++++++++++++++++++ init/systemd/peerd.service | 12 + internal/cache/syncmap.go | 71 ++ internal/cache/syncmap_test.go | 94 ++ internal/containerd/mirror.go | 158 ++++ internal/containerd/mirror_test.go | 229 +++++ internal/context/context.go | 158 ++++ internal/context/context_test.go | 259 ++++++ internal/files/cache/cache.go | 186 ++++ internal/files/cache/cache_test.go | 224 +++++ internal/files/cache/interface.go | 27 + internal/files/cache/item.go | 131 +++ internal/files/cache/item_test.go | 197 ++++ internal/files/cache/main_test.go | 59 ++ internal/files/files.go | 37 + internal/files/files_test.go | 84 ++ internal/files/store/file.go | 142 +++ internal/files/store/file_test.go | 240 +++++ internal/files/store/interface.go | 48 + internal/files/store/main_test.go | 42 + internal/files/store/mockstore.go | 26 + internal/files/store/store.go | 154 ++++ internal/files/store/store_test.go | 135 +++ internal/handlers/files/handler.go | 82 ++ internal/handlers/files/handler_test.go | 152 +++ internal/handlers/files/main_test.go | 42 + internal/handlers/root.go | 107 +++ internal/handlers/v2/handler.go | 93 ++ internal/k8s/events/events.go | 118 +++ internal/k8s/events/events_test.go | 70 ++ internal/k8s/events/interface.go | 19 + internal/math/math.go | 62 ++ internal/math/math_test.go | 56 ++ internal/math/reverse.go | 24 + internal/math/reverse_test.go | 57 ++ internal/metrics/interface.go | 19 + internal/metrics/main_test.go | 56 ++ internal/metrics/memory.go | 118 +++ internal/metrics/memory_test.go | 49 + internal/oci/distribution/v2.go | 45 + internal/oci/distribution/v2_test.go | 55 ++ internal/oci/mirror.go | 130 +++ internal/oci/mirror_test.go | 146 +++ internal/oci/registry.go | 147 +++ internal/oci/store/tests/mock.go | 53 ++ internal/remote/interface.go | 25 + internal/remote/reader.go | 285 ++++++ internal/remote/reader_test.go | 309 +++++++ internal/remote/tests/mockreader.go | 37 + internal/routing/interface.go | 32 + internal/routing/router.go | 237 +++++ internal/routing/router_test.go | 250 +++++ internal/routing/tests/mock.go | 89 ++ internal/state/state.go | 124 +++ internal/state/state_test.go | 54 ++ pkg/containerd/reference.go | 155 ++++ pkg/containerd/reference_test.go | 92 ++ pkg/containerd/store.go | 350 +++++++ pkg/containerd/store_test.go | 658 +++++++++++++ pkg/k8s/election/election.go | 124 +++ pkg/k8s/election/election_test.go | 118 +++ pkg/k8s/k8s.go | 27 + pkg/k8s/k8s_test.go | 12 + pkg/math/compare.go | 25 + pkg/math/compare_test.go | 57 ++ pkg/math/segments.go | 49 + pkg/math/segments_test.go | 143 +++ pkg/mocks/contentstore.go | 74 ++ pkg/mocks/eventservice.go | 30 + pkg/mocks/host.go | 75 ++ pkg/mocks/imagestore.go | 59 ++ pkg/mocks/peerstore.go | 135 +++ pkg/peernet/network.go | 125 +++ pkg/peernet/network_test.go | 130 +++ pkg/urlparser/azure.go | 37 + pkg/urlparser/azure_test.go | 63 ++ pkg/urlparser/parser.go | 28 + pkg/urlparser/parser_test.go | 30 + scripts/coverage.sh | 94 ++ tests/Makefile | 20 + tests/cmd/cmd.go | 17 + tests/cmd/main.go | 52 ++ tests/dockerfiles/random.Dockerfile | 25 + tests/dockerfiles/scanner.Dockerfile | 32 + tests/random/random.go | 256 +++++ tests/scanner/scanner.go | 65 ++ 117 files changed, 12174 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .dockerignore create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/kind.yml create mode 100644 .gitignore create mode 100644 .vscode/settings.json create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 SECURITY.md create mode 100644 SUPPORT.md create mode 100644 api/docs.go create mode 100644 api/swagger.yaml create mode 100644 assets/images/cluster.drawio create mode 100644 assets/images/cluster.png create mode 100644 build/ci/Makefile create mode 100644 build/ci/configs/certs.d/mcr.microsoft.com/hosts.toml create mode 100644 build/ci/configs/containerd.toml create mode 100644 build/ci/k8s/app.yml create mode 100644 build/ci/k8s/kind-cluster.yml create mode 100644 build/ci/k8s/rbac.yml create mode 100644 build/ci/k8s/test-random.yml create mode 100644 build/ci/scripts/install-deps.sh create mode 100644 build/ci/scripts/kind.sh create mode 100644 build/package/Dockerfile create mode 100644 cmd/proxy/cmd.go create mode 100644 cmd/proxy/main.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 init/systemd/peerd.service create mode 100644 internal/cache/syncmap.go create mode 100644 internal/cache/syncmap_test.go create mode 100644 internal/containerd/mirror.go create mode 100644 internal/containerd/mirror_test.go create mode 100644 internal/context/context.go create mode 100644 internal/context/context_test.go create mode 100644 internal/files/cache/cache.go create mode 100644 internal/files/cache/cache_test.go create mode 100644 internal/files/cache/interface.go create mode 100644 internal/files/cache/item.go create mode 100644 internal/files/cache/item_test.go create mode 100644 internal/files/cache/main_test.go create mode 100644 internal/files/files.go create mode 100644 internal/files/files_test.go create mode 100644 internal/files/store/file.go create mode 100644 internal/files/store/file_test.go create mode 100644 internal/files/store/interface.go create mode 100644 internal/files/store/main_test.go create mode 100644 internal/files/store/mockstore.go create mode 100644 internal/files/store/store.go create mode 100644 internal/files/store/store_test.go create mode 100644 internal/handlers/files/handler.go create mode 100644 internal/handlers/files/handler_test.go create mode 100644 internal/handlers/files/main_test.go create mode 100644 internal/handlers/root.go create mode 100644 internal/handlers/v2/handler.go create mode 100644 internal/k8s/events/events.go create mode 100644 internal/k8s/events/events_test.go create mode 100644 internal/k8s/events/interface.go create mode 100644 internal/math/math.go create mode 100644 internal/math/math_test.go create mode 100644 internal/math/reverse.go create mode 100644 internal/math/reverse_test.go create mode 100644 internal/metrics/interface.go create mode 100644 internal/metrics/main_test.go create mode 100644 internal/metrics/memory.go create mode 100644 internal/metrics/memory_test.go create mode 100644 internal/oci/distribution/v2.go create mode 100644 internal/oci/distribution/v2_test.go create mode 100644 internal/oci/mirror.go create mode 100644 internal/oci/mirror_test.go create mode 100644 internal/oci/registry.go create mode 100644 internal/oci/store/tests/mock.go create mode 100644 internal/remote/interface.go create mode 100644 internal/remote/reader.go create mode 100644 internal/remote/reader_test.go create mode 100644 internal/remote/tests/mockreader.go create mode 100644 internal/routing/interface.go create mode 100644 internal/routing/router.go create mode 100644 internal/routing/router_test.go create mode 100644 internal/routing/tests/mock.go create mode 100644 internal/state/state.go create mode 100644 internal/state/state_test.go create mode 100644 pkg/containerd/reference.go create mode 100644 pkg/containerd/reference_test.go create mode 100644 pkg/containerd/store.go create mode 100644 pkg/containerd/store_test.go create mode 100644 pkg/k8s/election/election.go create mode 100644 pkg/k8s/election/election_test.go create mode 100644 pkg/k8s/k8s.go create mode 100644 pkg/k8s/k8s_test.go create mode 100644 pkg/math/compare.go create mode 100644 pkg/math/compare_test.go create mode 100644 pkg/math/segments.go create mode 100644 pkg/math/segments_test.go create mode 100644 pkg/mocks/contentstore.go create mode 100644 pkg/mocks/eventservice.go create mode 100644 pkg/mocks/host.go create mode 100644 pkg/mocks/imagestore.go create mode 100644 pkg/mocks/peerstore.go create mode 100644 pkg/peernet/network.go create mode 100644 pkg/peernet/network_test.go create mode 100644 pkg/urlparser/azure.go create mode 100644 pkg/urlparser/azure_test.go create mode 100644 pkg/urlparser/parser.go create mode 100644 pkg/urlparser/parser_test.go create mode 100644 scripts/coverage.sh create mode 100644 tests/Makefile create mode 100644 tests/cmd/cmd.go create mode 100644 tests/cmd/main.go create mode 100644 tests/dockerfiles/random.Dockerfile create mode 100644 tests/dockerfiles/scanner.Dockerfile create mode 100644 tests/random/random.go create mode 100644 tests/scanner/scanner.go diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..dda2f12 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,20 @@ +# This is a dockerfile specifically for running as a devcontainer +FROM mcr.microsoft.com/oss/go/microsoft/golang:1.22-fips-cbl-mariner2.0 +RUN tdnf update -y && tdnf install make -y && tdnf install git -y && tdnf install gawk -y + +RUN go install github.com/cweill/gotests/gotests@latest && \ + go install github.com/fatih/gomodifytags@latest && \ + go install github.com/josharian/impl@latest && \ + go install github.com/haya14busa/goplay/cmd/goplay@latest && \ + go install github.com/go-delve/delve/cmd/dlv@latest && \ + go install honnef.co/go/tools/cmd/staticcheck@latest && \ + go install golang.org/x/tools/gopls@latest && \ + go install github.com/axw/gocov/gocov@latest && \ + go install gotest.tools/gotestsum@latest && \ + go install github.com/jandelgado/gcov2lcov@latest && \ + go install github.com/AlekSi/gocov-xml@latest && \ + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /usr/local/bin v1.54.2 + +RUN mkdir -p /go/src/github.com/azure/peerd + +WORKDIR /go/src/github.com/azure/peerd diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..8766805 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,21 @@ +{ + "build": { + "dockerfile": "Dockerfile" + }, + "customizations": { + "vscode": { + "extensions": [ + "golang.Go", + "hediet.vscode-drawio", + "eamodio.gitlens", + "yzhang.markdown-all-in-one", + "ms-vscode.makefile-tools", + "GitHub.copilot", + "ryanluker.vscode-coverage-gutters" + ], + "settings": { + "editor.formatOnSave": true + } + } + } +} \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e660fd9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +bin/ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..2a7857f --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,34 @@ +name: Build and Test + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.22' + + - name: Prerequisites + run: make install-linter install-gocov + + - name: All + run: make all + + - name: Coverage + run: make coverage + + - name: Save Coverage Report + uses: actions/upload-artifact@v3 + with: + name: code-coverage-report + path: bin/coverage/coverage.html \ No newline at end of file diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml new file mode 100644 index 0000000..585cfdc --- /dev/null +++ b/.github/workflows/kind.yml @@ -0,0 +1,36 @@ +name: Kind CI + +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + +jobs: + e2eBlobs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Prerequisites + run: sudo make tests-deps-install + + - name: "End to End: Blobs" + run: make build-image tests-random-image && docker system prune -f && make kind-create kind-deploy kind-test-random + + - name: "Clean up" + run: make kind-delete + + e2eCtr: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Prerequisites + run: sudo make tests-deps-install + + - name: "End to End: Ctr" + run: make build-image tests-random-image && docker system prune -f && make kind-create kind-deploy kind-test-ctr + + - name: "Clean up" + run: make kind-delete diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6b6e0b2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +bin/**/* + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..787e847 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,10 @@ +{ + "[markdown]": { + "editor.rulers": [ + 120 + ] + }, + "files.associations": { + "*Makefile": "makefile" + } +} \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..f9ba8cf --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,9 @@ +# Microsoft Open Source Code of Conduct + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). + +Resources: + +- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) +- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..ebf23ac --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,14 @@ +# Contributing + +This project welcomes contributions and suggestions. Most contributions require you to +agree to a Contributor License Agreement (CLA) declaring that you have the right to, +and actually do, grant us the rights to use your contribution. For details, visit +https://cla.microsoft.com. + +When you submit a pull request, a CLA-bot will automatically determine whether you need +to provide a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the +instructions provided by the bot. You will only need to do this once across all repositories using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..890fa0c --- /dev/null +++ b/Makefile @@ -0,0 +1,116 @@ +# Compiler variables. +VERSION = $$(git rev-parse --short HEAD) + +# Go build variables. +GOCMD = go +GOTEST = $(GOCMD) test +GOBUILD = $(GOCMD) build -installsuffix 'static' -ldflags "-X main.version=$(VERSION)" +GOLINT = golangci-lint run + +# Source repository variables. +ROOT_DIR := $(shell git rev-parse --show-toplevel) +BIN_DIR = $(ROOT_DIR)/bin +TEST_PKGS = $(shell go list ./...) +TESTS_BIN_DIR = $(BIN_DIR)/tests +COVERAGE_DIR=$(BIN_DIR)/coverage +SCRIPTS_DIR=$(ROOT_DIR)/scripts + +include $(ROOT_DIR)/build/ci/Makefile +include $(ROOT_DIR)/tests/Makefile + +.DEFAULT_GOAL := all + +.PHONY: all +all: lint test build ## Runs the peerd build targets in the correct order + +.PHONY: build +build: ## Build the peerd packages + @echo "+ $@" + @( $(GOBUILD) -o $(BIN_DIR)/peerd ./cmd/proxy ) + +.PHONY: install +install: build ## Installs the peerd service in the project bin directory + @echo "+ $@" + @( cp $(ROOT_DIR)/init/systemd/peerd.service $(BIN_DIR)/peerd.service ) + @( cp $(ROOT_DIR)/api/swagger.yaml $(BIN_DIR)/swagger.yaml ) + +.PHONY: help +help: info ## Generates help for all targets with a description. +# Read the makefile and print out all targets that have a comment after them. +# If external Makefiles are referenced, trim the external reference from the target name. ex. Makefile:help: -> help: +# Sort the output. +# Split the string based on the Field Separator (FS) and print the first and second fields. + @grep -E '^[^#[:space:]].*?## .*$$' $(MAKEFILE_LIST) | sed -E 's/^[^:]+:([^:]+:)/\1/' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +.PHONY: install-gocov +install-gocov: ## Install Go cov. + @echo "+ $@" + @( go install github.com/axw/gocov/gocov@latest && \ + go install gotest.tools/gotestsum@latest && \ + go install github.com/jandelgado/gcov2lcov@latest && \ + go install github.com/AlekSi/gocov-xml@latest ) + +.PHONY: install-linter +install-linter: ## Install Go linter. + @echo "+ $@" + @( curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /usr/local/bin v1.54.2 ) + +.PHONY: build-image +build-image: ## Build the peerd docker image + @echo "+ $@" +ifndef CONTAINER_REGISTRY + $(eval CONTAINER_REGISTRY := $(shell echo "localhost")) +endif + $(call build-image-internal,$(ROOT_DIR)/build/package/Dockerfile,peerd,$(ROOT_DIR)) + +info: header + +.PHONY: lint +lint: ## Run linter. + @echo "+ $@" + @( $(GOLINT) --timeout=10m ./... ) + +.PHONY: test +test: ## Runs tests. + @echo "+ $@" + @( $(GOTEST) ./... ) + +.PHONY: coverage +coverage: ## Generates test results for code coverage + @echo "+ $@" + @( COVERAGE_DIR=$(COVERAGE_DIR) $(SCRIPTS_DIR)/coverage.sh "$(ROOT_DIR)" "$(TEST_PKGS)" true ) + +.PHONY: swag +swag: ## Generates the swagger documentation of the p2p server. + @echo "+ $@" + cd $(ROOT_DIR)/internal/handlers; swag init --ot go,yaml -o $(ROOT_DIR)/api -g ./root.go + +define HEADER + + _____ _ + | __ \ | | + | |__) |__ ___ _ __ __| | + | ___/ _ \/ _ \ '__/ _` | + | | | __/ __/ | | (_| | + |_| \___|\___|_| \__,_| + +endef + +export HEADER + +header: + @echo "$$HEADER" + +# build-image-internal takes the dockerfile location, repository name and build context. +# Example: +define build-image-internal + @echo "\033[92mBuilding Image: $2\033[0m" + + @echo docker build -f $1 \ + -t localhost/$2:dev \ + $3 + + @docker build -f $1 \ + -t localhost/$2:dev \ + $3 +endef diff --git a/README.md b/README.md new file mode 100644 index 0000000..6cec7e0 --- /dev/null +++ b/README.md @@ -0,0 +1,230 @@ +# peerd + +[![Go Report Card]][go-report-card] +[![Build Status]][build-status] +[![Kind CI Status]][kind-ci-status] +![Code Coverage] + +This project implements peer to peer distribution of content (such as files or OCI container images) in a Kubernetes +cluster. The source of the content could be another node in the same cluster, an OCI container registry (like Azure +Container Registry) or a remote blob store (such as Azure Blob Storage). + +## Quickstart + +This section shows how to get started with `peerd`. + +```bash +$ make help + + _____ _ + | __ \ | | + | |__) |__ ___ _ __ __| | + | ___/ _ \/ _ \ '__/ _` | + | | | __/ __/ | | (_| | + |_| \___|\___|_| \__,_| + +all Runs the peerd build targets in the correct order +build-image Build the peerd docker image +build Build the peerd packages +coverage Generates test results for code coverage +help Generates help for all targets with a description. +install-gocov Install Go cov. +install-linter Install Go linter. +install Installs the peerd service in the project bin directory +kind-create Creates a kind cluster +kind-delete Deletes kind cluster +kind-deploy Deploys the p2p application to kind cluster +kind-get Shows the current kind cluster +kind-test-ctr Deploys test 'ctr' to the kind cluster +kind-test-random Deploys test 'random' to the kind cluster +lint Run linter. +swag Generates the swagger documentation of the p2p server. +test Runs tests. +tests-build Builds the tests binary +tests-deps-install Install dependencies for testing (supported only on Ubuntu) +tests-random-image Builds the 'random' tests image +tests-scanner-image Builds the 'scanner' tests image +``` + +### Build and Deploy to a Local Kind Cluster + +To build and deploy `peerd` to a 3 node kind cluster, run the following. These commands will build the `peerd` +docker image, create a kind cluster, and deploy the `peerd` application to each node in it. + +```bash +$ make build-image && \ + make kind-create kind-deploy + ... + ... + daemonset.apps/peerd created + service/peerd created + waiting for pods to connect + pods: peerd-5trwv peerd-q2c45 peerd-tkj5k + checking pod 'peerd-5trwv' for event 'P2PConnected' + checking pod 'peerd-q2c45' for event 'P2PConnected' + checking pod 'peerd-tkj5k' for event 'P2PConnected' + Success: All pods have event 'P2PConnected'. +``` + +On deployment, each `peerd` instance will try to connect to its peers in the cluster. + +* When connected successfully, it will generate an event `P2PConnected`. This event is used to signal that the + `peerd` instance is ready to serverequests from its peers or upstream. It can be found in the pod events. + +* When a request is served by downloading data from a peer, `peerd` will emit an event called `P2PActive`, + signalling that it's actively communicating with a peer and serving data from it. + +Clean up your deployment. + +```bash +$ make kind-delete +``` + +### Run a Test Workload + +There are two kinds of test workloads that can be run: + +1. Simple peer to peer sharing of a file, specified by the range of bytes to read. + * This scenario is useful for block level file drivers, such as [Overlaybd]. + * This test is run by deploying the `random` test workload to the kind cluster. + * The test deploys a workload to each node, and outputs performance metrics that are observed by the test app, + such as the speed of download aggregated at the 50th, 75th and 90th percentiles, and error rates. + + ```bash + $ make build-image tests-random-image && \ + make kind-create kind-deploy kind-test-random + ... + {"level":"info","node":"random-zb9vm","version":"bb7ee6a","mode":"upstream","size":22980743,"readsPerBlob":5,"time":"2024-03-07T21:50:29Z","message":"downloading blob"} + {"level":"info","node":"random-9gcvw","version":"bb7ee6a","upstream.p50":21.25170790666404,"upstream.p75":5.834663359546446,"upstream.p90":0.7871542327673121,"upstream.p95":0.2965091294200036,"upstream.p100":0.2645602612715345,"time":"2024-03-07T21:50:34Z","message":"speeds (MB/s)"} + {"level":"info","node":"random-9gcvw","version":"bb7ee6a","p2p.p50":5.802082290454193,"p2p.p75":1.986398855488793,"p2p.p90":0.6210418172329215,"p2p.p95":0.0523776186045032,"p2p.p100":0.023341096448268952,"time":"2024-03-07T21:50:34Z","message":"speeds (MB/s)"} + {"level":"info","node":"random-9gcvw","version":"bb7ee6a","p2p.error_rate":0,"upstream.error_rate":0,"time":"2024-03-07T21:50:34Z","message":"error rates"} + ... + + # Clean up + $ make kind-delete + ``` + +2. Peer to peer sharing of container images that are available in the containerd store of a node. + * This scenario is useful for downloading container images to a cluster. + * This test is run by deploying the `ctr` test workload to the kind cluster. + * The test deploys a workload to each node, and outputs performance metrics that are observed by the test app, + such as the speed of download aggregated at the 50th, 75th and 90th percentiles, and error rates. + + ```bash + $ make build-image tests-scanner-image && \ + make kind-create kind-deploy kind-test-ctr + ... + ... + ... + + # Clean up + $ make kind-delete + ``` + +### Build a Docker Image of `peerd` + +To build a docker image of `peerd`, which can then be deployed on each node of your cluster, run the following. + +```bash +$ make build-image + ... + ... + => naming to localhost/peerd:dev +``` + +### Build `peerd` Binary + +To build the `peerd` binary, run the following. + +```bash +$ make + ... +``` + +The build produces a binary and a systemd service unit file: + +``` +|-- peerd # The binary +|-- peerd.service # The service unit file for systemd +|-- swagger.yml # The swagger file for the REST API +``` + +### Throughput Improvements + +An Overlaybd image was created for a simple application that reads an entire file (see [scanner]). The performance is +compared when running this container in p2p vs non-p2p mode on a 3 node AKS cluster with [ACR Artifact Streaming]. + +| Mode | File Size (Mb) | Throughput (MB/s) | +| -------------------- | -------------- | ----------------- | +| Non P2P | 200 | 3.5, 3.8, 3.9 | +| P2P (no prefetching) | 600 | 3.8, 3.9, 4.9 | +| P2P with prefetching | 200 | 6.5, 11, 13 | + +## Features + +`peerd` allows a node to share content with other nodes in a cluster. Specifically: + +* A `peerd` node can share (parts of) a file with another node. The file itself may have been acquired from an upstream + source by `peerd`, if no other node in the cluster had it to begin with. + +* A `peerd` node can share a container image from the local `containerd` content store with another node. + +The APIs are described in the [swagger.yaml]. + +## Design and Architecture + +`peerd` is a self-contained binary that is designed to run as on each node of a cluster. It can be deployed as a +systemd service (`peerd.service`), or as a container, such as by using a Kubernetes DaemonSet. It relies on accessing +the Kubernetes API to run a leader election, and to discover other `peerd` instances in the cluster. + +> The commands `make kind-create kind-deploy` can be used as a reference for deployment. + +### Cluster Operations + +![cluster-arch] \ + +[Work in Progress] + +## Contributing + +Please read our [CONTRIBUTING.md] which outlines all of our policies, procedures, and requirements for contributing to +this project. + +## Acknowledgments + +A hat tip to: + +* [Spegel] +* [DADI P2P Proxy] + +## Glossary + +| Term | Definition | +| ---- | ------------------------- | +| ACR | Azure Container Registry | +| AKS | Azure Kubernetes Service | +| ACI | Azure Container Instances | +| DHT | Distributed Hash Table | +| OCI | Open Container Initiative | +| P2P | Peer to Peer | +| POC | Proof of Concept | + +--- + +[CONTRIBUTING.md]: CONTRIBUTING.md +[kubectl-node-shell]: https://github.com/kvaps/kubectl-node-shell +[Go Report Card]: https://goreportcard.com/badge/github.com/azure/peerd +[go-report-card]: https://goreportcard.com/report/github.com/azure/peerd +[Build Status]: https://github.com/azure/peerd/actions/workflows/build.yml/badge.svg +[build-status]: https://github.com/azure/peerd/actions/workflows/build.yml +[Kind CI Status]: https://github.com/azure/peerd/actions/workflows/kind.yml/badge.svg +[kind-ci-status]: https://github.com/azure/peerd/actions/workflows/kind.yml +[Code Coverage]: https://img.shields.io/badge/coverage-54.9%25-orange +[cluster-arch]: ./assets/images/cluster.png +[node-arch]: ./assets/images/http-flow.png +[Overlaybd]: https://github.com/containerd/overlaybd +[scanner]: ./tests/scanner/scanner.go +[ACR Artifact Streaming]: https://learn.microsoft.com/en-us/azure/container-registry/container-registry-artifact-streaming +[swagger.yaml]: ./api/swagger.yaml +[Spegel]: https://github.com/XenitAB/spegel +[DADI P2P Proxy]: https://github.com/data-accelerator/dadi-p2proxy diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..b3c89ef --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,41 @@ + + +## Security + +Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). + +If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. + +## Reporting Security Issues + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). + +If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). + +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). + +Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: + + * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) + * Full paths of source file(s) related to the manifestation of the issue + * The location of the affected source code (tag/branch/commit or direct URL) + * Any special configuration required to reproduce the issue + * Step-by-step instructions to reproduce the issue + * Proof-of-concept or exploit code (if possible) + * Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. + +## Preferred Languages + +We prefer all communications to be in English. + +## Policy + +Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). + + diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 0000000..fe7dc18 --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,11 @@ +# Support + +## How to file issues and get help + +This project uses GitHub Issues to track bugs and feature requests. Please search the existing +issues before filing new issues to avoid duplicates. For new issues, file your bug or +feature request as a new Issue. + +## Microsoft Support Policy + +Support for this **PROJECT or PRODUCT** is limited to the resources listed above. diff --git a/api/docs.go b/api/docs.go new file mode 100644 index 0000000..4c571f4 --- /dev/null +++ b/api/docs.go @@ -0,0 +1,139 @@ +// Package api Code generated by swaggo/swag. DO NOT EDIT +package api + +import "github.com/swaggo/swag" + +const docTemplate = `{ + "schemes": {{ marshal .Schemes }}, + "swagger": "2.0", + "info": { + "description": "{{escape .Description}}", + "title": "{{.Title}}", + "contact": {}, + "version": "{{.Version}}" + }, + "host": "{{.Host}}", + "basePath": "{{.BasePath}}", + "paths": { + "/blobs/{url}": { + "get": { + "summary": "Get a blob by URL", + "parameters": [ + { + "type": "string", + "description": "The URL of the blob", + "name": "url", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "The blob content", + "schema": { + "type": "string" + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "string" + } + } + } + } + }, + "/v2/{repo}/blobs/{digest}": { + "get": { + "summary": "Get a manifest or a blob by repository and reference or digest", + "parameters": [ + { + "type": "string", + "description": "The repository name", + "name": "repo", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "The digest of the blob", + "name": "digest", + "in": "path" + } + ], + "responses": { + "200": { + "description": "The manifest or blob information", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "string" + } + } + } + } + }, + "/v2/{repo}/manifests/{reference}": { + "get": { + "summary": "Get a manifest or a blob by repository and reference or digest", + "parameters": [ + { + "type": "string", + "description": "The repository name", + "name": "repo", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "The reference of the manifest", + "name": "reference", + "in": "path" + } + ], + "responses": { + "200": { + "description": "The manifest or blob information", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "string" + } + } + } + } + } + } +}` + +// SwaggerInfo holds exported Swagger Info so clients can modify it +var SwaggerInfo = &swag.Spec{ + Version: "", + Host: "", + BasePath: "", + Schemes: []string{}, + Title: "", + Description: "", + InfoInstanceName: "swagger", + SwaggerTemplate: docTemplate, + LeftDelim: "{{", + RightDelim: "}}", +} + +func init() { + swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo) +} diff --git a/api/swagger.yaml b/api/swagger.yaml new file mode 100644 index 0000000..74cc02c --- /dev/null +++ b/api/swagger.yaml @@ -0,0 +1,70 @@ +info: + contact: {} +paths: + /blobs/{url}: + get: + parameters: + - description: The URL of the blob + in: path + name: url + required: true + type: string + responses: + "200": + description: The blob content + schema: + type: string + "404": + description: Not Found + schema: + type: string + summary: Get a blob by URL + /v2/{repo}/blobs/{digest}: + get: + parameters: + - description: The repository name + in: path + name: repo + required: true + type: string + - description: The digest of the blob + in: path + name: digest + type: string + responses: + "200": + description: The manifest or blob information + schema: + additionalProperties: + type: string + type: object + "404": + description: Not Found + schema: + type: string + summary: Get a manifest or a blob by repository and reference or digest + /v2/{repo}/manifests/{reference}: + get: + parameters: + - description: The repository name + in: path + name: repo + required: true + type: string + - description: The reference of the manifest + in: path + name: reference + type: string + responses: + "200": + description: The manifest or blob information + schema: + additionalProperties: + type: string + type: object + "404": + description: Not Found + schema: + type: string + summary: Get a manifest or a blob by repository and reference or digest +swagger: "2.0" diff --git a/assets/images/cluster.drawio b/assets/images/cluster.drawio new file mode 100644 index 0000000..0ba98e8 --- /dev/null +++ b/assets/images/cluster.drawio @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/assets/images/cluster.png b/assets/images/cluster.png new file mode 100644 index 0000000000000000000000000000000000000000..5dd7c677db8c5109a59f99e5ae4c98cec5d95195 GIT binary patch literal 35167 zcmeFZ1z6Nuzb_8MpbX7`AW9D1CEX<*0umz45YmlEcPJ%{q97<>(4cgO3Rs8;h@^lh zNT-1EUjvM?*-|_xazmAD+Eu)^Dx#TkD&j?`N&qak| zJOb#}{YhN>RXnZS9kkqRkt^YwUL4HAzgORqE4_X8Sgv1Y8>^(8>+n+{|AKjqu z7;yLk0sg~oK906XufxFyP2O&9&fbphe{8gIb9F`9c>kfJm8Ykh-ygf#xj7&1^6;(h zU?YDRgl^Ecv2y-vtC6Fvx5GhSG2z4Ip?j+$9qk=3Zx-M`eBxq->3rDW0#t@Fz)Ps@buUEJ&~@5FaO5_4mIIxAL?| zkRLsRm9wKg`n%_WfDWz57wPGZJhYI%*~NYn#;TBRE=X_B%Rnla0V0RuJTxdlp~LTf zzqKPQc-ZD}L^DDdi?TYj2>V}i1Jgc~GWg))zpsYBR|)8e_Je;UL!d)on?Pj$EBRS@ zxgQv;oufZmj{hmr5kGUVLX31U(mEm?vA+=xrtSYkHbVcG%SIR@3yf@p42f{oDpX|~7tZdQ2#?i~s)t*IA5%jcm^s;dSZ6Kl?yaIUq!CMXn{3RqJU2Wyj zX#_+uS2tI*TmK>M{V5*)F$4;M-d^6GZca#}!{7wc<^I|a^PGSf_zuFi=VfCsKR>Uy zpfIMvWWPaF06oWk4<&Q}zt*Bq$bdjEpAWkAZF^&F?-Ul=8&j-HkPlp~1KL4Hx z{RK18!OoGke@TbG^UGnd_Hy&_+|LPMh=Az9IBzRYd*opn+Slh_1NaeB!dTbQ_>5`u zL^@k}JNh1`sKZzOU0gbRgRYz7zLB8C1J)ua`p5im24mTnDGpYDI0`KFSAPA|L_(M( zf))^%_zWh|U?w`)@?R#}Pa!Ip+_JyO|5a82O8#qF`G3;`_=UtUCcba7ydu9{;d#(tpoISn57^h|f0qXciyr1RjNzU+^azaM{<9u%1~c$*B0=#( z2#J~KKhXpJEH)t){(%RG{>lmekprOJ;m^j5@d`8yM}Iy#*MFD;&>jCg(ZA*u zhdKRdTKI2uieKm-;}k-Iyni<5f7&mE4&yb(FGR(8#gG1TXw3fwZX*02CJJi|5L!*sDOazDp=RY-JYCjtg zYtg_&VIPTs!dm{YD0FeOwM7@6e_JR0u0+uw=MRwU?@LWIh{K>NG|D;@)c;ATDJb|S zJ&PTR{|J)7)D%aN%%6e6zfx#oaMd3whJ%23L|F$7zcl(knm&MPhoI5j>QB)1uNnCe zGzy9T2{dBB=)XhLV*lgQ`wsrc+6kSv{tJ9s_{?F!f{ESY|3041dlwKum47#ZYVcP@YU6C><>hDtM0F%U+xU3;?(=f!$oa=O3WgrW zNHog!0@)c8GtuAwEbe`fd<0peVFnsT>_duu2#$82Uq$FIKz^{He+EY%g$MtwRpX!e z4-iQYc7e&^|135T#3Td^&cIZWn2G-LFf+PjL}%+G=7dS<|1d>>ogL&2P?r8m9Y@-Z z4E`T5@QZgARl=4&Hws$}v?WKkt7vk$4!){*V#&&Hk^T?T8*QIPEC>#c>V8Jam||eg5sdTK8v1|I=D(6Q&-~eH z{*7#j&|fPnMQ0x`H18zHuZ{HfbMtfppLE=jp5ScB&2`@>|BmPSXUVN2<<^l@D|iM| zMIBae=&hI>mX90r~H>MQ2e>n5x^Xf9Z9RdF0mZVtC-UOjC=npw3g6e zn*T>GX{fDifbmn{^+#Owf69?x<;X9s9mY;!jQ1VUBBm5Qs>FYRn*=)eeM#vc4FA6C zg^A6WTV$B;ziyb-k&6amVKHE-Dajl7o6lzv3~QbLx!0^$o5r3xR>Lfq$7-Ng8%gmt zR@1dMYFJSHA){Y&C(pI|hj_JwoY!<+$$|q69DcIjm=aVZtJBq*-P`^qK9_-%ywUi2 zbSAU$&G({#xy#ZsmtSAFQo3$F*4XbCGT;|vL~Mn?hBF-eh4Di@lks86SciX+OmZw? zp7Ic8x&1#<6jnME|JbkR>CkRW&*V@q$P;EU`=^2&bo@A4jVGbb5h4MPk5ys%qe%4EGISj+{mkEhYjEj2JpH2|UW@&v zYP$4m6X1hf=@OpLRxvM=*`Er+5A|r7m6^Q9+!lK z&b@fm+b0M1>SLu?5LccA2J69dzVuP1OdvcT#hxKgc?0iwIaRSemE`$D$u=!E^RZcn z6H9BUl;$9#+r2dCRh#4+F1ahsN_2W(((@}>Eg;&cEZhZ_YVh-d$guqjZVTSPC?)JM6k@gyqsi6Wz z<*^M6hyjVBo_a*@$bDu>?o+YOC^L4$CHBHO$m0!cdo@s@gW{7L)Edi zYx82gi8zjZR}lSnLU8XVE5ZP7y%|1|8VY}>_<4x_$tQDDxHZvsxzj=tO`I;gJ93If z!&y~#_*rbH6j`emsoDlxLgzcgz0KRQ?6sxJb)5-U;0@tUK$RDn)}G=nkfn$!L9g%m zA$nQPs9LWKynD*1&qWc=f=JCLN`0FaY3xS#w#i5aKCdVPKiQ)DoJ4u*RS#B(gn`-) zvvo{isgMFXL{tld$7G%@;kVWEa9u z>U=efYFxWG5D^MZ2z7Y}F2rI?7vYB(J;G^f*pFib9kV@Sb-Ri(45fb26i!kWx-v^I zuO^Mhq^e$SxS%8vtE&I%Io=DmhioVv-&njm>&7eh?qnOk5lX(f(fnG+WY5#-4cj)A zAaQW=xq2OgxE?`A9I{%bJ~sb51M!_2GA6E91<&Ly8X3UD$Er`VPn;YO?V}J2m*z!W z`s6dzro;|Ad2@myN=@H{@7wxYHk=0$G$jdLSlaFrB?;KHGE};mY4>bH>Ln;{Jv<-O z;;wpf57O8)kp>M+3L}hfOVR!8pdKD1_++(5YeL)qA}>O}a!2MtB{gyQtfXz&3LX*< z_C}`vbGm;1_UpLsbQj;2ju)kpH!Y2|m^O`gkP-WCmE6A1AQye&jjG^oee$HuE8S@0 zAb+H4SfpntGzvEeHydvDb7DLJR-kh8w2kKiGh(iC5zdj+OqF7!__}jKEQch`dP>Qh zUq(qE{%JlWi#{-`WoZKM4I8(ZF`ZtMl(0g4sOOo=YJIodcgt75OCjV~TA7kL8Z?8+ zGRQ8RRuLZ8jA?yyhDAZG=A06+YSGBqEh;i`M#RY?fefqfIBG--?s|QVWAiS$ZVK*t z5q=`0z6h53=1`fq4=(~a5%Wq@lP7!Dy67V;@9KWk)ui7-aKLVAwlMU&9_MdY!Y@3_ zO;<1Mgw0J>#w#b^VM;^uy{{udV)o9rx?xM*7&qKe)roiEam@YOG8DLd%X6ecY=M4J zA4ago$bS&gOS;jWAe*|D?yUh^Y5R0nXLO=lF(X=$8(yTXyPzFy;+o*=w(}aL)HS(B zAk8+uTAG}CoQrr{J0#ek`QGdDDHc>JsoUPi=SdE`mMP)s_;1)LI^yomkWk{SNe8z> z;-t2S`5=Q?kQiroOLYV`Crm9eE622z;(I{nRHA7|eK9wgdic+0;l-i;rUrJG%))w( z6Fp%Zx>X?akb!UX=d$67i>hPGShsC)v&j&ffaY1_g3xz&gG^ZUjJ{r%eK~ydss$nP zSzdHLf##ExG-%_ilMEhv(|36;SI3k@Rr=I(NS#Vjz25kP=k)uFQ~?PcEBL5OIR=(m z793Rk*oD}qzgxWzt;FsoN2%V5^nJsQ3&DqFBIkpmo7R385A4jOvb*TyNzMLn9V>WPP5Xk_fr?_u_zg)@lC4@<_0zn) z_T@P~M1rDEcs~A{N9vanDo4+WbWa#RLMB6eOeV#UPukEC-Cp z#cAE&CI%O4u4S;4&e6^HiCXzZ_WpKy1ve`HkGgJ;za0B<<%ibW*J^%4-a|nfUG&-s zWuGi*PpA=1HCFoL2V_DUV6X6}qXZ-9&E(%U5AZ0w^&h-JWUx1RjTAq&i3VqS@@LOb znanwMO3MO@<7Qva1-kRGA2Vh;+si|xy)~2$-BOYBP@r9oj;&*2jeeI`%}B>0&u5c8 zRhAdpbpA&~l}#LOKXqg2U4F(5)r#qI)U$3b7I`BYt(~#gZ+-@TNH@%94y8BuT)_8; zS|rHSzDpQLu@dD`&>@+A`m+!Na1Sh$&f_6^8L->IbJ+O|!&RD^#{4EWuV~z|0zz*) zTt}3whqX8Luwm~of9^Qr(yS#$fSrjgV8Z#}+|5#f!Gau(x>VAx2aFfJ)xX&YTTWL6w5|2I#1U2u`XRW|MzCf3!ZJM zkkRl7rm*UtC+><`vXiVU^e4W(TWzZ;vPBn9H2bY%PFinlFg?;e(1ru%d%VY(1ZGRX z>Vf2m%isD?%$pJjM{v;8g%_~em8M0yH|Sb$%M$e??nc{uN>3fXzh~oeH*rH6rCB}) zl~hNtY=v3I#L$NqJWYM^Wc!Y~!czUarh06*8ULH(SCOg1Tg+eVR@|-3C(2@4iQPDj z!|`S)V%A-q1r$9Cn7@2pFm}1NE<#r*)s!^l6k6@n5Y{gL?hXi1{<+UP=9!-s)brx_ zuf~+(P~hHlpXi835^d?WIFh)%G3LEVdh4Q3Gf~jx&TA};z09Yz)mlt#-zHC};5_y&+xgPzzO8$Q%zee95d0(R^tv8)YQU zN|>)Hb{@V~&1V@}OX}8psouEAcuDtUeAMVwU8>4?wQS8?t@!eOEhZnWqPTvw8)x<4 zOEh-1v1(v1Pwj){6^Li=daAZB<4PzQ@Re5+cQrp&RL}7H@ohu zr&6C9O8V*M;KYqk7|Hd>cSsz50xC9ax7;$-$ON)aXS5+DE*ARLJIo9x$yY2Btk1|2 zi>^Fe>mL{JBiF|OCmV*cw$~0*Gu)_ zWyepkDzQUU>)09;?(5WT->#IIu)IRYP3j&H8IGN=G`0fE$K7nH6Ppd48+U=Lg{-;@ z#yy?wc337E&nHnitN9Fs-5r#4r4Wv0DTxVXU&T4*7x7s!_&;s^uoP+6im%lvJ}Afz zGJ;Lv86I0YE4b|H*Cwjhp}f=IV@!#i#aATi?yYYxGMIZ!A%*?Kh1b1r&XlWkU;Ui5 zx0u|{R>ZfFMZ3Y%Y2Ka7>lSd8;E9FBsaQKTJT|@*)QRD>kOuYel7=CifV)|Bg0|L= z;?}uWuf%|QKn!{!)cItd)@#?$#|-`V-ZSdFrCV{hGCuF%tr6Zdb855QHjzEaU9Y2G zlAY=%#k>F!zg1gclA7>vB*6pro9#@v4hk6#9&pW3!}Xo#@p%s=1qwXtSQt^)QJlG- znAy2>xartqPLkK@Melu$CYpx>KHzK$~>5 zcKF`lizw4Ekq%hBUTcf}g+k?uEmc%d%<+KC?Vr-M` zSUkw`LyCFE>X;|*OB0Tt9)0c}-!2$k!YtRcWQgw;aFe7x-`y&2ekXUH+_gwG(f@X2 z2Hb8#m5gdq+(ZzSYUyXK)HtuA+^#{3BX#d>l*R;WnXy!#PZ_FkBY!gFVQGAegWpj5 zW?iL%S$kI=0WLQAO**o-#i^k?)+^yFF%(W8)a_#`2H|)t9JeC4yDc-0?`)EJ=SomC-c%$d`{L`U*16>5# z64{kxR_sg9n;$W6<rd^LaXI3h3Dtzg2!o

f6op>W~$76do&dMuA;-JnY9t*|D!4^afeh z%eoQ8v&VN`#-i;iko*_F1$R`3I?})Pdi&bdeYVD7tRz8HrIUn$ojoZQ^5``x_Pyv9 zo%$);yUJgNaand^pJUawm@tkPxmBMZAY$8FqLP0}z&>*GOIGH{c%{_P4lSj)X0NVq zi;l^uZ!#2br*H$ygvD-Belo~W1V60Xx4@gAvTKmJgQ8vzBEt-_e@s>zXUak49s2+0DT2?#Jrg?f5qkN0 z`RZ(5pI?3$Pn2BPnqHb{7aa{;eKoc?R5@4o(XIJMp-j;Blim61v12oxDK`8e*Kwvp z-(Ww-PQ(eOIj)^8^P{=)()$4SwM-=vX@t;9wSl*NYoWM(<2kCIQ*AdkGt`iRM@@(`yMeazEIg~Wag8e#%&^q z7xY>J!?D;ybL5JQ%N4rQI3C*!mRZS4?=H1R)O&wT)leE;`ojdtr_ToP&6^u+2L*&AB7MQEjbzYT6*Ui~~jSm`uiK=z$C zBQW(5^RSFnTjZRm>#&$5c-u7i?IjiKFCwZFs(nX_LnVnWJwwReb)-3DuU9#N@)cPM zUur8iwYtK5$j{a68^p8{-fcPZ5hZ+jd8&d#Pp=#@S8_vSVm`BJYhT_@CiLcOBvDKG zFR_w+Ky{vF)l*|1C^RZvsr8y!7`vyOpe{PU8~(c6vrC=J6@gd$!LpB#BP? z!uukV)ojEH5^*Yc8*tUF>}pm++o z5DyxTeX2~=^=ylTRT9&P@mqJqk_}kCK#*hDGr3QAyIYe zrrx$bU@=zUGuPjk?f)qbAJ4Z#2}9f^2SO}@zvu5x((f6Mm6+APf1X57KU;OoH9b

KMz1XXIIk2OyBY7A7CH9n zj&!|i7zCG9W~l`SkG|{I+s4TuZ=^Sebq36@xD50cn>OZpB|Kw1g}+?==z9N%{W-_B#~UsdK+NK}D_8^pRe^asm}!(_YY<*nV9Y5kz&7&C8v_zVoClX8V;Sy;PudnRPyJZU-rJp;Op^V1@k0fYb}W(_#McAT zQBTDZ_^X)Mtj-BegO=X*H`-3tdCY_-`E$Oe8<~q2+P}FYHk5(!IaLM*=zZ<1?aAA2 zP2&F3-RVqOh9Am-`5TwBoxOJI{R3c<%30=_-w!4S|K3%!oFA&RsB-?`x$-twMMeBk zx<1R@Zt(HA`Q_>Kb-uo)%>KW%llP&Vyz01qP4w7>!CUN@@a2s=oAocK*X-`*->RVnMHN zVPcU$masWeoJ%5(51U4AglOqY)8m7%WFU@a0Sb;rl1PBydhJYJ(zR)m>-)*#E;b{UDi2;s@37jEp z73Mr#bFScgAuF(@`9VW;vS;ILg-LlJkMT$V!I%+?@zu`lhb1KrLj) zz1VYC@l{Jz+6iC+q~F3^`-?I(Z=F$dt(~THs!$0R3=KQs*b<8E`Kbfuu`IgfZ$Y2#kc4Bnjhiko3h3vJvZi|*uu?jX!Z6gyt7SqQjOZA2^LlGJP+8a_JduyfU2>vTKt)Qt!7nqotsF{!q9pJgMQNyzuBfC2xrJ`)6Cg#7CgG7cl6t1-%r@xrP%z(5@9t|A&*W-m2nxJ-U>a>?5w!jS=x>uW zC4YtkmL0tP)eKnZhi7vo^}HiPnIm_kuWaBoG*Izgbbwt13(VKflF-XNOVo9PFMG~c zfAAvH6s{B^XXn%x8JI7NytwEP5BP8Eoj19cRs@+zQ?SfGftbyiti>H8>^4dxAM*MI z2@CcNewNhb-t9dQY+X5sf_644QExv{4^%sUK&;NZsZMid|7hGB8pWvJ`}x)FX`s|A zk0;t=`1;Z#hpUG@U#I|t$DJ+dxA4SfLTM2G_KPM}l@kB7a2B~Pxum)!8;)j?Vl*L{ z@mu~aMj~2?zU^XbCb_t0#K*t6vIET35%}7vHIv!$tR+ZDf&ps|;s9ED&-H83UHMiX zOLbDxw`kfsp*o~tOJ3Z6_R)imLYCF9?;gCE>blMXVnP3ka>OyBm#4^EhCxbreCAT0 z!|KZPi@DfN>gnscXybRAev`eofmOv}ovszh^LGBVl9k38ee-NkZUo_3cKPke=-Z%ewSU4 z&{b7}Aa6Uqb=9}hu*((F8cgj%Z34uH#~KD;k!e4UbA1#`i(?yV z6krVX3hr^fhNW<1ZRmMyAMJrBp3~~w%e&KarOkAX955lfmtXqtYu#46anVz4{3A_i z8B6^{2Ew`PbN3FYL!dEOsHQr>B37sG+x~6_IJT?`E|lrp9G@r=lHj7%R}lA+8+(5yF>j7fWQZlKgO zu{)j1=whwsG@Ja@%UV_%R35gHAd3lx$>pBCR>U3)fcK>#;mx-y^p&548@_+O?fA9| zq%untm2w)pp`fH#PQu%dBGijM(1UG$BOR!>t!qn)QbSojq5bG5pPj({0@-W$bhG5lw5Mk9k21 z8Iw?1pM8I$abw77oy?2}uZzoyK*98#@?dz&EgxoVxGEv`*0uT3MpG1TWg^5q9T9bW zfbELrk}gGAz9&FIkQs$_YEz#1=l5guS<{e+(Qbefb%AiH9PH0JuLo{T#28H*H~23v z9j6a0<7>OHEq$58IoPeRz(NAgRM+_N9biDC@Rrw)mnWZ_x~-cV7oHz6J^%_sNaeoW0?3TvwuvF;F2MPUiVCR@ zpC9g$#Z&TJ(wgnQkVAnjHpF3pQ@IvO+Px&?(3{tV8;_M3{fwGb{cV{k)39D7Zm#Ea z_X;o(T)}ZTO7l{GQLXUvhRdtwmjJApB0n>nx{}iX-Y=h?8W5L5V5(=eYzKDR#Na+) z=BEa{3Qozc#v@&1-Z&)jK{T@xt2NPmICMp*=LEJ-Xojb|W{27X_ujzy$^lyt-aIf{o&%3ljeZ;z}6`-*>gMC$mo3E}o_8CJ8P8xRy#B8|B9 zciOAeLZ8F4h}`|3i|n~eo@Uj$#T&Fgam$9aRp7kOZD6e8D&H}pqy@NZ|Itl$MAj$R9K zjCXVb^!2^r{$BB&QXUvZRi`I~k`=*A{nmJZpF5PX9!RvsFLkkHcbu0Y$3I_S&{e?- zojwU2y8vf*E~med12g?bDR3Obq-2=o<&*cljo~-j$U|AM4Oen#Ak(HAkSLxFCdxBb z2_uNlfOW#-3>@Y+rez1+iXoj)B)x;^UFY=9QPu7Zk$Re>Ob^+k83A!ro>#Du_Z0 z1ri40v`PIbkd3J#XSeSqjsAB;jRG$0Sf7Brb7HDu}&w1odlw?F=pf zB_Y?6JgLZ-%%h@{z_f;;uJ!XG84^APJ4NvfyOZ`l+b?8!zAvT%Z15hMLqs0gn$v|-XR*w%Cb^t9|=Ph9;yW#p5D@p!J1WPpv_ zcWh{&8up3JD2=QZ=XH&2bc@alz<(64GdqH>p#LjbLT zn#NW?n@4eU^$aSBG%JA2d}=21j$k`08`xKH%(TJH_r2|)5Xu%9Nn_m z+P6{yPozC%NcCa7m>$9443m3>wyM0Fa>+O1^sj1%muY_E(ooQi((=2&ZB#ORxt^lz zBGv;sW5~FDj<2|!*hFXS2r0<(^_KQHO(gbavCMt)q*Vs8$qllw<_wIJ#pqC*g`&2D3hUliCJDIWClTtLD1V^(u@2kxXS za@3u|X`p=XH6uu&Z;Vh-eA8XTX{c2~vky0#L^py=w0(Ub0V%dH`7n1P(tW$Bf_dp% z^p>*Z(HYukU*4F-dlHO9mKu=~e_jPH^+4_!-w;^#mg)0UV41-z$?54VSrR^}Ix^px z>*F6K%FP)WZUlfZtY#kYmF3aJ+6uQqv|PljR(Zs;&vWCoeHOAUA_hBFfszU@Z7olJ zxSOF*8`hs|$Fe3km}2-)myzp2e)e^+B`hSBh-~|+#z;8AIe6k}0KlmJ8abd$S^<@< zJU#_G0|-A)PFTyNlQ}fHm|2$qI!e&VX%|oxa4(#sl35TohM-mvp~4P5ExD}s_mE(s zuVduSj4)f)Fi`fq0F;Xh2Mp945+n)*h@S06HvTteLF?~(OD$wAjNbd9pe)WkWj8a) z@Z!tFort{9fc;v_bNUc&bWn!0e9-4a^c)MLN6fvxEvkbiDlCi!{7{h&ZJ(V|If^do zM)E|(`a1zPfReLYG&MAA&ldK_MbAzR87Svyx$B!*1OpZoV!sMfKwXpJJ-%^C4aljY z$YS+$&N$bQ@ltwbD*`EJ5j6dS2)*u-;-1#19&wxm_3nX=>D)K8xHnzH*F{0?1hEU& z0dX7aCa-hw@td>TxN3uXui`fWmEx|0g%3{jQ!K@G8PRP~ zHxyvjx9HOOKynqC@g!%q@*3s*ydPt+*_qY{-{3@8DgiU%K|D*VR% ztlF9EfFZB+{W(6>0~8hAct)xVu#qkqfvZzplN%dkFXrY0zub+n?&q>PF%hM&fHj%a z)R)6qZv7M=fQiM+Vd;8$^7LaA0xoB-p0vvPc{*}RLj%%sZV-|11_TY;&o49j^8^4B z--2gKN{3hNqnVegTV^R={eHEmVPJl($htwddmm?1F#F$Se$XI5XM`mwZK-Bl?V9!6 zNPBoX&`C}1I*0RwgQL* z&Gi1`*mGJMA%!INCvJ+k+)B5gJx;8Fm3RuxkQfvhpF=}4T+>IU)g=wPD=+$kEY(`7 zIE+sid60yxphvArZ#SyUvrsQNb17bd^ULF+oH$f=2(aYINew%+e}(- zEa21pTE+^HTLe?=wU$?+W6maI;S|R^t5MLh{4(X>XYo{gy!sEcrF90ar8~?6Wu5xD zok$%AN=9(g+tG)2CtYM!%zEZ3Cw` z#*}nH*6o`+n~T-mU#30gfpZPt8q_rpdKO9RSc^@Ls-tA>vCj@NymMti92Bm8Rn`y| zY`}F1JN7a=J!&%#p&`A`B|X8+rzh#d!l>8iF1^ES0n$gcj zmzK3PfC69X)7Lq74v)-#B8tiukTqe;%DS{OP8twSHLLn1>I7!zayskaMOS#wl{T3P zn>PK==QUflOttPT>GZJaRr=A8Q?oj+A0s5dPA$1Ab`HK4@PP-Wp7M4vY-f(VDoF6! znSw%J#!60^Tr1Q#D*N1pxvHNfR=f>Wbn0?ffd+VdE(Vy_UG!)CG^VYF;jy*f4QHAvjbuaWTEGu09`W4Wt zsP#*rj6vawsdf#ZprLtXS$x`WBVQ3{$a8x!9U~;PoOoGY6M|^o>>f_2yM62Wd*;HI zJ6}$46Fd|(A31j}TQgnoEvSc`SH-)n;idr1UCttSTzj1bf4b)3?cw!l{K^h&jc`44 zZx(Dx>wvW{jFZoks!>##?68SI(r@H`PXfXs0(J({p*wp%`Kl($+nbASCU#5F7Wb6m zR#7Gta*cf6&$ZcUo%)N&mY(b3P)sXHcpr0I^3yX~(o+w=z8gTdMz!q@jt%cCzvGof zG2Uf{GV+GgHh~i4A%E0-^9ZHZU|MRZXo~xF%OUB@YR<_)f?n2Yp@9y1S4(%DO zxs4u@A4YO?R*Xwx^O>1@Q*##UVh#8nprGzyPjSBSf3Rs&h=0!DJ0xC)K)xmtBPp`Ksb>!kF!?Nu5$Cg+V! zVE4W=M?Plmn7dB;C~I3!Jue_E!L?C${`u)(=4{AQ$MKf1dzs|R&F(?itSMF2pUlCD zQQ}vx2`Uqf2LTK(e->Ada+;W!p!U@6=c7?5ILn#NO_EvpK~0ph%}R=b8r*54G#&J2 z%hkMcl6=HGNEDp9d(RJ@NAX$QnHMkB+}k3S-8L;;Sb9(;Fm!{jZC<0POChE9InE4aGQ`Dk1G z;^3G&3hHyswj)vX3E{EzZNR9Mp{jb`0BV5XlF-$;Ygx+v{W&y#-v-~-G+5w6!6kT0 za{|*mAG-$HSvP^LqGsG7Hp1!U#K4rQ%e7yZyvamF35`WOpx-1#%aTr$H)nqmtZ{4{ zndp60*;~nN&TkpF`ze7BUSaSZ90i#EgaKV?8AVb(DFjDADEhXC=W-zCZTKV9o-XF=cfHSzqduRB{OG{2ylNxT{yiT-YdZIcd+=v}Nt|K&g%`(W0p0Sz)k-kc{Lq ziZSdZB9!N5+y=4@wSiEMH~xA4>#gy;%iyrL<*8z9AAQ(k=7-UMl?s4LUop2nSAElLnAREzAeiacV#{s!{as5lcB?8K}XCff* zUg5fZAwaGL1?m+f*QxDZMX47==J=;I!OeI$^6Z)?)0`=40h^P>kNnSRSw(<}c=-x5u>U zcf2V@9uVF{qjL{v-*?hSiTRK{L7J(}Zmv>jz!hdxjt3oHTiNjuHG#}N+@aK*Q&Ul8 z$`|x>%O~2!nPy+W2alI<;nq#&>hmpoQ&n{kU1<;m@RJ;(ONw6oyk#g-;C(#Ax2RQ( z@5Kd8*fMiDr#_t7 zFW4TW8-}zilirUQ&$m&2z39(|L%a&GDl{QfB1xVD7W`1!dYqwxm8P|P>U88|I#N1% zzC{1HOhE8j;;@K+-Fs376nP8Zv?i3h#_)+4Ucfk2N5X&~@0Y~nfXEsL+7xxG_S3ku zr?RA2Un)YrDz=pJ9mD&C%Yr>wk*$-Yv?OqKXI3g@E#>MGkJ@RC@KGuJ+d}10Ud&a} zt=~>A^oiRWS@EI|Vwa+^h}YUVGQ^Jk$gscTL2L8u_Jtoaoc=HxanHOH#I&!+fXLB~dV2e^F1fF4iv>abvvGoGa`{4oN2yDnoj+7OmH2du zjC>-Q_<{P?U8J?TsYZCC5QlRH`>3-J%$7IITjGT8n>*5|TKbRg-egI#0jlW+U=viY zof4LcdmLIk0~q8R;JC6&a(0Lt2ykjAMFnZmy{YFH)kEH3G{+`3I) zr3y$|FQAl2retDNr2Bk$goOFFJNNf!qSJYS-&fE#Oy)Mhae>=>VhuRb%aaR(+#D~0 z(Z}N^`pbeV7BrH+n&bRUev9TGYP}vff2?}7ZPpmzBd;rEOo+z)_~&Ego*4tm=`~U& zYg8_?`j$92Yz}RZauEk)$ZVmcIb!LP!p%E@2LX+_}&;tf$z|YpA#kH3471 z1CIV8rbCTER%H;8=J0tfrlD}AfTC)zSM8Y307Lh$Xa(CQPJsd0)WdbFz} z^3?(aFk2{gLkmMFO*)hl7DwuT0Zya)R8h?Ln{ikL)^v_hsX4WymM-zNlgSQR9t>hH z6(DMemg!@>qRi@j4r;OThSi9NMH!d0TI8TnFgI>ZunaU4b$M^$NGJ}52 zD%KiuQ4}k<2M{&|2fvj{6y<@P+lqROu&onr&8jIs1KjsCWboG2_Q5aYQ<(_#&#l^~ z^H>=1Wk=%OKOk>(l{Ww6q%!kP0e5;xsON8!5>rod)?6VqxdpW*YY~dRZhDsi(Dqd2 z6{|a>;F8&bQWz={Z`V?p-QZ#JiAQt28hn-gpRxm2)TqVWbHEuihh*vi;KqeKA4DV; z0}3VF#n{`ydy~K746FlzHDe`4R1Gw2Xc(-+(n?O9!@>wT`_0aOACyu(L|4ZC2c*!A;5i!ubqjot6(mhXC9XYKPs!4BfkJA+yaG6`zK0n9C>R) zAb>ow@tS#JVIJ7AU3TYVWjE77)INdDIa;!Wr21SlpU8xHqF^H4lObdSgXR;=}a`>sL1gL}7B}(L(&eHdHu20*AYx&t3d>9n|>8Rjh1-w$u^W(EDDMfOd*0}e6GS;H?kQL zc;ZXiG?BhJ%S~w77%?P8Al}+~J9)B7vg8T#$)(&f>X!J_6L*mlwXgVJk1kD}57&^J z$GYd`_G!OU)vrz|C){-XuUI3(G}&5=5pn5Qi60k0acbF;)Mkg3V1m9P({daNq%#No zhs!P!t7!Ws0m>RH_93zH^4LX~WgO9i=e;JPwPsJjrrj{3pQfC@9PsY00I@&!Cntz{ z)m;XLK0*BzJojPW=eIDM^stR$k$1UvHCHSlIkXgJX+)CgLrulOIA; zJ>nyV(V*1J_@2r#=AN%F6dyKFz-;lbftVzIEtk$d|9ihpP6N$fp1f2?UuBYE0D3oL z>-paoD>vF2{<52=Ma-mOE#PWQKHviMKw!N*m7;RdYx~>dH_(|N%}lX#E1zFEnbiXp zRS!@SWdox%9=R{?NSi*pcA5=@E03T(RjQtk-+L5Eo)#Ikai&h7y+a!dBP}q0GKA$W zxOUhyu}~>Ebvqij2!R&JNsK^l3Nyi8W=R@>TOr8GMujN+3d>dqhiO&O`evC`yDb3T zy$zSW<`$3B`L{@jnEWpF_aaDS%8GExpoFv%2H?1r6kU^nu>N$twY3qI4$ zFqR3FscZS@==uGUA4M>d%5V4bj&@brW^R7Dnpr2hP36 zK^lBa|KvH_$5H5ZUo!&%s_+-CLJK6(v0ACBTpEJ(#6Z;!?|!LHC=rWQP*YQvg5PmM zs232?MSJqpEsLOXGbOhI7cJ2@&t|low}Qc?4MXZisgfVyI-e=|>F&ZeH_MoKL>a20 z|51&uL0vu&0zm5whxCMm*q@5@+WD%F+~yB&=<$OyKTdEgE#n4LGzvy>n;^h7kuug0 zcJhhth;p-XxgRMF#>ZA(;GpeJ2W@wJ*P~uJz2g{2A%gPdn9P&Quu2Jkw}d17L22)fq~=gi|DdaBma+QC_RhgD(qUfd$3Ang zR_$L;5>;|l>yDJTDH*}@GpF2s)Poyerud!U?qwce-35wSB6?gFYo1^117}?bW4P?lqTCa1^Qn z@$)4 z&l0B7QZyng6X2F?zimVu-t{J4`#K%S$`?w!^{P@7MW5b3taj_Tl?$k9zVs7JlZT1! zs>gB%+KV%`eK79;a@Va8|KY{xlfCe5fnbi0fq^^&+FG{w5Hyxadnv4GUG*sm5b~9(L zq$lUsz8e?H`mm5ZnJn^VOw{E}gwdvND(u>7=BbPLsqV3~hPq1NpB_lzBmCCxV;d2J zP^=i@aRroV^=S8xDV@ptT;B9#q65n|AG~R*uXkktSq^<=cfP3dGf7r~JINa@OLoWP z_a#zu0`!(vEtKug>8@}u@tmJmQiRin37#+OMQ>jNz5NHdWl^T}Q(*fWOJ1}e#wRQd z;Y%E0=!xUd0Ua|jmOFST4<6!?9z4uOKSXC>na*w>TY7Iup&W#R742ZhF@c>6*Q7Qv z6Y`~j1-6xm&g!&EN3uFSxtzn4$zwYctJsX*+vvwXKn-w!I6#=shKGjh78>%*5k>*j zdmebw%)46^7tGO#ED`Z5@TNlpHL)fV(xb@PH!*hsQpYDRRa~M&_ar{r6F`nYxzNd~ zdhHw_WBRC9Sca!;%>Z$WMBP`DAdZmXh;jnt5d0%KWE=<5eahgmot8F%U3WMhA}Ek%z@^~9t`>hhnXgwV3?cx0`GP?v>XSu-5;39%TO2)+S0H=Qd&U;y0=h+?+Z z-BQ#&bxqjAnEWQtke&{DdKrv{C?BWFh2yoTErVM%_Mv&uWv%KLo?vt?W;6i0s#w^_ z;#RjvllxCuv6J6@OGW2qcFfyvu1i6@3s$um#&JFLt7M=rZMWP6tGBV8IdMe9q1f1W?l=N~dkp|jkv1QBI}APrpi+!E2W95| zQpi`3KPB`AS}?h4zXao>jAgn<>JKBguVcG&Cu2c6b7;c@VAtRKJ0*(Ej@P}b9rLV4 zpn?HrjGln}H+nAQg2a-!Z;tp0spoRVb8CX%;S6_Ufqg+AF`sNfmocCq#oh;)VMkXZ*8ONPT$qs9`Nk0&cG)L(F3W8AjOrtCWBu?CZq`})KEJ>C(@Pi>W0uzoTFE|+Hr1BlR24le}@l#@9-KHtL=$r)NTvGYQc|Px1tEpaLh!M6tWe{mShwe zAz5XW6^h7uK0kHmzVGXKp8s|J*Hzc`JLk8)<1=3GcR4iC&vYq_ta?cjj!zV|7vu79udhkWNDUS!!z`a2Y+T~Uj(8ia488`K-v_nmzNQ<;&SHzu(A3Tn7A zl)3LpOXS++cJhXpt#r%K)$Dm2xfYDfuZkXFKF7H59st2YHO?Xe$}pp=;z(ianHXq_5?V>M49qA&aP8~?eSrgzIK`JtO%)=Dq^~>+R{!{!x z04%nA>CBt}D61>qT*_y*_%+ctaMKZjucVeZ9&P9yWdSqP1tER>`%o!vLQpm(@U`@3^qJzGl@oAy?B&r38la5^cI~`pcFrew_ zFJaxsa$M)0?{rSch{!NY)j|W6xJ?~NTQ1?mu~@t~G91*A!1?#Sks^xf&wmF!O+Mgz zvT!z6gGcDEsN;8deaLxZE7>=OZJiAj;R;X0?S6|Eg>+Fi_Cs=aPG&E;P<>8`YhMAX zHuN{_o99llGwhsqhg}CoZu3XCQPp*0D<4$^Rq7~A&!d9m^el*R+~)$AC0z_lF0`%N z^X{zD_x_%1nvc}Ysj+>JG+9cMjI`fM-jCT2XW&gHT0Xw75^ffN(pYe2h8h|0r#wKy zyIfNMP1ytmlrXiBX7}Z9&?vvxC%i%Pk@e*@ru`(J1xE+^NrRSPLFC<85VQWSPrhmy zAinYh_SnpcuQ9I2o<{X$%{nP%1El!*nyeog-#ZQN^|>!aZw6Y5A`#u=B(2ea4Uz!49z=mRp1Qo}uRHh0$mL^p zcklO-q*ZHU{gd886uGG6jhCmUUl`~CkmAN=3sW-D?x6wYg1IpkAZ@!>3c$+$Mx_=d z4l{;=WGz&iX0guhAU_5Zk4&@C-_xuBqNa;K=Z8wXz8V2Fbh4(pjeZTz_m2H>Qq z?bqqEv-YADrU##wN;r2l@@FOr6#d9ZWYOarx32OocJLt(KOL9+%>t`Csg<*tZprOj zHhOo{x`67Rln&VNZd0u5(G=@F^_!T=f0^E3N>!v<3>^H2Hrf>jOuT+;IDa}kXOr^V zYx1cF??*t&di^7(UDrAC$FMiaave}Oy8E-jD2jNmnf!7sBO@c_G8xb=xDeH2I49Bx za|wSzssc6F&`~P==bjF0UoxYaJz9ix63dZQfrsMn*zK*L7z9~X*P~M`9Vo7^x;I59 zW)QT>W4xKJ?I2wM`AXXUg&TN5vh{Xy3BlCM>;>fEd}yAx(3m}S{QSW9doG`}eLG3b z)NV)@g&DOVTV)?@b`XjCPs*x`>hUX&^RMXo^jy4quO|^>?({+BvtVR5c2Glmq;Td@ z=wX9B7_aM(eQv&0%9??zOR4MT93yGw_^=a&kpEp6p3sptszBEemk9Tw{P<2!bLZ2D&WvT5%tAqmjA2 zH-2|$C*z?2kRNwsjy>*(HCxoNhM%N7Y6a%y!cpcekGP1)0_nAT^|vWT4b@HP6xPP+ z)z^-RodFb8&=qyX4B|pQnEQ_!+dFmx*UKkKYA8grUJ~mV?vV&$Oo*S@_2Of^;SJpm z9|&ywSCV+s)DGKU>J&ZbGnM8EO3q}TWAZBZl|$`;jWK{lN=LnZg#|lWah~E$V%pye zKNLrDjwDs#+5n|vBVe}F$*t*%ieub*dbi*XkE>m=LM8%K!=b9|s)L$nODfm(&I3*F zo2onaY=N2zj^nsGKX~UWZ+`$1!|V_0B+@F10_ZktvgD7F&b$wk#mL)^+X?M)p%I91 z??=~28NM>%BWJh{7%97@0!SUk_O+z;D1Cto-kz#7?l z;c(Cawb)`DIZ0U$%o&kGe8Z*zjx>y=YPdfn-8qshX?t`~!XJFj^Kh0aj(%r7pV^%g z5eO{HE**JLmL>TOg-y81``oixcCT+c?ztmtw{!6FzaNZ<;b8TPMMTp?_8?Gcw!!}n zK=*`k-ARL{5-YSHnDsJ=HjqDXV)49_z{zTXGE=i9rjEtEioMkFEdV7d7vAWi7O~LY zu<{gCM_t5LK=sZZ9!T&AAiHH3hWJ4iN4Md~K?+oM=!O2G`hiU%tQZkC&}Bokil1*h zA%CC(i3d^y7NLme)o)nRaGlmVq`n2Wwhfi6>G6f!deE#gOMtx}B>bc6?%`DT(idiR3|A^u3z$-bmV zd)V{gEb)Z*xI-65;SGORe=ahbj6C$;@quwdZ3=)LCMV(BPE)xVL_* z++_@E-$MTq8lKqeSL0=>v{9v=gX%kuHqw|zaYxBGO{r^v~#|kf9cSKxIX^Y zq3xXJrh}&U8ax>1k($lS3nGk-OFbIb`(5h`j*a0OM*?F%aY{TZciqpCy0S@YLDHv4Eiao$lhI~t$i(FId^R{`UK zco6welFvdAVz%~U+4h-((4dioU*jt#D{m!OP;rV|tC>K`X$e=(mUuB9DT_iAwMZ%> zAVMk&9QU4oKJ`d$Z9L8rdJlJdC|4&Odo(t?=wFY_U?2`CB)js$L3Z~U4moVu8KfIa41 zqt;#m3Bw`;>sw;#Fzv&@1fr(UbAq@nJ{?@MH$sG~k$!7)r$y_KXprsP`uhnXeuE5=mGRHKxA( zDN(#;ZIY!;M&FckG&}8ABru2Ku}Z+oPbDy&Opn7@RJU=qS73ZR+w zN-R!Z6gpIsE#TH%+5=Ty58MP7(u2A+*>9b-?Nd*I~q9!yW2N3!rNFCmvRU za|lqBxoYK3`QsmlhjpU0E;(7fz&$|7?SRRi8=^F5;=~$4+}z$%FYkncN{Gt zp%j)cG6ZwBZQWBl7=oc6o*%AWU)HQpbO#*`8wW+_ts~UY@+-_fC)q@={qiy`bGw6x z745+%A_j+oT)ZFhny6NerBVZ|uxn3t=MrQZhP5Dvi8I_k9EKLZTNimi015hn_UFal z8b%PATg+k&p1$^=Ifd1s5eLcm@M}n@3xzqHN~FGnmq-QZ?c}#2N9hrUqJ^C>wI8S! zrvtdh(owjPlnyx#eh#6?Pu1Sd7d$ z^SF1?_VmSK2X<`4O%i{4m!53))AFHHL=4(vALwsqzTU)Ngy)E@>-~)Qeu!ND+LcB1 zAnIcR`tbV!TM#cE1JpgC(PhyH7xMvK<~|~Pb?9oU%k6_WRuz|-Jf-0ai z^m;xI@=L-GV1cFS2Lao5?#BTd=87_i^x6>3I1AB4WH=Hneh(r`yk;^Cil)+khO3@u z;X-cSj9K)c0E+~T4C~qebM;%y`Uffxfl(Yq-Vc*+!`0Tx3s@L5LE}&dz7Jk5OGmY> z9l1~HA<92N0Tj5I;Fl<1<6?Tx32?}mH9FFNH=Wf7kT%?tU&(MOyrL zABYgQ9pEW(&Mjk};3ArjfO!9|m~+=dmd$r0y+LU48#}t*H6B6B7ZP?ULq)bKZ8Ww$ z+4={&9goMOHpEMdhiJum zy5&EGs2^UFgK6XC&i_#zRl`YeXb0lIaLamYDQV&pKT`{lP~`y6=)~45ChMx``zhD@ z3iI^HY4IB&z%f#!u60S)Ykr3s&8_g5f+KIPtfkZedoDsFw@_CGf7rIA`{0_3V+&k+ zoF3r2+#Jh!bM-}5-eJr`K2lyvs9-plnK9TX)#-Q0Rq>q)O?tZ|AB3UjK0M!<`TCpc zxZA^4wNjjk1E9s*-azYeb1x+u9p()>HU_}~TST7~XaQ!1kuLkWfh0jw1M+>$Y~bQi zv{eX}cb^tQzA3KH9&-lo2{!4ddqwu*whibtFj^6WMWl+L3bCNWV8ulUYgtFG@(U#U zpo@OMO606P-(7!)PB+rKp!jWy_P#qj9d5;n6gD@&CSd;VQ?P@X?{r}}qe&ZU0Ltmt zEMU8|@>aS}ULQkd19g*egdwO(jUD}jgs|ZIar-jmVRA;DrB#-6PCg8HF5K|7(XH^~ z%C)Q##&|1{wBM4JTmp|0%#Rffw@Yp#HPy><-I|?LEu)eeb*Us)%|`*CI1hK9E5yMw z#E;<(T!uko?DcJUGx6ja53c||j{B706G92|E%5$f6f|Me(*tZ;FaOszh*kM9lgcl# zTReohSZOg&kUj<>RAeNSEX{aiJ@002Pee)TfcTjW>T(XJ5O=7X%{;|sUmJ-wmSY31 zg>w~9Hc;3@$x!zzbgHFhJVwgh$`ymcXpZyLk zlgPzZ{+7bi^1f4&o*{J-!(UVc^9rOG7DV*tH_tP2q&0#0P!DO>+l|S)?&LDg#dchz zOqGxZq<@1TIvQd%9{WWrYjVN_$=04${tOYJ3}M(h5;fzvTL9${*2B&!V4#`*woI3o zD)gH3ON`(oJ83O#-E(q1EZcT~f;{%IOoMpDRI-!Us;aL^vliAg+V>r$+=juuho_bLpm@k*8SY9lZ$)eo9RkGq!)~4iw9Cqi z2>)*kF*!uS%vJ$ZHm~_XIj9;oSKV)8yy<411Rg1DRr>n^vdXvzB`4U@OBlXrktecq zaGel7F-`4ANr99@O(`YsvGBfIFxNO6Tw4#6Z@v>#t1oby24@cpCqeoLY}U2?EXOUL zUt7P@sIuCQ){)af6{Hse1#=S$R+p?n;iWUrbgWQe?TO_+UVCF~&q)6qo{OE^$g!CO z)xSwHg`cHuDAPRr+yeIfkaB<6;d|-&`r9LhG$>+1hNzHTDu<`&pH2?ByCNc%FBl%n z2DqAKg3;cL>NdcrKC@5*XYGaXYw(fO?RaZbvIY4oqBc3~sYJqdjt~)yFXrPiuBc?Y zb`nR=x-Xsxz#$cAfkzEcC!c_Nwn1&cwIFe8kS(A7EXdYg4_l7|B*jahbcDdiW?qr@NK6X&5PdC(`uS^n@gY^DmH~^F-!vddsexU zEU>4KAh0x>61s>5#TW(wX7m3;jRqf4OC#tl-gKXL-lgCZ(MeoE(wL2n?;T69RL;)y&&@8T1j$Q2`Lee_qbEGg8=O!ua8Sh!J#$T({aAjQ`lx~+SdJuQ(c;Aj!6d4rlw=Iy z|J6h=p^gB@1zn3hJ`NBc8KbwB3j^7|43t5GV{s069~!UAVs!5DA$nZ7gJ5|w=GdO< z!$Z!v>sB^SSEr4K8~F+5PczXRC2col{myB=<0n>vDlc;Yss=?6>-1R#tW8+;#P&ho z@AoGHKMJ~@cnFwRo**Ld&%lU0zS*o_XycQQ%l5T2g93vPLWmGW8@MCfli#2Vl*1D2d7`!oJk?l;3;k zauw^&*T9N`K(=VT{i`qS=isH&eGmfKPBpZjEYf&L%$PRMg{sLZsssEzWiYcQ076QM zxU!G}&lmZ`te{72HXu@E1?vILTKMW5eJl36R6j=HSC=<5QBmCp{Ikp&L4gHoF1dQB9D=v-+e}r{U z1vG`ckwdgTOSthj+4iF2^3Y31K6PhM(p^6A$d?KOH)qRmGNI) za01_*?J4BwCh&>d;^T0SXfDjjw&71eS65Taq?Q9UXPGE$>H7sH>ep%z1{VUU6u1s@ zqC#)z`tE5Z /dev/null && pwd ) + +[[ $- == *i* ]] && { + # Prompt if interactive + read -p "Are you sure you want to run the install script? (y)" -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]] + then + [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 # handle exits from shell or function but don't exit interactive shell + fi +} + +if [ -z "$SUDO_USER" ]; then + echo "Error: Invoke the script with sudo since script needs to install packages." + exit 1 +fi + +# For now this script is supported only on ubuntu + +CURR_DIR=$(pwd) +TMP_DIR="$( dirname -- "$0"; )"/.kraterdev/tmp +echo "Making temp directory: " $TMP_DIR +mkdir -p $TMP_DIR +cd $TMP_DIR + +echo "┌─────────────────────────────┐" +echo "│ INSTALL PACKAGES │" +echo "└─────────────────────────────┘" + +apt-get update + +# Install packages +apt-get install -y \ + jq \ + libssl-dev \ + gettext-base \ + uuid \ + ca-certificates curl \ + clang \ + cmake \ + zlib1g-dev \ + libboost-dev \ + libboost-thread-dev + +echo "┌─────────────────────────────┐" +echo "│ INSTALL KUBECTL │" +echo "└─────────────────────────────┘" + +kubectl version --client=true >/dev/null 2>&1 || { + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + curl -LO "https://dl.k8s.io/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl.sha256" + echo "$(cat kubectl.sha256) kubectl" | sha256sum --check + install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl +} + +kubectl version --client=true -o json 2>/dev/null + +# Installing kind +echo "┌─────────────────────────────┐" +echo "│ INSTALL KIND │" +echo "└─────────────────────────────┘" + +kind --version >/dev/null 2>&1 || { + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.17.0/kind-linux-amd64 + chmod +x ./kind + mv ./kind /usr/local/bin/kind +} + +kind --version + +kubectl version --client=true -o json 2>/dev/null + +echo "┌─────────────────────────────┐" +echo "│ INSTALL KUBELOGIN │" +echo "└─────────────────────────────┘" + +az aks install-cli + +cd $CURR_DIR +echo "Deleting " $TMP_DIR +rm -rf $TMP_DIR + +echo +echo "Excellent! Your prerequisites are installed 👍" +echo \ No newline at end of file diff --git a/build/ci/scripts/kind.sh b/build/ci/scripts/kind.sh new file mode 100644 index 0000000..3d39223 --- /dev/null +++ b/build/ci/scripts/kind.sh @@ -0,0 +1,391 @@ +#!/bin/bash +set -e + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +KIND_CLUSTER_NAME="p2p" +KIND_CLUSTER_CONTEXT="kind-$KIND_CLUSTER_NAME" +CLUSTER_CONFIG_FILE="$SCRIPT_DIR/../k8s/kind-cluster.yml" +APP_DEPLOYMENT_FILE="$SCRIPT_DIR/../k8s/app.yml" +RBAC_DEPLOYMENT_FILE="$SCRIPT_DIR/../k8s/rbac.yml" +export GIT_ROOT="$(git rev-parse --show-toplevel)" + +# Console colors and helpers +BG_BLUE="\e[44m" +GREEN="\e[32m" +YELLOW="\e[33m" +COLOR_RESET="\\033[0m" +NC='\033[0m' # No Color +RED='\e[01;31m' + +echo_header() { + echo -e "${BG_BLUE}=> $@ ${NC}" +} + +indent() { + sed 's/^/ /' +} + +pipe_indent() { + sed 's/^/│ /' +} + +close_pipe_indent() { + sed 's/^/└─ /' +} + +print_and_exit_if_dry_run() { + if [ "$DRY_RUN" == "true" ]; then + echo + echo + echo "DRY RUN SUCCESSFUL: to confirm execution, re-run script with '-y'" + exit 0 + fi +} + +show_help() { + usageStr=" +Usage: $(basename $0) [OPTIONS] + +This script is used for deploying apps to a local kind cluster for testing purposes. + +Options: + -h Show help + -y Confirm execution, otherwise, it's a dry-run + +Sub commands: + cluster + get + create + delete + + app + deploy + +* dry run: create new local environment + $(basename $0) cluster create +* confirm: create new local environment + $(basename $0) cluster create -y + +* dry run: deploy app + $(basename $0) app deploy +* confirm: deploy app + $(basename $0) app deploy -y +" + echo "$usageStr" +} + +validate_prerequisites() { + if ! get_prerequisites_versions ; then + echo "You can also install prerequisites with:" + echo + echo make deps-install + echo + exit -1 + fi +} + +# Validate tools available +get_prerequisites_versions() { + local ec=1 + jq --version >/dev/null 2>&1 || { + echo "jq not found: to install, try 'apt install jq'" + return $ec + } + + kubectl version --client=true >/dev/null 2>&1 || { + echo "kubectl not found: see https://kubernetes.io/docs/tasks/tools/" + return $ec + } + + envsubst --version >/dev/null 2>&1 || { + echo "envsubst not found: to install, try 'apt-get install gettext-base'" + return $ec + } + + kind --version >/dev/null 2>&1 || { + echo "kind not found: see https://kind.sigs.k8s.io/docs/user/quick-start/#installing-from-release-binaries" + return $ec + } +} + +validate_params() { + local ec=2 + if [[ "$DRY_RUN" != "true" ]] && [[ "$DRY_RUN" != "false" ]]; then + show_help + echo "ERROR: dry run parameter invalid, expect true or false" + exit $ec + fi +} + +print_p2p_metrics() { + p=$(kubectl --context=$KIND_CLUSTER_CONTEXT -n peerd-ns get pods -l app=peerd -o jsonpath='{.items[*].metadata.name}') + echo "pods: $p" + + for pod in $( echo "$p" | tr -s " " "\012" ); do + echo "checking pod '$pod' for metrics" + kubectl --context=$KIND_CLUSTER_CONTEXT -n peerd-ns exec -i $pod -- bash -c "cat /var/log/peerdmetrics" + done +} + +show_cluster_info() { + echo_header "Cluster Info for $KIND_CLUSTER_CONTEXT" + kind get clusters | grep $KIND_CLUSTER_NAME 1>/dev/null + if [ $? -ne 0 ]; then + echo "Kind cluster not found" + fi + kubectl cluster-info --context=$KIND_CLUSTER_CONTEXT + echo_header "Services" + kubectl get services --all-namespaces +} + +create_cluster() { + echo_header "New cluster requested" + if [ "$DRY_RUN" == "false" ]; then + if [ $(kind get clusters | grep $KIND_CLUSTER_NAME) ]; then + echo "Cannot create cluster since it $KIND_CLUSTER_CONTEXT already exists" + exit 1 + fi + envsubst < $CLUSTER_CONFIG_FILE | kind create cluster --config - + echo + + ns="peerd-ns" + echo "creating namespace $ns" && \ + kubectl --context=$KIND_CLUSTER_CONTEXT create namespace $ns + + # ns="peerd-tests" + # echo "creating namespace $ns" && \ + # kubectl --context=$KIND_CLUSTER_CONTEXT create namespace $ns + + echo "creating service account" && \ + kubectl --context=$KIND_CLUSTER_CONTEXT apply -f $RBAC_DEPLOYMENT_FILE + echo + fi +} + +delete_cluster() { + echo_header "Deleting kind cluster: $KIND_CLUSTER_NAME" + if [ "$DRY_RUN" == "false" ]; then + kind delete cluster -n $KIND_CLUSTER_NAME + fi +} + +wait_for_events() { + local context=$1 + local event=$2 + local minimumRequired=$3 + + local ns="peerd-ns" + local found=0 + + # # Get app pods + pods=$(kubectl --context=$context -n $ns get pods -o jsonpath='{.items[*].metadata.name}') + echo "pods: $pods" + total=`echo "$pods" | tr -s " " "\012" | wc -l` + + if [ -z "$minimumRequired" ]; then + minimumRequired=$total + fi + + # # Loop until all pods have the event or an error occurs. + for ((i=1; i<=10; i++)); do + found=0 + for pod in $( echo "$pods" | tr -s " " "\012" ); do + echo "checking pod '$pod' for event '$event'" + + foundEvent=$(kubectl --context=$context get events --field-selector involvedObject.kind=Pod,involvedObject.name=$pod -o json | jq -r ".items[] | select(.reason == \"$event\")") + [[ "$foundEvent" == "" ]] && echo "Event '$event' not found for pod '$pod'" || found=$((found+1)) + + errorEvent=$(kubectl --context=$context get events --field-selector involvedObject.kind=Pod,involvedObject.name=$pod -o json | jq -r '.items[] | select(.reason == "P2PDisconnected" or .resosn == "P2PFailed")') + [[ "$errorEvent" == "" ]] || (echo "Error event found for pod '$pod': $errorEvent" && exit 1) + done + + if [ $found -eq $total ]; then + echo "Success: All pods have event '$event'." + break + else + echo "Waiting: $found out of $total pods have event '$event'. Attempt $i of 10." + sleep 15 + fi + done + + if [ $found -eq $total ]; then + return + elif [ $found -ge $minimumRequired ]; then + echo "Warning: only $found out of $total pods have event '$event', but it meets the minimum criteria of $minimumRequired." + return + else + echo "Validation failed" + exit 1 + fi +} + +cmd__test__ctr() { + local context=$KIND_CLUSTER_CONTEXT + local img="mcr.microsoft.com/hello-world:latest" + + echo "initializing test 'ctr': pulling image '$img'" + + if [ "$DRY_RUN" == "true" ]; then + echo "[dry run] would have initialized test 'ctr'" + else + # Get nodes + nodes=$(kubectl --context=$context get nodes -o jsonpath='{.items[*].metadata.name}') + echo "nodes: $nodes" + total=`echo "$nodes" | tr -s " " "\012" | wc -l` + + # Pull the image on all nodes and verify that at least one P2PActive event is generated. + for node in $( echo "$nodes" | tr -s " " "\012" ); do + echo "pulling image '$img' on node '$node'" && \ + docker exec $node bash -c "ctr -n k8s.io images pull --hosts-dir '/etc/containerd/certs.d' $img" && + sleep 3 + done + + wait_for_events $context "P2PActive" 1 + fi + + echo "fetching metrics from pods" + print_p2p_metrics +} + +cmd__test__random() { + local img=$1 + + echo "test image: $img" + echo "initializing test 'random'" + + totalRequested=15 + ctr=0 + + repos="oss/kubernetes/kubectl oss/kubernetes/kube-proxy oss/kubernetes/tiller oss/kubernetes/kube-api-server" + + echo "collecting $totalRequested secrets from mcr.microsoft.com repos: $repos" + + for repo in $repos; do + if [ $ctr -ge $totalRequested ]; then + break + fi + + tags=$(curl https://mcr.microsoft.com/v2/$repo/tags/list 2> /dev/null | jq -r ".tags[]") + + for tag in $tags; do + if [ $ctr -ge $totalRequested ]; then + break + fi + + manifest=`curl -s -H "Accept: application/vnd.oci.image.manifest.v1+json" -H "Accept: application/vnd.docker.distribution.manifest.v2+json" https://mcr.microsoft.com/v2/$repo/manifests/$tag` + layers=`echo $manifest | jq -r ".layers[].digest"` + + for layer in $layers; do + sas=`curl -v -s -H "Accept: application/vnd.oci.image.manifest.v1+json" -H "Accept: application/vnd.docker.distribution.manifest.v2+json" https://mcr.microsoft.com/v2/$repo/blobs/$layer 2>&1 | grep "< location: " | awk -F ': ' '{print $2}'` + secret="$secret $sas" + ctr=$((ctr+1)) + done + + done + + done + + if [ "$DRY_RUN" == "true" ]; then + echo "[dry run] collected $ctr secrets" + else + echo "collected $ctr secrets" + fi + + echo "loading test image and deploying" + if [ "$DRY_RUN" == "false" ]; then + kind load docker-image $img --name $KIND_CLUSTER_NAME + export TEST_RANDOM_CONTAINER_IMAGE=$img + export SECRETS=$secret + export NODE_COUNT=3 + envsubst < $SCRIPT_DIR/../k8s/test-random.yml | kubectl --context=$KIND_CLUSTER_CONTEXT apply -f - + + echo "waiting for logs" && \ + sleep 10 && \ + kubectl --context=$KIND_CLUSTER_CONTEXT -n peerd-ns logs -l app=random -f + + kubectl --context=$KIND_CLUSTER_CONTEXT -n peerd-ns delete ds/random + + echo "checking p2p active pods" && sleep 2 + wait_for_events $KIND_CLUSTER_CONTEXT "P2PActive" 1 + fi + + echo "fetching metrics from pods" + print_p2p_metrics +} + +cmd__cluster__get() { + show_cluster_info +} + +cmd__cluster__create() { + create_cluster + print_and_exit_if_dry_run + echo + echo "Hooray! Local cluster is available 🙌🐳" + echo + exit 0 +} + +cmd__cluster__delete() { + delete_cluster +} + +cmd__app__deploy() { + export P2P_CONTAINER_IMAGE=$1 + + kubectl cluster-info --context=$KIND_CLUSTER_CONTEXT && \ + echo_header "Deploying app: $P2P_CONTAINER_IMAGE" + + if [ "$DRY_RUN" == "false" ]; then + echo "loading image" + kind load docker-image $P2P_CONTAINER_IMAGE --name $KIND_CLUSTER_NAME + envsubst < $APP_DEPLOYMENT_FILE | kubectl --context=$KIND_CLUSTER_CONTEXT apply -f - + + echo "waiting for pods to connect" + wait_for_events $KIND_CLUSTER_CONTEXT "P2PConnected" 3 + fi +} + +get_opts() { + while getopts 'yh' OPTION; do + case "$OPTION" in + y) + DRY_RUN="false" + ;; + h) + show_help + exit 1 # exit non-zero to break invocation of command + ;; + esac + done + shift $((OPTIND-1)) +} + +# Initialize script and validate prerequisites. +if [[ -z "$DRY_RUN" ]]; then + DRY_RUN="true" +fi + +validate_params +validate_prerequisites + +# Check sub command then check fall through to +# main command if sub command doesn't exist +# functions that are entry points should be of the form +# cmd__{command}__{subcommand} or cmd__{command} +if declare -f "cmd__${1}__${2}" >/dev/null; then + func="cmd__${1}__${2}" + shift; shift; + get_opts $@ + # pop $1 $2 off the argument list + "$func" "$2" # invoke our named function w/ all remaining arguments +elif declare -f "cmd__$1" >/dev/null; then + func="cmd__$1" + shift; # pop $1 off the argument list + get_opts $@ + "$func" "$@" # invoke our named function w/ all remaining arguments +else + echo "Neither command $1 nor subcommand ${1} ${2} recognized" >&2 + show_help + exit 1 +fi diff --git a/build/package/Dockerfile b/build/package/Dockerfile new file mode 100644 index 0000000..dda79f3 --- /dev/null +++ b/build/package/Dockerfile @@ -0,0 +1,28 @@ +FROM mcr.microsoft.com/oss/go/microsoft/golang:1.22-fips-cbl-mariner2.0 as builder + +COPY ./ / + +RUN tdnf install make -y && \ + tdnf install git -y + +WORKDIR / + +RUN make install + +FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 as runtime + +ARG USER_ID=6191 + +# Install useful packages +RUN tdnf update -y && \ + tdnf install ca-certificates-microsoft -y && \ + tdnf install shadow-utils -y && \ + tdnf install net-tools -y && \ + tdnf install tcpdump -y + +RUN groupadd -g $USER_ID peerd && \ + useradd -g peerd -u $USER_ID peerd + +COPY --from=builder --chown=peerd:root /bin/ /bin/ + +ENTRYPOINT ["/bin/peerd", "run"] diff --git a/cmd/proxy/cmd.go b/cmd/proxy/cmd.go new file mode 100644 index 0000000..960e5cc --- /dev/null +++ b/cmd/proxy/cmd.go @@ -0,0 +1,16 @@ +package main + +type ServerCmd struct { + HttpAddr string `arg:"--http-addr" help:"address of the server" default:"127.0.0.1:5000"` + HttpsAddr string `arg:"--https-addr" help:"address of the server" default:"0.0.0.0:5001"` + RouterAddr string `arg:"--router-addr" help:"address of the router (p2p)" default:"0.0.0.0:5003"` + PrefetchWorkers int `arg:"--prefetch-workers" help:"number of workers to prefetch content" default:"50"` +} + +type Arguments struct { + Server *ServerCmd `arg:"subcommand:run" help:"run the server"` + Version bool `arg:"-v" help:"show version and exit"` + LogLevel string `arg:"--log-level" help:"set the log level" default:"info" valid:"debug,info,warn,error,fatal,panic"` +} + +var version string diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go new file mode 100644 index 0000000..79ff834 --- /dev/null +++ b/cmd/proxy/main.go @@ -0,0 +1,166 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/alexflint/go-arg" + p2pcontext "github.com/azure/peerd/internal/context" + "github.com/azure/peerd/internal/files/store" + "github.com/azure/peerd/internal/handlers" + "github.com/azure/peerd/internal/k8s/events" + "github.com/azure/peerd/internal/routing" + "github.com/azure/peerd/internal/state" + "github.com/azure/peerd/pkg/containerd" + "github.com/rs/zerolog" + "golang.org/x/sync/errgroup" +) + +func main() { + args := &Arguments{} + arg.MustParse(args) + + ll, err := zerolog.ParseLevel(args.LogLevel) + if err != nil { + fmt.Fprintf(os.Stderr, "invalid log level: %s\n", args.LogLevel) + os.Exit(1) + } + + zerolog.SetGlobalLevel(ll) + zerolog.TimeFieldFormat = time.RFC3339Nano + + l := zerolog.New(os.Stdout).With().Timestamp().Str("self", p2pcontext.NodeName).Str("version", version).Logger() + ctx := l.WithContext(context.Background()) + + err = run(ctx, args) + if err != nil { + l.Error().Err(err).Msg("server error") + os.Exit(1) + } + + l.Info().Msg("server shutdown") +} + +func run(ctx context.Context, args *Arguments) error { + ctx, cancel := signal.NotifyContext(ctx, syscall.SIGTERM) + defer cancel() + + switch { + case args.Version: + zerolog.Ctx(ctx).Info().Msg("version") // version field is already added to the logger + return nil + case args.Server != nil: + return serverCommand(ctx, args.Server) + default: + return fmt.Errorf("unknown subcommand") + } +} + +func serverCommand(ctx context.Context, args *ServerCmd) (err error) { + l := zerolog.Ctx(ctx) + + store.PrefetchWorkers = args.PrefetchWorkers + + _, httpsPort, err := net.SplitHostPort(args.HttpsAddr) + if err != nil { + return err + } + + ctx, err = events.WithContext(ctx) + if err != nil { + return err + } + eventsRecorder := events.FromContext(ctx) + defer func() { + if err != nil { + eventsRecorder.Failed() + } + }() + + eventsRecorder.Initializing() + + r, err := routing.NewRouter(ctx, args.RouterAddr, httpsPort) + if err != nil { + return err + } + + containerdStore, err := containerd.NewDefaultStore([]string{"mcr.microsoft.com"}) + if err != nil { + return err + } + err = containerdStore.Verify(ctx) + if err != nil { + return err + } + + filesStore, err := store.NewFilesStore(ctx, r) + if err != nil { + return err + } + + g, ctx := errgroup.WithContext(ctx) + + g.Go(func() error { + state.Advertise(ctx, r, containerdStore, filesStore.Subscribe()) + return nil + }) + + handler, err := handlers.Handler(ctx, r, containerdStore, filesStore) + if err != nil { + return err + } + + httpsSrv := &http.Server{ + Addr: args.HttpsAddr, + Handler: handler, + TLSConfig: r.Net().DefaultTLSConfig(), + } + + g.Go(func() error { + if err := httpsSrv.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil + }) + + httpSrv := &http.Server{ + Addr: args.HttpAddr, + Handler: handler, + } + + g.Go(func() error { + if err := httpSrv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil + }) + + g.Go(func() error { + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + return httpsSrv.Shutdown(shutdownCtx) + }) + + g.Go(func() error { + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + return httpSrv.Shutdown(shutdownCtx) + }) + + l.Info().Str("https", args.HttpsAddr).Str("http", args.HttpAddr).Msg("server start") + err = g.Wait() + if err != nil { + return err + } + + return nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7d3ab0b --- /dev/null +++ b/go.mod @@ -0,0 +1,210 @@ +module github.com/azure/peerd + +go 1.22 + +require ( + github.com/alexflint/go-arg v1.4.3 + github.com/containerd/typeurl/v2 v2.1.1 + github.com/dgraph-io/ristretto v0.1.1 + github.com/distribution/distribution v2.8.3+incompatible + github.com/google/uuid v1.6.0 + github.com/libp2p/go-libp2p v0.33.0 + github.com/libp2p/go-libp2p-kad-dht v0.25.2 + github.com/multiformats/go-multiaddr v0.12.2 + github.com/opencontainers/go-digest v1.0.0 + github.com/opencontainers/image-spec v1.1.0 + github.com/pelletier/go-toml/v2 v2.1.1 + github.com/rs/zerolog v1.32.0 + github.com/schollz/progressbar/v3 v3.14.2 + github.com/swaggo/swag v1.16.3 + golang.org/x/sync v0.6.0 + k8s.io/api v0.29.2 + k8s.io/apimachinery v0.29.2 +) + +require ( + github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect + github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect + github.com/KyleBanks/depth v1.2.1 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Microsoft/hcsshim v0.12.0 // indirect + github.com/alexflint/go-scalar v1.2.0 // indirect + github.com/benbjohnson/clock v1.3.5 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/bytedance/sonic v1.11.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect + github.com/chenzhuoyu/iasm v0.9.1 // indirect + github.com/containerd/cgroups v1.1.0 // indirect + github.com/containerd/cgroups/v3 v3.0.3 // indirect + github.com/containerd/continuity v0.4.3 // indirect + github.com/containerd/errdefs v0.1.0 // indirect + github.com/containerd/fifo v1.1.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/ttrpc v1.2.3 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/elastic/gosigar v0.14.2 // indirect + github.com/emicklei/go-restful/v3 v3.11.3 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/flynn/noise v1.1.0 // indirect + github.com/francoispqt/gojay v1.2.13 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/jsonpointer v0.20.3 // indirect + github.com/go-openapi/jsonreference v0.20.5 // indirect + github.com/go-openapi/spec v0.20.4 // indirect + github.com/go-openapi/swag v0.22.10 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.19.0 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v1.2.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/gopacket v1.1.19 // indirect + github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect + github.com/gorilla/websocket v1.5.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/huin/goupnp v1.3.0 // indirect + github.com/imdario/mergo v0.3.16 // indirect + github.com/ipfs/boxo v0.10.0 // indirect + github.com/ipfs/go-datastore v0.6.0 // indirect + github.com/ipfs/go-log v1.0.5 // indirect + github.com/ipld/go-ipld-prime v0.20.0 // indirect + github.com/jackpal/go-nat-pmp v1.0.2 // indirect + github.com/jbenet/goprocess v0.1.4 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.7 // indirect + github.com/koron/go-ssdp v0.0.4 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/libp2p/go-cidranger v1.1.0 // indirect + github.com/libp2p/go-flow-metrics v0.1.0 // indirect + github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect + github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect + github.com/libp2p/go-libp2p-record v0.2.0 // indirect + github.com/libp2p/go-libp2p-routing-helpers v0.7.2 // indirect + github.com/libp2p/go-msgio v0.3.0 // indirect + github.com/libp2p/go-nat v0.2.0 // indirect + github.com/libp2p/go-netroute v0.2.1 // indirect + github.com/libp2p/go-reuseport v0.4.0 // indirect + github.com/libp2p/go-yamux/v4 v4.0.1 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/miekg/dns v1.1.58 // indirect + github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect + github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/moby/locker v1.0.1 // indirect + github.com/moby/sys/mountinfo v0.7.1 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/signal v0.7.0 // indirect + github.com/moby/sys/user v0.1.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect + github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/onsi/ginkgo/v2 v2.15.0 // indirect + github.com/opencontainers/runtime-spec v1.2.0 // indirect + github.com/opencontainers/selinux v1.11.0 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/polydawn/refmt v0.89.0 // indirect + github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect + github.com/prometheus/common v0.47.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/quic-go/qpack v0.4.0 // indirect + github.com/quic-go/quic-go v0.41.0 // indirect + github.com/quic-go/webtransport-go v0.6.0 // indirect + github.com/raulk/go-watchdog v1.3.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + go.uber.org/dig v1.17.1 // indirect + go.uber.org/fx v1.20.1 // indirect + go.uber.org/mock v0.4.0 // indirect + golang.org/x/arch v0.7.0 // indirect + golang.org/x/mod v0.16.0 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.19.0 // indirect + gonum.org/v1/gonum v0.13.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240304212257-790db918fca8 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240304212257-790db918fca8 // indirect + google.golang.org/grpc v1.62.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20240102154912-e7106e64919e // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) + +require ( + github.com/containerd/containerd v1.7.13 + github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect + github.com/gin-gonic/gin v1.9.1 + github.com/hashicorp/go-metrics v0.5.3 + github.com/ipfs/go-cid v0.4.1 + github.com/ipfs/go-log/v2 v2.5.1 // indirect + github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/libp2p/go-buffer-pool v0.1.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/mr-tron/base58 v1.2.0 // indirect + github.com/multiformats/go-base32 v0.1.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multicodec v0.9.0 + github.com/multiformats/go-multihash v0.2.3 + github.com/multiformats/go-multistream v0.5.0 // indirect + github.com/multiformats/go-varint v0.0.7 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/spf13/afero v1.11.0 + github.com/stretchr/testify v1.9.0 + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect + golang.org/x/sys v0.18.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect + k8s.io/client-go v0.29.2 + lukechampine.com/blake3 v1.2.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..2d56b9f --- /dev/null +++ b/go.sum @@ -0,0 +1,872 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= +dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= +dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= +dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= +dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= +git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 h1:dIScnXFlF784X79oi7MzVT6GWqr/W1uUt0pB5CsDs9M= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2/go.mod h1:gCLVsLfv1egrcZu+GoJATN5ts75F2s62ih/457eWzOw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= +github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/hcsshim v0.12.0 h1:rbICA+XZFwrBef2Odk++0LjFvClNCJGRK+fsrP254Ts= +github.com/Microsoft/hcsshim v0.12.0/go.mod h1:RZV12pcHCXQ42XnlQ3pz6FZfmrC1C+R4gaOHhRNML1g= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alexflint/go-arg v1.4.3 h1:9rwwEBpMXfKQKceuZfYcwuc/7YY7tWJbFsgG5cAU/uo= +github.com/alexflint/go-arg v1.4.3/go.mod h1:3PZ/wp/8HuqRZMUUgu7I+e1qcpUbvmS258mRXkFH4IA= +github.com/alexflint/go-scalar v1.1.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o= +github.com/alexflint/go-scalar v1.2.0 h1:WR7JPKkeNpnYIOfHRa7ivM21aWAdHD0gEWHCx+WQBRw= +github.com/alexflint/go-scalar v1.2.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= +github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= +github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A= +github.com/bytedance/sonic v1.11.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= +github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= +github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= +github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0= +github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= +github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= +github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= +github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= +github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= +github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= +github.com/containerd/containerd v1.7.13 h1:wPYKIeGMN8vaggSKuV1X0wZulpMz4CrgEsZdaCyB6Is= +github.com/containerd/containerd v1.7.13/go.mod h1:zT3up6yTRfEUa6+GsITYIJNgSVL9NQ4x4h1RPzk0Wu4= +github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= +github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM= +github.com/containerd/errdefs v0.1.0/go.mod h1:YgWiiHtLmSeBrvpw+UfPijzbLaB77mEG1WwJTDETIV0= +github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= +github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/ttrpc v1.2.3 h1:4jlhbXIGvijRtNC8F/5CpuJZ7yKOBFGFOOXg1bkISz0= +github.com/containerd/ttrpc v1.2.3/go.mod h1:ieWsXucbb8Mj9PH0rXCw1i8IunRbbAiDkpXkbfflWBM= +github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= +github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= +github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= +github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= +github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/distribution/distribution v2.8.3+incompatible h1:RlpEXBLq/WPXYvBYMDAmBX/SnhD67qwtvW/DzKc8pAo= +github.com/distribution/distribution v2.8.3+incompatible/go.mod h1:EgLm2NgWtdKgzF9NpMzUKgzmR7AMmb0VQi2B+ZzDRjc= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= +github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= +github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= +github.com/emicklei/go-restful/v3 v3.11.3 h1:yagOQz/38xJmcNeZJtrUcKjkHRltIaIFXKWeG1SkWGE= +github.com/emicklei/go-restful/v3 v3.11.3/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= +github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= +github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= +github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.20.3 h1:jykzYWS/kyGtsHfRt6aV8JTB9pcQAXPIA7qlZ5aRlyk= +github.com/go-openapi/jsonpointer v0.20.3/go.mod h1:c7l0rjoouAuIxCm8v/JWKRgMjDG/+/7UBWsXMrv6PsM= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/jsonreference v0.20.5 h1:hutI+cQI+HbSQaIGSfsBsYI0pHk+CATf8Fk5gCSj0yI= +github.com/go-openapi/jsonreference v0.20.5/go.mod h1:thAqAp31UABtI+FQGKAQfmv7DbFpKNUlva2UPCxKu2Y= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.22.10 h1:4y86NVn7Z2yYd6pfS4Z+Nyh3aAUL3Nul+LMbhFKy0gA= +github.com/go-openapi/swag v0.22.10/go.mod h1:Cnn8BYtRlx6BNE3DPN86f/xkapGIcLWzh3CLEb4C1jI= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= +github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 h1:E/LAvt58di64hlYjx7AsNS6C/ysHWYo+2qPCZKTQhRo= +github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-metrics v0.5.3 h1:M5uADWMOGCTUNU1YuC4hfknOeHNaX54LDm4oYSucoNE= +github.com/hashicorp/go-metrics v0.5.3/go.mod h1:KEjodfebIOuBYSAe/bHTm+HChmKSxAOXPBieMLYozDE= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= +github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= +github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= +github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= +github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= +github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= +github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= +github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= +github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= +github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= +github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= +github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= +github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= +github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= +github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= +github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= +github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= +github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= +github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= +github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= +github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= +github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= +github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= +github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= +github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= +github.com/libp2p/go-libp2p v0.33.0 h1:yTPSr8sJRbfeEYXyeN8VPVSlTlFjtMUwGDRniwaf/xQ= +github.com/libp2p/go-libp2p v0.33.0/go.mod h1:RIJFRQVUBKy82dnW7J5f1homqqv6NcsDJAl3e7CRGfE= +github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= +github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= +github.com/libp2p/go-libp2p-kad-dht v0.25.2 h1:FOIk9gHoe4YRWXTu8SY9Z1d0RILol0TrtApsMDPjAVQ= +github.com/libp2p/go-libp2p-kad-dht v0.25.2/go.mod h1:6za56ncRHYXX4Nc2vn8z7CZK0P4QiMcrn77acKLM2Oo= +github.com/libp2p/go-libp2p-kbucket v0.6.3 h1:p507271wWzpy2f1XxPzCQG9NiN6R6lHL9GiSErbQQo0= +github.com/libp2p/go-libp2p-kbucket v0.6.3/go.mod h1:RCseT7AH6eJWxxk2ol03xtP9pEHetYSPXOaJnOiD8i0= +github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= +github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= +github.com/libp2p/go-libp2p-routing-helpers v0.7.2 h1:xJMFyhQ3Iuqnk9Q2dYE1eUTzsah7NLw3Qs2zjUV78T0= +github.com/libp2p/go-libp2p-routing-helpers v0.7.2/go.mod h1:cN4mJAD/7zfPKXBcs9ze31JGYAZgzdABEm+q/hkswb8= +github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= +github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= +github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= +github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= +github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk= +github.com/libp2p/go-nat v0.2.0/go.mod h1:3MJr+GRpRkyT65EpVPBstXLvOlAPzUVlG6Pwg9ohLJk= +github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= +github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= +github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= +github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= +github.com/libp2p/go-yamux/v4 v4.0.1 h1:FfDR4S1wj6Bw2Pqbc8Uz7pCxeRBPbwsBbEdfwiCypkQ= +github.com/libp2p/go-yamux/v4 v4.0.1/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= +github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= +github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4= +github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY= +github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= +github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= +github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= +github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= +github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= +github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= +github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI= +github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= +github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= +github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= +github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= +github.com/multiformats/go-multiaddr v0.12.2 h1:9G9sTY/wCYajKa9lyfWPmpZAwe6oV+Wb1zcmMS1HG24= +github.com/multiformats/go-multiaddr v0.12.2/go.mod h1:GKyaTYjZRdcUhyOetrxTk9z0cW+jA/YrnqTOvKgi44M= +github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= +github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= +github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= +github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= +github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= +github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= +github.com/multiformats/go-multistream v0.5.0 h1:5htLSLl7lvJk3xx3qT/8Zm9J4K8vEOf/QGkvOGQAyiE= +github.com/multiformats/go-multistream v0.5.0/go.mod h1:n6tMZiwiP2wUsR8DgfDWw1dydlEqV3l6N3/GBsX6ILA= +github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= +github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= +github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= +github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= +github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= +github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= +github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= +github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.47.0 h1:p5Cz0FNHo7SnWOmWmoRozVcjEp0bIVU8cV7OShpjL1k= +github.com/prometheus/common v0.47.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= +github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= +github.com/quic-go/quic-go v0.41.0 h1:aD8MmHfgqTURWNJy48IYFg2OnxwHT3JL7ahGs73lb4k= +github.com/quic-go/quic-go v0.41.0/go.mod h1:qCkNjqczPEvgsOnxZ0eCD14lv+B2LHlFAB++CNOh9hA= +github.com/quic-go/webtransport-go v0.6.0 h1:CvNsKqc4W2HljHJnoT+rMmbRJybShZ0YPFDD3NxaZLY= +github.com/quic-go/webtransport-go v0.6.0/go.mod h1:9KjU4AEBqEQidGHNDkZrb8CAa1abRaosM2yGOyiikEc= +github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= +github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= +github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/schollz/progressbar/v3 v3.14.2 h1:EducH6uNLIWsr560zSV1KrTeUb/wZGAHqyMFIEa99ks= +github.com/schollz/progressbar/v3 v3.14.2/go.mod h1:aQAZQnhF4JGFtRJiw/eobaXpsqpVQAftEQ+hLGXaRc4= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= +github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= +github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= +github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= +github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= +github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= +github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= +github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= +github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= +github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= +github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= +github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= +github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= +github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= +github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= +github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= +github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= +github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= +github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= +github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= +github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= +github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= +github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/dig v1.17.1 h1:Tga8Lz8PcYNsWsyHMZ1Vm0OQOUaJNDyvPImgbAu9YSc= +go.uber.org/dig v1.17.1/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= +go.uber.org/fx v1.20.1 h1:zVwVQGS8zYvhh9Xxcu4w1M6ESyeMzebzj2NbSayZ4Mk= +go.uber.org/fx v1.20.1/go.mod h1:iSYNbHf2y55acNCwCXKx7LbWb5WG1Bnue5RDXz1OREg= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= +golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM= +gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20240304212257-790db918fca8 h1:Fe8QycXyEd9mJgnwB9kmw00WgB43eQ/xYO5C6gceybQ= +google.golang.org/genproto v0.0.0-20240304212257-790db918fca8/go.mod h1:yA7a1bW1kwl459Ol0m0lV4hLTfrL/7Bkk4Mj2Ir1mWI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240304212257-790db918fca8 h1:IR+hp6ypxjH24bkMfEJ0yHR21+gwPWdV+/IBrPQyn3k= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240304212257-790db918fca8/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= +k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= +k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= +k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= +k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= +k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/init/systemd/peerd.service b/init/systemd/peerd.service new file mode 100644 index 0000000..43835d5 --- /dev/null +++ b/init/systemd/peerd.service @@ -0,0 +1,12 @@ +[Unit] +Description=Peer to peer distribution service +After=network.target + +[Service] +Type=simple +ExecStart=/opt/peerd/bin/peerd run +Restart=always +StandardOutput=journal + +[Install] +WantedBy=multi-user.target diff --git a/internal/cache/syncmap.go b/internal/cache/syncmap.go new file mode 100644 index 0000000..bc1596d --- /dev/null +++ b/internal/cache/syncmap.go @@ -0,0 +1,71 @@ +package cache + +import ( + "sync" +) + +const defaultEvictionPercentage int = 5 //The default eviction percentage used when map reaches its capacity at insertion + +// SyncMap is a map with synchronized access support +type SyncMap struct { + mapObj *map[string]interface{} + lock *sync.RWMutex + capacity int + evictionPercentage int +} + +// Get retrieves the value associated with the given key from the SyncMap. +// It returns the value and a boolean indicating whether the key was found. +func (sm *SyncMap) Get(key string) (entry interface{}, ok bool) { + sm.lock.RLock() + defer sm.lock.RUnlock() + entry, ok = (*sm.mapObj)[key] + return +} + +// Set sets a new entry or updates an existing one. +// Set adds or updates an entry in the SyncMap with the specified key. +// If the key already exists in the map, the entry will be updated. +// If the key does not exist and the map is at capacity, some entries will be evicted first. +func (sm *SyncMap) Set(key string, entry interface{}) { + sm.lock.Lock() + defer sm.lock.Unlock() + if _, ok := (*sm.mapObj)[key]; !ok { //We will need to add an entry + if numEntries := len(*sm.mapObj); numEntries >= sm.capacity { //exceeding capacity, remove evictionPercentage of the entries + numToEvict := numEntries * sm.evictionPercentage / 100 + if numToEvict <= 1 { //We will evict one as the minimum + numToEvict = 1 + } + numEvicted := 0 + for k := range *sm.mapObj { // GO map iterator will randomize the order. We just delete the first in the iterator + delete(*sm.mapObj, k) + numEvicted++ + if numEvicted >= numToEvict { + break + } + } + } + } + + (*sm.mapObj)[key] = entry +} + +// Delete removes the entry with the specified key from the SyncMap. +// If the key does not exist, this method does nothing. +func (sm *SyncMap) Delete(key string) { + sm.lock.Lock() + defer sm.lock.Unlock() + delete(*sm.mapObj, key) +} + +// MakeSyncMap creates a new SyncMap with the specified maximum number of entries. +// If the maximum number of entries is less than or equal to 0, it will be set to 1. +func MakeSyncMap(maxEntries int) *SyncMap { + if maxEntries <= 0 { + maxEntries = 1 + } + return &SyncMap{mapObj: &map[string]interface{}{}, + lock: &sync.RWMutex{}, + capacity: maxEntries, + evictionPercentage: defaultEvictionPercentage} +} diff --git a/internal/cache/syncmap_test.go b/internal/cache/syncmap_test.go new file mode 100644 index 0000000..a8f36dc --- /dev/null +++ b/internal/cache/syncmap_test.go @@ -0,0 +1,94 @@ +package cache + +import ( + "fmt" + "sync" + "testing" +) + +func TestSyncMapAddEvict(t *testing.T) { + sm := MakeSyncMap(100) + sm.evictionPercentage = 10 + var wg sync.WaitGroup + addEntry := func(key string, value int) { + sm.Set(key, value) + wg.Done() + } + + wg.Add(100) + for i := 0; i < 100; i++ { + go addEntry(fmt.Sprintf("%d", i), i) + } + wg.Wait() + + if mapLen := len(*sm.mapObj); mapLen != 100 { + t.Fatalf("unexpected length of map after adding to capacity: %d", mapLen) + } + + sm.Set("200", 200) //Now it's beyond the map capacity. 10% of entries will be evicted + if mapLen := len(*sm.mapObj); mapLen != 91 { + t.Fatalf("unexpected length of map after adding beyond capacity: %d", mapLen) + } +} + +func TestSyncMapAddDelete(t *testing.T) { + sm := MakeSyncMap(10) + var wg sync.WaitGroup + + addEntry := func(key string, value int) { + sm.Set(key, value) + wg.Done() + } + + deleteEntry := func(key string) { + sm.Delete(key) + wg.Done() + } + + wg.Add(10) + for i := 0; i < 10; i++ { + go addEntry(fmt.Sprintf("%d", i), i) + } + + wg.Wait() + + wg.Add(10) + for i := 0; i < 10; i++ { + go deleteEntry(fmt.Sprintf("%d", i)) + } + + wg.Wait() + mapLen := len(*sm.mapObj) + if mapLen != 0 { + t.Fatalf("unexpected length of map: %d", mapLen) + } +} + +func TestSyncMapUpdate(t *testing.T) { + sm := MakeSyncMap(10) + var wg sync.WaitGroup + addEntry := func(key string, value int) { + sm.Set(key, value) + wg.Done() + } + + wg.Add(10) + for i := 0; i < 10; i++ { + go addEntry(fmt.Sprintf("%d", i), i) + } + wg.Wait() + wg.Add(10) + for i := 0; i < 10; i++ { + go addEntry(fmt.Sprintf("%d", i), i*2) + } + wg.Wait() + + entry0, ok0 := sm.Get("0") + entry9, ok1 := sm.Get("9") + if !ok0 || !ok1 { + t.Fatalf("no matching items in map") + } + if entry0.(int) != 0 || entry9.(int) != 18 { + t.Fatalf("value is not correct") + } +} diff --git a/internal/containerd/mirror.go b/internal/containerd/mirror.go new file mode 100644 index 0000000..4275415 --- /dev/null +++ b/internal/containerd/mirror.go @@ -0,0 +1,158 @@ +package containerd + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "path" + + "github.com/pelletier/go-toml/v2" + "github.com/rs/zerolog" + "github.com/spf13/afero" +) + +const ( + backupDir = "_backup" +) + +type hostFile struct { + Server string `toml:"server"` + HostConfigs map[string]hostConfig `toml:"host"` +} + +type hostConfig struct { + Capabilities []string `toml:"capabilities"` +} + +// AddMirrorConfiguration adds mirror configuration to containerd host configuration. +// Refer to containerd registry configuration documentation for mor information about required configuration. +// https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration +// https://github.com/containerd/containerd/blob/main/docs/hosts.md#registry-configuration---examples +func AddMirrorConfiguration(ctx context.Context, fs afero.Fs, configPath string, registryURLs, mirrorURLs []url.URL, resolveTags bool) error { + log := zerolog.Ctx(ctx).With().Str("component", "containerd-mirror").Logger() + + if err := validate(registryURLs); err != nil { + return err + } + + // Create config path dir if it does not exist + ok, err := afero.DirExists(fs, configPath) + if err != nil { + return err + } + + if !ok { + err := fs.MkdirAll(configPath, 0755) + if err != nil { + return err + } + } + + // Backup files and directories in config path + backupDirPath := path.Join(configPath, backupDir) + if _, err := fs.Stat(backupDirPath); os.IsNotExist(err) { + files, err := afero.ReadDir(fs, configPath) + if err != nil { + return err + } + if len(files) > 0 { + err = fs.MkdirAll(backupDirPath, 0755) + if err != nil { + return err + } + for _, fi := range files { + oldPath := path.Join(configPath, fi.Name()) + newPath := path.Join(backupDirPath, fi.Name()) + err := fs.Rename(oldPath, newPath) + if err != nil { + return err + } + log.Info().Str("path", oldPath).Str("target", newPath).Msg("backing up Containerd host configuration") + } + } + } + + // Remove all content from config path to start from a clean slate + files, err := afero.ReadDir(fs, configPath) + if err != nil { + return err + } + for _, fi := range files { + if fi.Name() == backupDir { + continue + } + filePath := path.Join(configPath, fi.Name()) + err := fs.RemoveAll(filePath) + if err != nil { + return err + } + } + + // Write mirror configuration + capabilities := []string{"pull"} + if resolveTags { + capabilities = append(capabilities, "resolve") + } + for _, registryURL := range registryURLs { + // Need a special case for Docker Hub as docker.io is just an alias. + server := registryURL.String() + if registryURL.String() == "https://docker.io" { + server = "https://registry-1.docker.io" + } + + hostConfigs := map[string]hostConfig{} + for _, u := range mirrorURLs { + hostConfigs[u.String()] = hostConfig{Capabilities: capabilities} + } + + cfg := hostFile{ + Server: server, + HostConfigs: hostConfigs, + } + + b, err := toml.Marshal(&cfg) + if err != nil { + return err + } + + fp := path.Join(configPath, registryURL.Host, "hosts.toml") + err = fs.MkdirAll(path.Dir(fp), 0755) + if err != nil { + return err + } + + err = afero.WriteFile(fs, fp, b, 0644) + if err != nil { + return err + } + + log.Info().Str("host", registryURL.String()).Str("path", fp).Msg("added containerd mirror configuration") + } + + return nil +} + +// validate validates registry URLs. +func validate(urls []url.URL) error { + errs := []error{} + for _, u := range urls { + if u.Scheme != "http" && u.Scheme != "https" { + errs = append(errs, fmt.Errorf("invalid registry url, scheme must be http or https, got: %s", u.String())) + } + + if u.Path != "" { + errs = append(errs, fmt.Errorf("invalid registry url, path has to be empty, got: %s", u.String())) + } + + if len(u.Query()) != 0 { + errs = append(errs, fmt.Errorf("invalid registry url, query has to be empty, got: %s", u.String())) + } + + if u.User != nil { + errs = append(errs, fmt.Errorf("invalid registry url, user has to be empty, got: %s", u.String())) + } + } + return errors.Join(errs...) +} diff --git a/internal/containerd/mirror_test.go b/internal/containerd/mirror_test.go new file mode 100644 index 0000000..2cb66a8 --- /dev/null +++ b/internal/containerd/mirror_test.go @@ -0,0 +1,229 @@ +package containerd + +import ( + "context" + iofs "io/fs" + "net/url" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func TestMirrorConfiguration(t *testing.T) { + registryConfigPath := "/etc/containerd/certs.d" + + tests := []struct { + name string + resolveTags bool + registries []url.URL + mirrors []url.URL + createConfigPathDir bool + existingFiles map[string]string + expectedFiles map[string]string + }{ + { + name: "multiple mirros", + resolveTags: true, + registries: stringListToUrlList(t, []string{"http://foo.bar:5000"}), + mirrors: stringListToUrlList(t, []string{"http://127.0.0.1:5000", "http://127.0.0.1:5001"}), + expectedFiles: map[string]string{ + "/etc/containerd/certs.d/foo.bar:5000/hosts.toml": `server = 'http://foo.bar:5000' + +[host] +[host.'http://127.0.0.1:5000'] +capabilities = ['pull', 'resolve'] + +[host.'http://127.0.0.1:5001'] +capabilities = ['pull', 'resolve'] +`, + }, + }, + { + name: "resolve tags disabled", + resolveTags: false, + registries: stringListToUrlList(t, []string{"https://docker.io", "http://foo.bar:5000"}), + mirrors: stringListToUrlList(t, []string{"http://127.0.0.1:5000"}), + expectedFiles: map[string]string{ + "/etc/containerd/certs.d/docker.io/hosts.toml": `server = 'https://registry-1.docker.io' + +[host] +[host.'http://127.0.0.1:5000'] +capabilities = ['pull'] +`, + "/etc/containerd/certs.d/foo.bar:5000/hosts.toml": `server = 'http://foo.bar:5000' + +[host] +[host.'http://127.0.0.1:5000'] +capabilities = ['pull'] +`, + }, + }, + { + name: "config path directory does not exist", + resolveTags: true, + registries: stringListToUrlList(t, []string{"https://docker.io", "http://foo.bar:5000"}), + mirrors: stringListToUrlList(t, []string{"http://127.0.0.1:5000"}), + createConfigPathDir: false, + expectedFiles: map[string]string{ + "/etc/containerd/certs.d/docker.io/hosts.toml": `server = 'https://registry-1.docker.io' + +[host] +[host.'http://127.0.0.1:5000'] +capabilities = ['pull', 'resolve'] +`, + "/etc/containerd/certs.d/foo.bar:5000/hosts.toml": `server = 'http://foo.bar:5000' + +[host] +[host.'http://127.0.0.1:5000'] +capabilities = ['pull', 'resolve'] +`, + }, + }, + { + name: "config path directory does exist", + resolveTags: true, + registries: stringListToUrlList(t, []string{"https://docker.io", "http://foo.bar:5000"}), + mirrors: stringListToUrlList(t, []string{"http://127.0.0.1:5000"}), + createConfigPathDir: true, + expectedFiles: map[string]string{ + "/etc/containerd/certs.d/docker.io/hosts.toml": `server = 'https://registry-1.docker.io' + +[host] +[host.'http://127.0.0.1:5000'] +capabilities = ['pull', 'resolve'] +`, + "/etc/containerd/certs.d/foo.bar:5000/hosts.toml": `server = 'http://foo.bar:5000' + +[host] +[host.'http://127.0.0.1:5000'] +capabilities = ['pull', 'resolve'] +`, + }, + }, + { + name: "config path directory contains configuration", + resolveTags: true, + registries: stringListToUrlList(t, []string{"https://docker.io", "http://foo.bar:5000"}), + mirrors: stringListToUrlList(t, []string{"http://127.0.0.1:5000"}), + createConfigPathDir: true, + existingFiles: map[string]string{ + "/etc/containerd/certs.d/docker.io/hosts.toml": "Hello World", + "/etc/containerd/certs.d/ghcr.io/hosts.toml": "Foo Bar", + }, + expectedFiles: map[string]string{ + "/etc/containerd/certs.d/_backup/docker.io/hosts.toml": "Hello World", + "/etc/containerd/certs.d/_backup/ghcr.io/hosts.toml": "Foo Bar", + "/etc/containerd/certs.d/docker.io/hosts.toml": `server = 'https://registry-1.docker.io' + +[host] +[host.'http://127.0.0.1:5000'] +capabilities = ['pull', 'resolve'] +`, + "/etc/containerd/certs.d/foo.bar:5000/hosts.toml": `server = 'http://foo.bar:5000' + +[host] +[host.'http://127.0.0.1:5000'] +capabilities = ['pull', 'resolve'] +`, + }, + }, + { + name: "config path directory contains backup", + resolveTags: true, + registries: stringListToUrlList(t, []string{"https://docker.io", "http://foo.bar:5000"}), + mirrors: stringListToUrlList(t, []string{"http://127.0.0.1:5000"}), + createConfigPathDir: true, + existingFiles: map[string]string{ + "/etc/containerd/certs.d/_backup/docker.io/hosts.toml": "Hello World", + "/etc/containerd/certs.d/_backup/ghcr.io/hosts.toml": "Foo Bar", + "/etc/containerd/certs.d/test.txt": "test", + "/etc/containerd/certs.d/foo": "bar", + }, + expectedFiles: map[string]string{ + "/etc/containerd/certs.d/_backup/docker.io/hosts.toml": "Hello World", + "/etc/containerd/certs.d/_backup/ghcr.io/hosts.toml": "Foo Bar", + "/etc/containerd/certs.d/docker.io/hosts.toml": `server = 'https://registry-1.docker.io' + +[host] +[host.'http://127.0.0.1:5000'] +capabilities = ['pull', 'resolve'] +`, + "/etc/containerd/certs.d/foo.bar:5000/hosts.toml": `server = 'http://foo.bar:5000' + +[host] +[host.'http://127.0.0.1:5000'] +capabilities = ['pull', 'resolve'] +`, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fs := afero.NewMemMapFs() + if tt.createConfigPathDir { + err := fs.Mkdir(registryConfigPath, 0755) + require.NoError(t, err) + } + + for k, v := range tt.existingFiles { + err := afero.WriteFile(fs, k, []byte(v), 0644) + require.NoError(t, err) + } + + err := AddMirrorConfiguration(context.TODO(), fs, registryConfigPath, tt.registries, tt.mirrors, tt.resolveTags) + require.NoError(t, err) + + if len(tt.existingFiles) == 0 { + ok, err := afero.DirExists(fs, "/etc/containerd/certs.d/_backup") + require.NoError(t, err) + require.False(t, ok) + } + + err = afero.Walk(fs, registryConfigPath, func(path string, fi iofs.FileInfo, _ error) error { + if fi.IsDir() { + return nil + } + expectedContent, ok := tt.expectedFiles[path] + require.True(t, ok, path) + b, err := afero.ReadFile(fs, path) + require.NoError(t, err) + require.Equal(t, expectedContent, string(b)) + return nil + }) + require.NoError(t, err) + }) + } +} + +func TestMirrorConfigurationInvalidMirrorURL(t *testing.T) { + fs := afero.NewMemMapFs() + mirrors := stringListToUrlList(t, []string{"http://127.0.0.1:5000"}) + + registries := stringListToUrlList(t, []string{"ftp://docker.io"}) + err := AddMirrorConfiguration(context.TODO(), fs, "/etc/containerd/certs.d", registries, mirrors, true) + require.EqualError(t, err, "invalid registry url, scheme must be http or https, got: ftp://docker.io") + + registries = stringListToUrlList(t, []string{"https://docker.io/foo/bar"}) + err = AddMirrorConfiguration(context.TODO(), fs, "/etc/containerd/certs.d", registries, mirrors, true) + require.EqualError(t, err, "invalid registry url, path has to be empty, got: https://docker.io/foo/bar") + + registries = stringListToUrlList(t, []string{"https://docker.io?foo=bar"}) + err = AddMirrorConfiguration(context.TODO(), fs, "/etc/containerd/certs.d", registries, mirrors, true) + require.EqualError(t, err, "invalid registry url, query has to be empty, got: https://docker.io?foo=bar") + + registries = stringListToUrlList(t, []string{"https://foo@docker.io"}) + err = AddMirrorConfiguration(context.TODO(), fs, "/etc/containerd/certs.d", registries, mirrors, true) + require.EqualError(t, err, "invalid registry url, user has to be empty, got: https://foo@docker.io") +} + +func stringListToUrlList(t *testing.T, list []string) []url.URL { + t.Helper() + urls := []url.URL{} + for _, item := range list { + u, err := url.Parse(item) + require.NoError(t, err) + urls = append(urls, *u) + } + return urls +} diff --git a/internal/context/context.go b/internal/context/context.go new file mode 100644 index 0000000..ec99c80 --- /dev/null +++ b/internal/context/context.go @@ -0,0 +1,158 @@ +package context + +import ( + "errors" + "fmt" + "net/http" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/rs/zerolog" +) + +// P2P network. +const ( + KeyTTL = 30 * time.Minute +) + +// Cache constants. +const ( + P2pLookupCacheTtl = 500 * time.Millisecond + P2pLookupNotFoundValue = "PEER_NOT_FOUND" +) + +// Context keys. +const ( + CorrelationIdCtxKey = "correlation_id" + DigestCtxKey = "digest" + FileChunkCtxKey = "file_chunk" + BlobUrlCtxKey = "blob_url" + BlobRangeCtxKey = "blob_range" + NamespaceCtxKey = "namespace" + ReferenceCtxKey = "reference" + RefTypeCtxKey = "ref_type" + LoggerCtxKey = "logger" +) + +// Request headers. +const ( + P2PHeaderKey = "X-MS-Cluster-P2P-RequestFromPeer" + CorrelationHeaderKey = "X-MS-Cluster-P2P-CorrelationId" + NodeHeaderKey = "X-MS-Cluster-P2P-Node" +) + +// Log messages. +const ( + PeerResolutionStartLog = "peer resolution start" + PeerResolutionStopLog = "peer resolution stop" + PeerNotFoundLog = "peer not found" + PeerResolutionExhaustedLog = "peer resolution exhausted" + PeerRequestErrorLog = "peer request error" +) + +var ( + NodeName, _ = os.Hostname() + Namespace = "peerd-ns" + + // KubeConfigPath is the path of the kubeconfig file, which is used if run in an environment outside a pod. + KubeConfigPath = "/opt/peerd/kubeconfig" +) + +// IsRequestFromAPeer indicates if the current request is from a peer. +func IsRequestFromAPeer(c *gin.Context) bool { + return c.Request.Header.Get(P2PHeaderKey) == "true" +} + +func FillCorrelationId(c *gin.Context) { + correlationId := c.Request.Header.Get(CorrelationHeaderKey) + if correlationId == "" { + correlationId = uuid.New().String() + } + c.Set(CorrelationIdCtxKey, correlationId) +} + +// Logger gets the logger with request specific fields. +func Logger(c *gin.Context) zerolog.Logger { + var l zerolog.Logger + obj, ok := c.Get(LoggerCtxKey) + if !ok { + fmt.Println("WARN: logger not found in context") + l = zerolog.Nop() + } else { + ctxLog := obj.(*zerolog.Logger) + l = *ctxLog + } + + return l.With().Str("correlationid", c.GetString(CorrelationIdCtxKey)).Str("url", c.Request.URL.String()).Str("range", c.Request.Header.Get("Range")).Bool("p2p", IsRequestFromAPeer(c)).Str("ip", c.ClientIP()).Str("peer", c.Request.Header.Get(NodeHeaderKey)).Logger() +} + +// BlobUrl extracts the blob URL from the incoming request URL. +func BlobUrl(c *gin.Context) string { + return strings.TrimPrefix(c.Param("url"), "/") + "?" + c.Request.URL.RawQuery +} + +// SetOutboundHeaders sets the mandatory headers for all outbound requests. +func SetOutboundHeaders(r *http.Request, c *gin.Context) { + r.Header.Set(P2PHeaderKey, "true") + r.Header.Set(CorrelationHeaderKey, c.GetString(CorrelationIdCtxKey)) + r.Header.Set(NodeHeaderKey, NodeName) +} + +// Merge merges multiple input channels into a single output channel. +// It starts a goroutine for each input channel and sends the values from each input channel to the output channel. +// Once all input channels are closed, it closes the output channel. +// The function returns the output channel. +func Merge[T any](cs ...<-chan T) <-chan T { + var wg sync.WaitGroup + out := make(chan T) + + output := func(c <-chan T) { + for n := range c { + out <- n + } + wg.Done() + } + wg.Add(len(cs)) + for _, c := range cs { + go output(c) + } + + go func() { + wg.Wait() + close(out) + }() + return out +} + +// RangeStartIndex returns the start index of a byte range specified in the given range header value. +// It expects the range value to be in the format "bytes=startIndex-endIndex". +func RangeStartIndex(rangeValue string) (int64, error) { + if rangeValue == "" { + return 0, errors.New("no range header") + } + + // split the range value by "=" + parts := strings.Split(rangeValue, "=") + if len(parts) != 2 || parts[0] != "bytes" { + return 0, errors.New("invalid range format") + } + + // split the byte range by "-" + ranges := strings.Split(parts[1], "-") + if len(ranges) != 2 { + return 0, errors.New("invalid range format") + } + + // convert the start index to an integer + startIndex, err := strconv.Atoi(ranges[0]) + if err != nil { + return 0, err + } + + return int64(startIndex), nil +} diff --git a/internal/context/context_test.go b/internal/context/context_test.go new file mode 100644 index 0000000..f6eb7b8 --- /dev/null +++ b/internal/context/context_test.go @@ -0,0 +1,259 @@ +package context + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/rs/zerolog" +) + +var ( + hostAndPath = "https://avtakkartest.blob.core.windows.net/d18c7a64c5158179-ff8cb2f639ff44879c12c94361a746d0-782b855128//docker/registry/v2/blobs/sha256/d1/d18c7a64c5158179bdee531a663c5b487de57ff17cff3af29a51c7e70b491d9d/data" + query = "?se=2023-09-20T01%3A14%3A49Z&sig=m4Cr%2BYTZHZQlN5LznY7nrTQ4LCIx2OqnDDM3Dpedbhs%3D&sp=r&spr=https&sr=b&sv=2018-03-28®id=01031d61e1024861afee5d512651eb9f" + u = hostAndPath + query +) + +func TestLogger(t *testing.T) { + req, err := http.NewRequest("GET", "http://127.0.0.1:5000/blobs/"+u, nil) + if err != nil { + t.Fatal(err) + } + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = req + + l := Logger(c) + if l.Info().Enabled() { + t.Fatal("expected logger to be disabled") + } + + testL := zerolog.New(os.Stdout).With().Timestamp().Logger() + c.Set(LoggerCtxKey, &testL) + + l = Logger(c) + if !l.Info().Enabled() { + t.Fatal("expected logger to be enabled") + } +} + +func TestSetOutboundHeaders(t *testing.T) { + // Create a new request with a URL that has a query string. + req, err := http.NewRequest("GET", "http://127.0.0.1:5000/blobs/"+u, nil) + if err != nil { + t.Fatal(err) + } + + // Create a new context with the request. + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = req + FillCorrelationId(ctx) + + SetOutboundHeaders(req, ctx) + + if req.Header.Get(P2PHeaderKey) != "true" { + t.Errorf("expected: %v, got: %v", "true", req.Header.Get(P2PHeaderKey)) + } + + if req.Header.Get(CorrelationHeaderKey) == "" { + t.Errorf("expected: %v, got: %v", "not empty", req.Header.Get(CorrelationHeaderKey)) + } + + if req.Header.Get(NodeHeaderKey) != NodeName { + t.Errorf("expected: %v, got: %v", NodeName, req.Header.Get(NodeHeaderKey)) + } +} + +func TestMerge(t *testing.T) { + + ch1 := make(chan string, 10) + ch2 := make(chan string) + ch3 := make(chan string, 100) + ch4 := make(chan string, 1000) + + mergedChan := Merge(ch1, ch2, ch3, ch4) + + // Write to the channels. + go func() { + for i := 0; i < 100; i++ { + ch1 <- fmt.Sprintf("ch1-%d", i) + } + close(ch1) + }() + + go func() { + for i := 0; i < 100; i++ { + ch2 <- fmt.Sprintf("ch2-%d", i) + } + close(ch2) + }() + + go func() { + for i := 0; i < 100; i++ { + ch3 <- fmt.Sprintf("ch3-%d", i) + } + close(ch3) + }() + + go func() { + for i := 0; i < 100; i++ { + ch4 <- fmt.Sprintf("ch4-%d", i) + } + close(ch4) + }() + + // Read from the merged channel. + total := 0 + for val := range mergedChan { + if strings.HasPrefix(val, "ch1-") || + strings.HasPrefix(val, "ch2-") || + strings.HasPrefix(val, "ch3-") || + strings.HasPrefix(val, "ch4-") { + total++ + } else { + t.Errorf("unexpected value: %v", val) + } + } + + if total != 400 { + t.Errorf("expected: %v, got: %v", 400, total) + } +} + +func TestBlobUrl(t *testing.T) { + // Create a new request with a URL that has a query string. + req, err := http.NewRequest("GET", "http://127.0.0.1:5000/blobs/"+u, nil) + if err != nil { + t.Fatal(err) + } + + // Create a new context with the request. + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = req + ctx.Params = []gin.Param{ + {Key: "url", Value: hostAndPath}, + } + + // Call BlobUrl and verify the result. + got := BlobUrl(ctx) + if got != u { + t.Errorf("expected: %v, got: %v", u, got) + } +} + +func TestFillCorrelationId(t *testing.T) { + // Create a new request without any correlation ID headers. + req, err := http.NewRequest("GET", "http://127.0.0.1:5000/blobs/fdsfsdsd", nil) + if err != nil { + t.Fatal(err) + } + + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = req + + FillCorrelationId(ctx) + cid, ok := ctx.Get(CorrelationIdCtxKey) + if !ok || cid == "" { + t.Fatal("expected correlation ID to be set") + } + + sample := uuid.New().String() + + ctx, _ = gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = req + ctx.Request.Header.Set(CorrelationHeaderKey, sample) + FillCorrelationId(ctx) + cid, ok = ctx.Get(CorrelationIdCtxKey) + if !ok || cid == "" { + t.Fatal("expected correlation ID to be set") + } else if cid != sample { + t.Errorf("expected: %v, got: %v", sample, cid) + } +} + +func TestIsRequestFromPeer(t *testing.T) { + // Create a new request without any correlation ID headers. + req, err := http.NewRequest("GET", "http://127.0.0.1:5000/blobs/fdsfsdsd", nil) + if err != nil { + t.Fatal(err) + } + + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = req + + if IsRequestFromAPeer(ctx) { + t.Fatal("expected request to not be from a peer") + } + + ctx.Request.Header.Set(P2PHeaderKey, "true") + if !IsRequestFromAPeer(ctx) { + t.Fatal("expected request to be from a peer") + } +} + +func TestRangeStartIndex(t *testing.T) { + for _, tc := range []struct { + name string + r string + want int64 + expectedError string + }{ + { + name: "no range header", + r: "", + want: 0, + expectedError: "no range header", + }, + { + name: "invalid range format", + r: "bytes=0", + want: 0, + expectedError: "invalid range format", + }, + { + name: "invalid range format", + r: "bytes=0-", + want: 0, + expectedError: "invalid range format", + }, + { + name: "invalid range format", + r: "bytes=0-100-200", + want: 0, + expectedError: "invalid range format", + }, + { + name: "valid range format", + r: "bytes=91-100", + want: 91, + expectedError: "", + }, + { + name: "invalid range format", + r: "count=91-100", + want: 0, + expectedError: "invalid range format", + }, + { + name: "invalid range format", + r: "bytes=9.1-100", + want: 0, + expectedError: "strconv.Atoi: parsing \"9.1\": invalid syntax", + }, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := RangeStartIndex(tc.r) + if err != nil { + if err.Error() != tc.expectedError { + t.Errorf("expected: %v, got: %v", tc.expectedError, err.Error()) + } + } else if got != tc.want { + t.Errorf("expected: %v, got: %v", tc.want, got) + } + }) + } +} diff --git a/internal/files/cache/cache.go b/internal/files/cache/cache.go new file mode 100644 index 0000000..b60ea4e --- /dev/null +++ b/internal/files/cache/cache.go @@ -0,0 +1,186 @@ +package cache + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "sync" + "sync/atomic" + "time" + + syncmap "github.com/azure/peerd/internal/cache" + "github.com/azure/peerd/internal/files" + "github.com/dgraph-io/ristretto" + "github.com/rs/zerolog" +) + +// fileCache implements FileCache. +type fileCache struct { + fileCache *ristretto.Cache + metadataCache *syncmap.SyncMap + path string + lock sync.RWMutex + log zerolog.Logger +} + +var _ Cache = &fileCache{} + +// Exists checks if the file exists in the cache. +func (c *fileCache) Exists(name string, offset int64) bool { + key := c.getKey(name, offset) + val, found := c.fileCache.Get(key) + if found { + cacheItem := val.(*item) + cacheItem.lock.Lock() + defer cacheItem.lock.Unlock() + + if info, err := cacheItem.file.Stat(); err != nil { + return false + } else { + return info.Size() > 0 + } + } + return false +} + +// GetOrCreate gets the cached value if available, otherwise fetches it. +func (c *fileCache) GetOrCreate(name string, alignedOffset int64, count int, fetch func() ([]byte, error)) ([]byte, error) { + key := c.getKey(name, alignedOffset) + val, found := c.fileCache.Get(key) + if !found { + c.lock.Lock() + if val, found = c.fileCache.Get(key); found && val != nil { + c.lock.Unlock() + } else { + var err error + val, err = newItem(key, c.log) + if err != nil { + c.lock.Unlock() + return nil, err + } + ok := c.fileCache.Set(key, val, 0) + if !ok { + c.lock.Unlock() + return nil, io.ErrUnexpectedEOF + } + + // wait for value to pass through buffers + waitForSet() + + c.lock.Unlock() + } + } + + cacheItem := val.(*item) + + cacheItem.lock.RLock() + info, err := cacheItem.file.Stat() + + if err != nil { + cacheItem.lock.RUnlock() + return nil, err + } + + if info.Size() != int64(count) { + cacheItem.lock.RUnlock() + + cacheItem.lock.Lock() + + // check again after acquiring lock + info, err = cacheItem.file.Stat() + if err != nil { + cacheItem.lock.Unlock() + return nil, err + } else if info.Size() != int64(count) { + + n, err := cacheItem.fill(c.log, fetch) + cacheItem.lock.Unlock() + + if err != nil { + return nil, err + } else if int64(n) != int64(count) { + return nil, fmt.Errorf("fill did not retrieve expected number of bytes, expected: %v, got: %v", count, n) + } + } else { + cacheItem.lock.Unlock() + } + cacheItem.lock.RLock() + } + + result := cacheItem.bytes(c.log) + cacheItem.lock.RUnlock() + + if len(result) != count { + return result, fmt.Errorf("bytes did not retrieve expected number of bytes, expected: %v, got: %v", count, len(result)) + } + + return result, nil +} + +// Size gets the length of the file. +func (c *fileCache) Size(name string) (int64, bool) { + key := filepath.Join(name, "metainfo") + // c.metadataCache.Wait() + val, found := c.metadataCache.Get(key) + if !found { + return 0, false + } + return val.(int64), true +} + +// PutSize puts the length of the file. +func (c *fileCache) PutSize(name string, len int64) bool { + key := filepath.Join(name, "metainfo") + c.metadataCache.Set(key, len) + c.log.Debug().Str("key", key).Int64("len", len).Msg("put len") + return true +} + +func (c *fileCache) getKey(name string, offset int64) string { + return filepath.Join(c.path, name, strconv.FormatInt(offset, 10)) +} + +func waitForSet() { + time.Sleep(10 * time.Millisecond) +} + +// New creates a new cache of files. +func New(ctx context.Context) Cache { + log := zerolog.Ctx(ctx).With().Str("component", "cache").Logger() + + atomic.StoreInt32(&fdCnt, 0) + if err := os.MkdirAll(Path, 0755); err != nil { + // This will call os.Exit(1) + log.Fatal().Err(err).Str("path", Path).Msg("failed to initialize cache directory") + } + + cache := &fileCache{ + log: log, + path: Path, + metadataCache: syncmap.MakeSyncMap(1e7), + } + + var err error + if cache.fileCache, err = ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e7, + MaxCost: FilesCacheMaxCost, + BufferItems: 64, + + OnExit: func(val interface{}) { + item := val.(*item) + item.drop(log) + }, + + Cost: func(val interface{}) int64 { + return int64(files.CacheBlockSize) + }, + }); err != nil { + // This will call os.Exit(1) + log.Fatal().Err(err).Msg("failed to initialize file cache") + } + + return cache +} diff --git a/internal/files/cache/cache_test.go b/internal/files/cache/cache_test.go new file mode 100644 index 0000000..e4d5788 --- /dev/null +++ b/internal/files/cache/cache_test.go @@ -0,0 +1,224 @@ +package cache + +import ( + "bytes" + "context" + "fmt" + "math/rand" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/azure/peerd/pkg/math" + "github.com/rs/zerolog" + "golang.org/x/sync/errgroup" +) + +func TestGetKey(t *testing.T) { + name := newRandomStringN(10) + offset := int64(100) + c := New(context.Background()) + got := c.(*fileCache).getKey(name, offset) + want := fmt.Sprintf("%v/%v/%v", Path, name, offset) + if got != want { + t.Errorf("expected: %v, got: %v", want, got) + } +} + +func TestExists(t *testing.T) { + c := New(context.Background()) + + filesThatExist := []string{} + for i := 0; i < 5; i++ { + filename := newRandomStringN(10) + off := int64(100*i + 1) // 1, 101, 201, 301, 401 + filesThatExist = append(filesThatExist, fmt.Sprintf("%v_%v", filename, off)) + //nolint:errcheck + c.GetOrCreate(filename, off, 1024, func() ([]byte, error) { + return []byte(newRandomString()), nil + }) + } + + filesThatExistAndNotFilled := []string{} + for i := 0; i < 5; i++ { + fileName := strings.Split(filesThatExist[i], "_")[0] + off := int64(10*(i+1) + 1) // 11, 21, 31, 41, 51 + filesThatExistAndNotFilled = append(filesThatExistAndNotFilled, fmt.Sprintf("%v_%v", fileName, off)) + key := c.(*fileCache).getKey(fileName, off) + val, err := newItem(key, c.(*fileCache).log) + if err != nil { + t.Fatal(err) + } + c.(*fileCache).fileCache.Set(key, val, 0) + } + + filesThatDoNotExist := []string{} + for i := 0; i < 5; i++ { + filename := newRandomStringN(10) + off := int64(100*i + 1) // 1, 101, 201, 301, 401 + filesThatDoNotExist = append(filesThatDoNotExist, fmt.Sprintf("%v_%v", filename, off)) + } + + type tc struct { + name string + filename string + offset int64 + want bool + } + + tcs := []tc{} + + for i := 0; i < 15; i++ { + var filename, name, offset string + var want bool + + if i < 5 { + name = "exists" + filename = strings.Split(filesThatExist[i], "_")[0] + offset = strings.Split(filesThatExist[i], "_")[1] + want = true + } else if i < 10 { + name = "exists-not-filled" + filename = strings.Split(filesThatExistAndNotFilled[i-5], "_")[0] + offset = strings.Split(filesThatExistAndNotFilled[i-5], "_")[1] + want = false + } else { + name = "does-not-exist" + filename = strings.Split(filesThatDoNotExist[i-10], "_")[0] + offset = strings.Split(filesThatDoNotExist[i-10], "_")[1] + want = false + } + + o, _ := strconv.Atoi(offset) + tcs = append(tcs, tc{ + name: name, + filename: filename, + offset: int64(o), + want: want, + }) + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + got := c.Exists(tc.filename, tc.offset) + if got != tc.want { + t.Errorf("expected: %v, got: %v", tc.want, got) + } + }) + } +} + +func TestPutAndGetSize(t *testing.T) { + c := New(context.Background()) + var eg errgroup.Group + + for i := 0; i < 1000; i++ { + eg.Go(func() error { + filename := newRandomStringN(10) + size := rand.Int63n(1024) + + _, ok := c.Size(filename) + if ok == true { + return fmt.Errorf("expected false, got %v", ok) + } + + ok = c.PutSize(filename, size) + if ok != true { + return fmt.Errorf("expected true, got %v", ok) + } + + val, ok := c.Size(filename) + if !ok { + return fmt.Errorf("file size: expected true, got %v", ok) + } + if val != size { + return fmt.Errorf("expected %v, got %v", size, val) + } + return nil + }) + } + err := eg.Wait() + if err != nil { + t.Fatal(err) + } +} + +func TestGetOrCreate(t *testing.T) { + zerolog.TimeFieldFormat = time.RFC3339 + //c := New(zerolog.New(os.Stdout).With().Timestamp().Logger().WithContext(context.Background())) + c := New(context.Background()) + var eg errgroup.Group + + fileNames := new(sync.Map) + fileContents := new(sync.Map) + fileSizes := new(sync.Map) + + for i := 0; i < 100; i++ { + + fileNames.Store(i, newRandomStringN(10)) + b := []byte(newRandomString()) + size := int64(len(b)) + fileContents.Store(i, b) + fileSizes.Store(i, size) + + for j := 0; j < 10; j++ { + segs, err := math.NewSegments(0, 1024*1024, size, size) + if err != nil { + t.Fatal(err) + } + for seg := range segs.All() { + + s := seg + fileIndex := i + + eg.Go(func() error { + offset := s.Index + count := s.Count + fc, ok := fileContents.Load(fileIndex) + if !ok { + t.Fatalf("could not load fileContent from sync map: %v", fileIndex) + } + fcBytes, _ := fc.([]byte) + expected := make([]byte, count) + copy(expected, fcBytes[offset:offset+int64(count)]) + + fn, ok := fileNames.Load(fileIndex) + if !ok { + t.Fatalf("could not load fileName from sync map: %v", fileIndex) + } + name, _ := fn.(string) + + got, err := c.GetOrCreate(name, offset, count, func() ([]byte, error) { + return expected, nil + }) + if err != nil { + return fmt.Errorf("failed to get or create: %v -- %v", name, err) + } + + l := len(got) + if count != l { + return fmt.Errorf("size mismatch, expected %v, got %v, offset: %v, fileName: %v", count, len(got), offset, name) + } + + validationLen := math.Min(100, count) + if !bytes.Equal(expected[:validationLen], got[:validationLen]) { + return fmt.Errorf("leading bytes mismatch, expected: %v, got: %v", expected[:validationLen], got[:validationLen]) + } + + if !bytes.Equal(expected[l-validationLen:], got[l-validationLen:]) { + return fmt.Errorf("ending bytes mismatch, expected: %v, got: %v", expected[l-validationLen:], got[l-validationLen:]) + } + + return nil + }) + } + } + } + + err := eg.Wait() + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/files/cache/interface.go b/internal/files/cache/interface.go new file mode 100644 index 0000000..0cbcfe7 --- /dev/null +++ b/internal/files/cache/interface.go @@ -0,0 +1,27 @@ +package cache + +// Cache describes the cache of files. +type Cache interface { + // Size gets the size of the file. + Size(path string) (int64, bool) + + // PutSize sets size of the file. + PutSize(path string, length int64) bool + + // Exists checks if the given chunk of the file is already cached. + Exists(name string, offset int64) bool + + // GetOrCreate gets the cached value if available, otherwise downloads the file. + GetOrCreate(name string, offset int64, count int, fetch func() ([]byte, error)) ([]byte, error) +} + +var ( + // FilesCacheMaxCost is the capacity of the files cache in any unit. + FilesCacheMaxCost int64 = 4 * 1024 * 1024 * 1024 // 4 Gib + + // MemoryCacheMaxCost is the capacity of the memory cache in any unit. + MemoryCacheMaxCost int64 = 1 * 1024 * 1024 * 1024 // 1 Gib + + // Path is the path to the cache directory. + Path string = "/tmp/distribution/p2p/cache" +) diff --git a/internal/files/cache/item.go b/internal/files/cache/item.go new file mode 100644 index 0000000..4666d8d --- /dev/null +++ b/internal/files/cache/item.go @@ -0,0 +1,131 @@ +package cache + +import ( + "io" + "os" + "path" + "sync" + "sync/atomic" + + "github.com/rs/zerolog" +) + +var fdCnt int32 + +// item is a cached item. +type item struct { + key string + file *os.File + lock *sync.RWMutex +} + +// drop deletes the underlying file. +func (i *item) drop(l zerolog.Logger) { + i.lock.Lock() + defer i.lock.Unlock() + + count := atomic.AddInt32(&fdCnt, -1) + l.Debug().Str("name", i.file.Name()).Int32("count", count).Msg("cache item drop") + + if err := i.file.Close(); err != nil { + l.Error().Err(err).Str("name", i.file.Name()).Msg("failed to close file") + } + + if err := os.Remove(i.file.Name()); err != nil { + l.Error().Err(err).Str("name", i.file.Name()).Msg("failed to remove file") + } + + i.file = nil +} + +// bytes returns the file bytes. +func (i *item) bytes(l zerolog.Logger) []byte { + b, err := readFromStart(i.file) + if err != nil { + l.Error().Err(err).Str("name", i.file.Name()).Msg("failed to read file") + return nil + } + + return b +} + +// fill files the file with the given data. +func (i *item) fill(log zerolog.Logger, fetch func() ([]byte, error)) (int, error) { + buffer, err := fetch() + if err != nil { + if err := os.Remove(i.file.Name()); err != nil { + log.Error().Err(err).Str("name", i.file.Name()).Msg("attempted to remove file because the size read did not match the file size") + } + return 0, err + } + + l, err := writeAll(i.file, buffer) + if err != nil { + if err := os.Remove(i.file.Name()); err != nil { + log.Error().Err(err).Str("name", i.file.Name()).Msg("attempted to remove file because the size written did not match the file size") + } + return 0, err + } + + return l, nil +} + +// readFromStart reads the entire file from the beginning. +func readFromStart(file *os.File) ([]byte, error) { + info, err := file.Stat() + if err != nil { + return nil, err + } + + fileSize := info.Size() + fileContent := make([]byte, fileSize) + offset := int64(0) + + for offset < fileSize && err == nil { + var l int + l, err = file.ReadAt(fileContent[offset:], offset) + offset += int64(l) + } + if err == io.EOF { + err = nil + } + + if err != nil { + return nil, err + } + + if offset != fileSize { + return nil, io.ErrUnexpectedEOF + } + + return fileContent[:offset], nil +} + +// writeAll writes the file. +func writeAll(file *os.File, buff []byte) (int, error) { + offset := 0 + err := file.Truncate(0) + if err != nil { + return 0, err + } + + return file.Write(buff[offset:]) +} + +// newItem creates a new cache item that is ready to be filled. +func newItem(key string, l zerolog.Logger) (*item, error) { + cacheItem := &item{key: key, lock: new(sync.RWMutex)} + if err := os.MkdirAll(path.Dir(key), 0755); err != nil { + return nil, err + } + + fdCounter := atomic.AddInt32(&fdCnt, 1) + l.Debug().Str("key", key).Int32("count", fdCounter).Msg("create new cached item") + + var err error + if cacheItem.file, err = os.OpenFile(key, os.O_CREATE|os.O_RDWR, 0644); err != nil { + return nil, err + } + + return cacheItem, nil +} diff --git a/internal/files/cache/item_test.go b/internal/files/cache/item_test.go new file mode 100644 index 0000000..7c54b04 --- /dev/null +++ b/internal/files/cache/item_test.go @@ -0,0 +1,197 @@ +package cache + +import ( + "crypto/rand" + "io/fs" + "os" + "path" + "strings" + "testing" + + "github.com/rs/zerolog" +) + +func TestWriteAll(t *testing.T) { + // Setup + l := zerolog.Nop() + name := newRandomStringN(10) + filePath := path.Join(Path, name) + + i, err := newItem(filePath, l) + if err != nil { + t.Fatal(err) + } + data, err := randomBytesN(20) + if err != nil { + t.Fatal(err) + } + + // Test + got, err := writeAll(i.file, data) + + // Assert + if err != nil { + t.Fatal(err) + } else if got != 20 { + t.Fatalf("got %v, expected %v", got, 20) + } + + fileContent, err := os.ReadFile(filePath) + if err != nil { + t.Fatal(err) + } else if string(fileContent) != string(data) { + t.Fatalf("writeAll corrupted data: got %v, expected %v", fileContent, data) + } + +} + +func TestReadFromStart(t *testing.T) { + // Setup + l := zerolog.Nop() + name := newRandomStringN(10) + filePath := path.Join(Path, name) + + i, err := newItem(filePath, l) + if err != nil { + t.Fatal(err) + } + data, err := randomBytesN(20) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile(filePath, data, fs.FileMode(os.O_APPEND)) + if err != nil { + t.Fatal(err) + } + + // Test + got, err := readFromStart(i.file) + if err != nil { + t.Fatal(err) + } else if string(got) != string(data) { + t.Fatalf("got %v, expected %v", got, data) + } +} + +func TestFill(t *testing.T) { + // Setup + l := zerolog.Nop() + name := newRandomStringN(10) + filePath := path.Join(Path, name) + + i, err := newItem(filePath, l) + if err != nil { + t.Fatal(err) + } + data, err := randomBytesN(20) + if err != nil { + t.Fatal(err) + } + dataFunc := func() ([]byte, error) { + return data, nil + } + + // Test + got, err := i.fill(l, dataFunc) + + // Assert + if err != nil { + t.Fatal(err) + } else if got != 20 { + t.Fatalf("got %v, expected %v", got, 20) + } + + fileContent, err := os.ReadFile(filePath) + if err != nil { + t.Fatal(err) + } else if string(fileContent) != string(data) { + t.Fatalf("fill corrupted data: got %v, expected %v", fileContent, data) + } +} + +func TestBytes(t *testing.T) { + // Setup + l := zerolog.Nop() + name := newRandomStringN(10) + filePath := path.Join(Path, name) + + i, err := newItem(filePath, l) + if err != nil { + t.Fatal(err) + } + + data, err := randomBytesN(20) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile(filePath, data, fs.FileMode(os.O_APPEND)) + if err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(filePath) + if err != nil { + t.Fatal(err) + } else if string(got) != string(data) { + t.Fatalf("got %v, expected %v", got, data) + } + + // Test + got = i.bytes(l) + + // Assert + if string(got) != string(data) { + t.Fatalf("got %v, expected %v", got, data) + } +} + +func TestDrop(t *testing.T) { + // Setup + l := zerolog.Nop() + name := newRandomStringN(10) + filePath := path.Join(Path, name) + + i, err := newItem(filePath, l) + if err != nil { + t.Fatal(err) + } + + data, err := randomBytesN(20) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile(filePath, data, fs.FileMode(os.O_APPEND)) + if err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(filePath) + if err != nil { + t.Fatal(err) + } else if string(got) != string(data) { + t.Fatalf("got %v, expected %v", got, data) + } + + // Test + i.drop(l) + + // Assert + got, err = os.ReadFile(filePath) + if err == nil { + t.Fatalf("got %v, expected error", got) + } else if !strings.Contains(err.Error(), "no such file or directory") { + t.Fatal(err) + } +} + +func randomBytesN(n int) ([]byte, error) { + b := make([]byte, n) + _, err := rand.Read(b) + if err != nil { + return nil, err + } + return b, nil +} diff --git a/internal/files/cache/main_test.go b/internal/files/cache/main_test.go new file mode 100644 index 0000000..699db89 --- /dev/null +++ b/internal/files/cache/main_test.go @@ -0,0 +1,59 @@ +package cache + +import ( + "crypto/rand" + "fmt" + "math/big" + "os" + "testing" +) + +func TestMain(m *testing.M) { + setup() + code := m.Run() + err := teardown() + if code == 0 && err != nil { + code = 42 + } + os.Exit(code) +} + +func setup() { + suf := newRandomStringN(10) + Path += suf +} + +// teardown removes the cache directory. +func teardown() error { + if err := os.RemoveAll(Path); err != nil { + return fmt.Errorf("failed to remove cache dir: %v --- %v", Path, err) + } + + return nil +} + +// newRandomString creates a new random string. +func newRandomString() string { + const blockSize = 1024 * 1024 + r, err := rand.Int(rand.Reader, big.NewInt(4)) + if err != nil { + panic(err) + } + length := r.Int64() * blockSize + + r, err = rand.Int(rand.Reader, big.NewInt(blockSize)) + if err != nil { + panic(err) + } + length += r.Int64() + + return newRandomStringN(int(length)) +} + +// newRandomStringN creates a new random string of length n. +func newRandomStringN(n int) string { + randBytes := make([]byte, n/2) + _, _ = rand.Read(randBytes) + + return fmt.Sprintf("%x", randBytes) +} diff --git a/internal/files/files.go b/internal/files/files.go new file mode 100644 index 0000000..db7c48b --- /dev/null +++ b/internal/files/files.go @@ -0,0 +1,37 @@ +package files + +import ( + "fmt" + "io" + + "github.com/azure/peerd/internal/remote" + "github.com/azure/peerd/pkg/math" +) + +const ( + FileChunkKeySep = "_" +) + +// CacheBlockSize is the size of a single cached block. +var CacheBlockSize int = 1 * 1024 * 1024 // 1 Mib + +// FileChunkKey returns the p2p lookup key for the given chunk of a file. +func FileChunkKey(name string, offset, cacheBlockSize int64) string { + return name + FileChunkKeySep + fmt.Sprint(math.AlignDown(offset, cacheBlockSize)) +} + +// Fetchfile gets the content of a file from the given offset using a remote reader. +func FetchFile(r remote.Reader, name string, offset int64, count int) ([]byte, error) { + d := make([]byte, count) + l := r.Log().With().Str("name", name).Int64("offset", offset).Int("count", count).Logger() + l.Debug().Msg("fetch file start") + + _, err := r.PreadRemote(d, offset) + if err != nil && err != io.EOF { + l.Error().Err(err).Msg("fetch file error") + return nil, err + } + + l.Debug().Msg("fetch file stop") + return d, nil +} diff --git a/internal/files/files_test.go b/internal/files/files_test.go new file mode 100644 index 0000000..730d843 --- /dev/null +++ b/internal/files/files_test.go @@ -0,0 +1,84 @@ +package files + +import ( + "fmt" + "os" + "strconv" + "testing" + + "github.com/azure/peerd/internal/remote" + "github.com/rs/zerolog" +) + +func TestFileChunkKey(t *testing.T) { + d := "abc" + cacheBlockSize := int64(1024) + + key := FileChunkKey(d, 123, cacheBlockSize) + if key != "abc_0" { + t.Errorf("expected key %s, got %s", "abc_0", key) + } + + key = FileChunkKey(d, int64(123)+cacheBlockSize, cacheBlockSize) + exp := fmt.Sprintf("abc_%v", cacheBlockSize) + if key != exp { + t.Errorf("expected key %s, got %s", exp, key) + } +} + +func TestFetchFile(t *testing.T) { + d := map[string][]byte{ + "0": []byte("abc"), + "3": []byte("def"), + } + + r := &mockReader{data: d} + + if b, err := FetchFile(r, "test", 0, 3); err != nil { + t.Errorf("expected no error, got %v", err) + } else if string(b) != "abc" { + t.Errorf("expected %s, got %s", "abc", string(b)) + } + + if b, err := FetchFile(r, "test", 0, 4); err != nil { + t.Errorf("expected no error, got %v", err) + } else if string(b[:3]) != "abc" { + t.Errorf("expected %s, got %s", "abc", string(b)) + } + + if b, err := FetchFile(r, "test", 3, 3); err != nil { + t.Errorf("expected no error, got %v", err) + } else if string(b) != "def" { + t.Errorf("expected %s, got %s", "def", string(b)) + } + + if b, err := FetchFile(r, "test", 31, 4); err == nil { + t.Errorf("expected error, got %s", string(b)) + } +} + +type mockReader struct { + data map[string][]byte +} + +// FstatRemote implements remote.Reader. +func (*mockReader) FstatRemote() (int64, error) { + panic("unimplemented") +} + +// Log implements remote.Reader. +func (*mockReader) Log() *zerolog.Logger { + l := zerolog.Nop() + return &l +} + +// PreadRemote implements remote.Reader. +func (m *mockReader) PreadRemote(buf []byte, offset int64) (int, error) { + if d, ok := m.data[strconv.FormatInt(offset, 10)]; ok { + return copy(buf, d), nil + } else { + return 0, os.ErrNotExist + } +} + +var _ remote.Reader = &mockReader{} diff --git a/internal/files/store/file.go b/internal/files/store/file.go new file mode 100644 index 0000000..341bbb3 --- /dev/null +++ b/internal/files/store/file.go @@ -0,0 +1,142 @@ +package store + +import ( + "fmt" + "io" + + "sync" + + "github.com/azure/peerd/internal/files" + "github.com/azure/peerd/internal/remote" + "github.com/azure/peerd/pkg/math" +) + +var errOnlySingleChunkAvailable = fmt.Errorf("only single chunk available") + +// file describes a file that can be read from this content store. +// It implements the File interface. It is similar to os.File. +type file struct { + Name string + + cur int64 + size int64 + + statLock sync.Mutex + + chunkOffset int64 + + reader remote.Reader + store *store +} + +var _ File = &file{} + +// prefetch tries to prefetch the specified parts of the file in chunks of cacheBlockSize. +// It can silently fail. +func (f *file) prefetch(offset int64, count int64) { + go func() { + fileSize, err := f.Fstat() + if err != nil { + return + } + + segs, err := math.NewSegments(offset, files.CacheBlockSize, count, fileSize) + if err != nil { + f.reader.Log().Error().Err(err).Msg("prefetch error: failed to create segments") + return + } + + for seg := range segs.All() { + f.store.prefetchChan <- prefetchableSegment{ + name: f.Name, + reader: f.reader, + offset: seg.Index, + count: seg.Count, + } + } + }() +} + +// Seek sets the current file offset. +func (f *file) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekCurrent: + f.cur += offset + case io.SeekStart: + f.cur = offset + case io.SeekEnd: + f.cur = f.size + } + + return f.cur, nil +} + +// Fstat returns the size of the file. +func (f *file) Fstat() (int64, error) { + var hit bool + + f.size, hit = f.store.cache.Size(f.Name) + if !hit { + f.reader.Log().Debug().Str("name", f.Name).Int64("size", f.size).Msg("fstat getlen cache miss_1") + f.statLock.Lock() + f.size, hit = f.store.cache.Size(f.Name) + if !hit { + f.reader.Log().Debug().Str("name", f.Name).Int64("size", f.size).Msg("fstat getlen cache miss_2") + var err error + f.size, err = f.reader.FstatRemote() + if err != nil { + f.reader.Log().Error().Err(err).Msg("fstat error") + return 0, err + } + f.store.cache.PutSize(f.Name, f.size) + f.reader.Log().Debug().Str("name", f.Name).Int64("size", f.size).Msg("fstat putlen") + } + f.statLock.Unlock() + } + + return f.size, nil +} + +// Read reads up to len(p) bytes into p. It returns the number of bytes read (0 <= n <= len(p)) and any error encountered. +func (f *file) Read(p []byte) (n int, err error) { + ret, err := f.ReadAt(p, f.cur) + if err == nil { + f.cur += int64(ret) + } + return ret, err +} + +// ReadAt reads len(p) bytes from the File starting at byte offset off. It returns the number of bytes read and the error, if any. +func (f *file) ReadAt(buff []byte, offset int64) (int, error) { + fileSize, err := f.Fstat() + if err != nil { + return 0, err + } + + alignedOffset := math.AlignDown(offset, int64(files.CacheBlockSize)) + + if f.chunkOffset != 0 && alignedOffset != f.chunkOffset { + f.reader.Log().Error().Err(errOnlySingleChunkAvailable).Int64("chunk", f.chunkOffset).Int64("alignedOffset", alignedOffset).Int64("requestedOffset", offset).Msg("file can only read chunk") + return -1, errOnlySingleChunkAvailable + } + + count := int(math.Min64(int64(files.CacheBlockSize), fileSize-alignedOffset)) + + data, err := f.store.cache.GetOrCreate(f.Name, alignedOffset, count, func() ([]byte, error) { + return files.FetchFile(f.reader, f.Name, alignedOffset, count) + }) + if err != nil { + f.reader.Log().Error().Err(err).Msg("readat error") + return 0, fmt.Errorf("failed to ReadAt, path: %v, offset: %v, error: %v", f.Name, offset, err.Error()) + } + + pos := int(offset - alignedOffset) + ret := math.Min(len(buff), len(data)-pos) + ret = copy(buff[:ret], data[pos:pos+ret]) + + if offset+int64(len(buff)) > fileSize { + err = io.EOF + } + + return ret, err +} diff --git a/internal/files/store/file_test.go b/internal/files/store/file_test.go new file mode 100644 index 0000000..97f057f --- /dev/null +++ b/internal/files/store/file_test.go @@ -0,0 +1,240 @@ +package store + +import ( + "context" + "crypto/rand" + "io" + "os" + "strings" + "testing" + + "github.com/azure/peerd/internal/files" + "github.com/azure/peerd/internal/files/cache" + remotetests "github.com/azure/peerd/internal/remote/tests" + "github.com/azure/peerd/internal/routing/tests" +) + +func TestReadAtWithChunkOffset(t *testing.T) { + ctx := context.Background() + data := []byte("hello world") + + files.CacheBlockSize = 1 // 1 byte + + s, err := NewFilesStore(ctx, tests.NewMockRouter(make(map[string][]string))) + if err != nil { + t.Fatal(err) + } + + fWithChunkOffset := &file{ + Name: "test", + reader: remotetests.NewMockReader(data), + store: s.(*store), + chunkOffset: 4, + } + size, err := fWithChunkOffset.Fstat() + if err != nil { + t.Fatal(err) + } else if size != int64(11) { + t.Errorf("expected size %d, got %d", 11, size) + } + + // Read the first byte, should get an error. + buf := make([]byte, 1) + _, err = fWithChunkOffset.ReadAt(buf, 0) + if err == nil { + t.Fatalf("expected %v, got nil", errOnlySingleChunkAvailable) + } else if err != errOnlySingleChunkAvailable { + t.Fatalf("expected %v, got %v", errOnlySingleChunkAvailable, err) + } + + _, err = os.ReadFile(cache.Path + "/test/0") + if !strings.Contains(err.Error(), "no such file or directory") { + t.Fatalf("expected chunk file to not exist, got %v", err) + } + + // Read the allowed chunk. + n, err := fWithChunkOffset.ReadAt(buf, 4) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("expected to read %d bytes, got %d", 1, n) + } + if string(buf[0]) != "o" { + t.Errorf("expected to read %q, got %q", "o", string(buf[0])) + } + + chunkFile, err := os.ReadFile(cache.Path + "/test/4") + if err != nil { + t.Fatal(err) + } + if string(chunkFile) != "o" { + t.Errorf("expected chunk file to contain %q, got %q", "o", string(chunkFile)) + } +} + +func TestReadAt(t *testing.T) { + ctx := context.Background() + data := []byte("hello world") + + files.CacheBlockSize = 1 // 1 byte + + s, err := NewFilesStore(ctx, tests.NewMockRouter(make(map[string][]string))) + if err != nil { + t.Fatal(err) + } + + f := &file{ + Name: "test", + reader: remotetests.NewMockReader(data), + store: s.(*store), + } + size, err := f.Fstat() + if err != nil { + t.Fatal(err) + } else if size != int64(11) { + t.Errorf("expected size %d, got %d", 11, size) + } + + // Read the first byte. + buf := make([]byte, 1) + n, err := f.ReadAt(buf, 0) + if err != nil { + t.Fatal(err) + } else if n != 1 { + t.Errorf("expected to read %d byte, got %d", 1, n) + } + + chunkFile, err := os.ReadFile(cache.Path + "/test/0") + if err != nil { + t.Fatal(err) + } else if string(chunkFile) != "h" { + t.Errorf("expected chunk file to contain %q, got %q", "h", string(chunkFile)) + } + + // Read in the middle. + buf = make([]byte, 4) + n, err = f.ReadAt(buf, 3) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("expected to read %d bytes, got %d", 1, n) + } + if string(buf[0]) != "l" { + t.Errorf("expected to read %q, got %q", "l", string(buf[0])) + } +} + +func TestSeek(t *testing.T) { + ctx := context.Background() + + data := []byte("hello world") + + s, err := NewFilesStore(ctx, tests.NewMockRouter(make(map[string][]string))) + if err != nil { + t.Fatal(err) + } + + f := &file{ + Name: "test", + reader: remotetests.NewMockReader(data), + store: s.(*store), + } + size, err := f.Fstat() + if err != nil { + t.Fatal(err) + } else if size != int64(11) { + t.Errorf("expected size %d, got %d", 11, size) + } + + // Seek to the beginning. + c, err := f.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } else if c != 0 { + t.Errorf("expected cursor %d, got %d", 0, c) + } + + // Seek to the middle. + c, err = f.Seek(size/2, io.SeekStart) + if err != nil { + t.Fatal(err) + } else if c != size/2 { + t.Errorf("expected cursor %d, got %d", size/2, c) + } + + // Seek to the middle. + c, err = f.Seek(0, io.SeekCurrent) + if err != nil { + t.Fatal(err) + } else if c != size/2 { + t.Errorf("expected cursor %d, got %d", size/2, c) + } + + // Seek to the end. + c, err = f.Seek(size, io.SeekStart) + if err != nil { + t.Fatal(err) + } else if c != size { + t.Errorf("expected cursor %d, got %d", size, c) + } + + // Seek to the end. + c, err = f.Seek(0, io.SeekEnd) + if err != nil { + t.Fatal(err) + } else if c != size { + t.Errorf("expected cursor %d, got %d", size, c) + } +} + +func TestFstat(t *testing.T) { + ctx := context.Background() + + data, err := randomBytesN(100) + if err != nil { + t.Fatal(err) + } + + s, err := NewFilesStore(ctx, tests.NewMockRouter(make(map[string][]string))) + if err != nil { + t.Fatal(err) + } + + f := &file{ + Name: "test", + reader: remotetests.NewMockReader(data), + store: s.(*store), + } + + size, err := f.Fstat() + if err != nil { + t.Fatal(err) + } else if size != int64(100) { + t.Errorf("expected size %d, got %d", 100, size) + } + + f = &file{ + Name: "test2", + reader: remotetests.NewMockReader(data), + store: s.(*store), + chunkOffset: 14, + } + + size, err = f.Fstat() + if err != nil { + t.Fatal(err) + } else if size != int64(100) { + t.Errorf("expected size %d, got %d", 100, size) + } +} + +func randomBytesN(n int) ([]byte, error) { + b := make([]byte, n) + _, err := rand.Read(b) + if err != nil { + return nil, err + } + return b, nil +} diff --git a/internal/files/store/interface.go b/internal/files/store/interface.go new file mode 100644 index 0000000..10602e6 --- /dev/null +++ b/internal/files/store/interface.go @@ -0,0 +1,48 @@ +package store + +import ( + "time" + + "github.com/gin-gonic/gin" + "github.com/opencontainers/go-digest" +) + +// FilesStore describes a store for files. +type FilesStore interface { + // Key tries to find the cache key for the requested content or returns empty. + Key(c *gin.Context) (key string, d digest.Digest, err error) + + // Open opens the requested file and starts prefetching it. It also returns the size of the file. + Open(c *gin.Context) (File, error) + + // Subscribe returns a channel that will be notified when a blob is added to the store. + Subscribe() chan string +} + +// File is an abstraction for a file that can be read from this store. +// It is similar to os.File. +type File interface { + // Seek sets the current file offset. + Seek(offset int64, whence int) (int64, error) + + // Fstat returns the size of the file. + Fstat() (int64, error) + + // Read reads up to len(p) bytes into p. It returns the number of bytes read (0 <= n <= len(p)) and any error encountered. + Read(p []byte) (n int, err error) + + // ReadAt reads len(p) bytes from the File starting at byte offset off. It returns the number of bytes read and the error, if any. + ReadAt(buff []byte, off int64) (int, error) +} + +var ( + // PrefetchWorkers is the number of workers that will be used to prefetch files. + // To disable prefetch, set this to 0. + PrefetchWorkers = 50 + + // ResolveRetries is the number of times to attempt resolving a key before giving up. + ResolveRetries = 3 + + // ResolveTimeout is the timeout for resolving a key. + ResolveTimeout = 20 * time.Millisecond +) diff --git a/internal/files/store/main_test.go b/internal/files/store/main_test.go new file mode 100644 index 0000000..a3c2cc4 --- /dev/null +++ b/internal/files/store/main_test.go @@ -0,0 +1,42 @@ +package store + +import ( + "crypto/rand" + "fmt" + "os" + "testing" + + "github.com/azure/peerd/internal/files/cache" +) + +func TestMain(m *testing.M) { + setup() + code := m.Run() + err := teardown() + if code == 0 && err != nil { + code = 42 + } + os.Exit(code) +} + +func setup() { + suf := newRandomStringN(10) + cache.Path += suf +} + +// teardown removes the cache directory. +func teardown() error { + if err := os.RemoveAll(cache.Path); err != nil { + return fmt.Errorf("failed to remove cache dir: %v --- %v", cache.Path, err) + } + + return nil +} + +// newRandomStringN creates a new random string of length n. +func newRandomStringN(n int) string { + randBytes := make([]byte, n/2) + _, _ = rand.Read(randBytes) + + return fmt.Sprintf("%x", randBytes) +} diff --git a/internal/files/store/mockstore.go b/internal/files/store/mockstore.go new file mode 100644 index 0000000..2354caa --- /dev/null +++ b/internal/files/store/mockstore.go @@ -0,0 +1,26 @@ +package store + +import ( + "context" + + "github.com/azure/peerd/internal/files/cache" + "github.com/azure/peerd/internal/routing" +) + +type MockStore struct { + *store +} + +var _ FilesStore = &MockStore{} + +func (m *MockStore) Cache() cache.Cache { + return m.store.cache +} + +func NewMockStore(ctx context.Context, r routing.Router) (*MockStore, error) { + s, err := NewFilesStore(ctx, r) + if err != nil { + return nil, err + } + return &MockStore{s.(*store)}, nil +} diff --git a/internal/files/store/store.go b/internal/files/store/store.go new file mode 100644 index 0000000..021ec99 --- /dev/null +++ b/internal/files/store/store.go @@ -0,0 +1,154 @@ +package store + +import ( + "context" + "os" + "strconv" + "strings" + "time" + + p2pcontext "github.com/azure/peerd/internal/context" + "github.com/azure/peerd/internal/files" + "github.com/azure/peerd/internal/files/cache" + "github.com/azure/peerd/internal/remote" + "github.com/azure/peerd/internal/routing" + "github.com/azure/peerd/pkg/urlparser" + "github.com/gin-gonic/gin" + "github.com/opencontainers/go-digest" + "github.com/rs/zerolog" +) + +// NewFilesStore creates a new store. +func NewFilesStore(ctx context.Context, r routing.Router) (FilesStore, error) { + fs := &store{ + cache: cache.New(ctx), + prefetchChan: make(chan prefetchableSegment, PrefetchWorkers), + prefetchable: PrefetchWorkers > 0, + router: r, + resolveRetries: ResolveRetries, + resolveTimeout: ResolveTimeout, + blobsChan: make(chan string, 1000), + parser: urlparser.New(), + } + + go func() { + <-ctx.Done() + err := r.Close() + l := zerolog.Ctx(ctx).Debug() + if err != nil { + l = zerolog.Ctx(ctx).Error().Err(err) + } + l.Msg("router close") + }() + + for i := 0; i < PrefetchWorkers; i++ { + go fs.prefetch() + } + + return fs, nil +} + +// prefetchableSegment describes a part of a file to prefetch. +type prefetchableSegment struct { + name string + offset int64 + count int + + reader remote.Reader +} + +// store describes a content store whose contents can come from disk or a remote source. +type store struct { + cache cache.Cache + prefetchable bool + prefetchChan chan prefetchableSegment + router routing.Router + resolveRetries int + resolveTimeout time.Duration + blobsChan chan string + parser urlparser.Parser +} + +var _ FilesStore = &store{} + +// Subscribe returns a channel that will be notified when a blob is added to the store. +func (s *store) Subscribe() chan string { + return s.blobsChan +} + +// Open opens the requested file and starts prefetching it. +func (s *store) Open(c *gin.Context) (File, error) { + + chunkKey := c.GetString(p2pcontext.FileChunkCtxKey) + tokens := strings.Split(chunkKey, files.FileChunkKeySep) + name := tokens[0] + alignedOff, _ := strconv.ParseInt(tokens[1], 10, 64) + + log := p2pcontext.Logger(c) + if p2pcontext.IsRequestFromAPeer(c) { + // This request came from a peer. Don't serve it unless we have the requested range cached. + if ok := s.cache.Exists(name, alignedOff); !ok { + log.Info().Str("name", name).Msg("peer request not cached") + return nil, os.ErrNotExist + } + } + + f := &file{ + Name: name, + store: s, + cur: 0, + size: 0, + reader: remote.NewReader(c, s.router, s.resolveRetries, s.resolveTimeout), + } + + if p2pcontext.IsRequestFromAPeer(c) { + // Ensure this file can only serve the requested chunk. + // This is to prevent infinite loops when a peer requests a file that is not cached. + f.chunkOffset = alignedOff + } + + fileSize, err := f.Fstat() // Fstat sets up the file size appropriately. + + if s.prefetchable { + f.prefetch(0, fileSize) + } + + return f, err +} + +// Key tries to find the cache key for the requested content or returns empty. +func (s *store) Key(c *gin.Context) (string, digest.Digest, error) { + log := p2pcontext.Logger(c) + + blobUrl := p2pcontext.BlobUrl(c) + d, err := s.parser.ParseDigest(blobUrl) + if err != nil { + log.Error().Err(err).Msg("store key") + } + + startIndex := int64(0) // Default to 0 for HEADs. + if c.Request.Method == "GET" { + startIndex, err = p2pcontext.RangeStartIndex(c.Request.Header.Get("Range")) + if err != nil { + return "", "", err + } + } + key := files.FileChunkKey(d.String(), startIndex, int64(files.CacheBlockSize)) + + log.Info().Str("digest", d.String()).Str("key", key).Msg("store key") + return key, d, err +} + +// prefetch prefetches files. +func (s *store) prefetch() { + for p := range s.prefetchChan { + if _, err := s.cache.GetOrCreate(p.name, p.offset, p.count, func() ([]byte, error) { + return files.FetchFile(p.reader, p.name, p.offset, p.count) + }); err != nil { + p.reader.Log().Error().Err(err).Str("name", p.name).Msg("prefetch failed") + } else { + // Advertise the chunk. + s.blobsChan <- files.FileChunkKey(p.name, p.offset, int64(files.CacheBlockSize)) + } + } +} diff --git a/internal/files/store/store_test.go b/internal/files/store/store_test.go new file mode 100644 index 0000000..de69b29 --- /dev/null +++ b/internal/files/store/store_test.go @@ -0,0 +1,135 @@ +package store + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + + p2pcontext "github.com/azure/peerd/internal/context" + "github.com/azure/peerd/internal/files" + "github.com/azure/peerd/internal/routing/tests" + "github.com/gin-gonic/gin" + "github.com/opencontainers/go-digest" +) + +var ( + hostAndPath = "https://avtakkartest.blob.core.windows.net/d18c7a64c5158179-ff8cb2f639ff44879c12c94361a746d0-782b855128//docker/registry/v2/blobs/sha256/d1/d18c7a64c5158179bdee531a663c5b487de57ff17cff3af29a51c7e70b491d9d/data" + query = "?se=2023-09-20T01%3A14%3A49Z&sig=m4Cr%2BYTZHZQlN5LznY7nrTQ4LCIx2OqnDDM3Dpedbhs%3D&sp=r&spr=https&sr=b&sv=2018-03-28®id=01031d61e1024861afee5d512651eb9f" + u = hostAndPath + query +) + +func TestOpenP2p(t *testing.T) { + // Create a new request with a URL that has a query string. + req, err := http.NewRequest("GET", "http://127.0.0.1:5000/blobs/"+u, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Range", fmt.Sprintf("bytes=%v-%v", files.CacheBlockSize, files.CacheBlockSize+172)) + req.Header.Set(p2pcontext.P2PHeaderKey, "true") + + expD := "sha256:d18c7a64c5158179bdee531a663c5b487de57ff17cff3af29a51c7e70b491d9d" + expK := fmt.Sprintf("%v%v%v", expD, files.FileChunkKeySep, files.CacheBlockSize) + + // Create a new context with the request. + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = req + ctx.Params = []gin.Param{ + {Key: "url", Value: hostAndPath}, + } + ctx.Set(p2pcontext.FileChunkCtxKey, expK) + + PrefetchWorkers = 0 // turn off prefetching + s, err := NewFilesStore(context.Background(), tests.NewMockRouter(make(map[string][]string))) + if err != nil { + t.Fatal(err) + } + + _, err = s.Open(ctx) + if err != os.ErrNotExist { + t.Errorf("expected %v, got %v", os.ErrNotExist, err) + } +} + +func TestOpenNonP2p(t *testing.T) { + // Create a new request with a URL that has a query string. + req, err := http.NewRequest("GET", "http://127.0.0.1:5000/blobs/"+u, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Range", fmt.Sprintf("bytes=%v-%v", files.CacheBlockSize, files.CacheBlockSize+172)) + + expD := "sha256:d18c7a64c5158179bdee531a663c5b487de57ff17cff3af29a51c7e70b491d9d" + expK := fmt.Sprintf("%v%v%v", expD, files.FileChunkKeySep, files.CacheBlockSize) + + // Create a new context with the request. + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = req + ctx.Params = []gin.Param{ + {Key: "url", Value: hostAndPath}, + } + ctx.Set(p2pcontext.FileChunkCtxKey, expK) + + PrefetchWorkers = 0 // turn off prefetching + s, err := NewMockStore(context.Background(), tests.NewMockRouter(make(map[string][]string))) + if err != nil { + t.Fatal(err) + } + + s.Cache().PutSize(expD, 200) + + _, err = s.Open(ctx) + if err != nil { + t.Fatal(err) + } +} + +func TestKey(t *testing.T) { + // Create a new request with a URL that has a query string. + req, err := http.NewRequest("GET", "http://127.0.0.1:5000/blobs/"+u, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Range", fmt.Sprintf("bytes=%v-%v", files.CacheBlockSize, files.CacheBlockSize+172)) + + expD := "sha256:d18c7a64c5158179bdee531a663c5b487de57ff17cff3af29a51c7e70b491d9d" + expK := fmt.Sprintf("%v_%v", expD, files.CacheBlockSize) + + // Create a new context with the request. + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = req + ctx.Params = []gin.Param{ + {Key: "url", Value: hostAndPath}, + } + + s, err := NewFilesStore(context.Background(), tests.NewMockRouter(make(map[string][]string))) + if err != nil { + t.Fatal(err) + } + + k, d, err := s.Key(ctx) + if err != nil { + t.Fatal(err) + } + + if k != expK { + t.Errorf("expected key %s, got %s", expK, k) + } + + if d != digest.Digest(expD) { + t.Errorf("expected digest %s, got %s", expD, d) + } +} + +func TestSubscribe(t *testing.T) { + s, err := NewFilesStore(context.Background(), tests.NewMockRouter(make(map[string][]string))) + if err != nil { + t.Fatal(err) + } + ch := s.Subscribe() + if ch == nil { + t.Fatal("expected channel, got nil") + } +} diff --git a/internal/handlers/files/handler.go b/internal/handlers/files/handler.go new file mode 100644 index 0000000..aff1d02 --- /dev/null +++ b/internal/handlers/files/handler.go @@ -0,0 +1,82 @@ +package handlers + +import ( + "context" + "net/http" + "os" + "time" + + p2pcontext "github.com/azure/peerd/internal/context" + "github.com/azure/peerd/internal/files/store" + "github.com/azure/peerd/internal/metrics" + "github.com/gin-gonic/gin" +) + +// FilesHandler describes a handler for files. +type FilesHandler struct { + store store.FilesStore +} + +var _ gin.HandlerFunc = (&FilesHandler{}).Handle + +// Handle handles a request for a file. +func (h *FilesHandler) Handle(c *gin.Context) { + log := p2pcontext.Logger(c).With().Str("blob", p2pcontext.BlobUrl(c)).Bool("p2p", p2pcontext.IsRequestFromAPeer(c)).Logger() + log.Debug().Msg("files handler start") + s := time.Now() + defer func() { + dur := time.Since(s) + metrics.Global.RecordRequest(c.Request.Method, "files", float64(dur.Milliseconds())) + log.Debug().Dur("duration", dur).Msg("files handler stop") + }() + + err := h.fill(c) + if err != nil { + log.Debug().Err(err).Msg("failed to fill context") + // nolint + c.AbortWithError(http.StatusBadRequest, err) + return + } + + f, err := h.store.Open(c) + if err == os.ErrNotExist { + c.AbortWithStatus(http.StatusNotFound) + return + } + if err != nil { + // nolint + c.AbortWithError(http.StatusInternalServerError, err) + return + } + + w := c.Writer + + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Del("Content-Length") + w.Header().Set(p2pcontext.NodeHeaderKey, p2pcontext.NodeName) + w.Header().Set(p2pcontext.CorrelationHeaderKey, c.GetString(p2pcontext.CorrelationIdCtxKey)) + + http.ServeContent(w, c.Request, "file", time.Now(), f) +} + +// fill fills the context with handler specific information. +func (h *FilesHandler) fill(c *gin.Context) error { + c.Set("handler", "files") + + key, d, err := h.store.Key(c) + if err != nil { + return err + } + + c.Set(p2pcontext.DigestCtxKey, d.String()) + c.Set(p2pcontext.FileChunkCtxKey, key) + c.Set(p2pcontext.BlobUrlCtxKey, p2pcontext.BlobUrl(c)) + c.Set(p2pcontext.BlobRangeCtxKey, c.Request.Header.Get("Range")) + + return nil +} + +// New creates a new files handler. +func New(ctx context.Context, fs store.FilesStore) *FilesHandler { + return &FilesHandler{fs} +} diff --git a/internal/handlers/files/handler_test.go b/internal/handlers/files/handler_test.go new file mode 100644 index 0000000..5c2e8ab --- /dev/null +++ b/internal/handlers/files/handler_test.go @@ -0,0 +1,152 @@ +package handlers + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + + p2pcontext "github.com/azure/peerd/internal/context" + "github.com/azure/peerd/internal/files" + "github.com/azure/peerd/internal/files/store" + "github.com/azure/peerd/internal/routing/tests" + "github.com/gin-gonic/gin" +) + +var ( + hostAndPath = "https://avtakkartest.blob.core.windows.net/d18c7a64c5158179-ff8cb2f639ff44879c12c94361a746d0-782b855128//docker/registry/v2/blobs/sha256/d1/d18c7a64c5158179bdee531a663c5b487de57ff17cff3af29a51c7e70b491d9d/data" + query = "?se=2023-09-20T01%3A14%3A49Z&sig=m4Cr%2BYTZHZQlN5LznY7nrTQ4LCIx2OqnDDM3Dpedbhs%3D&sp=r&spr=https&sr=b&sv=2018-03-28®id=01031d61e1024861afee5d512651eb9f" + u = hostAndPath + query +) + +func TestPartialContentResponseInP2PMode(t *testing.T) { + files.CacheBlockSize = 10 + // Create a new request with a URL that has a query string. + req, err := http.NewRequest("GET", "http://127.0.0.1:5000/blobs/"+u, nil) + if err != nil { + t.Fatal(err) + } + expRange := fmt.Sprintf("bytes=%v-%v", 12, 100) + req.Header.Set("Range", expRange) + req.Header.Set(p2pcontext.P2PHeaderKey, "true") + + expD := "sha256:d18c7a64c5158179bdee531a663c5b487de57ff17cff3af29a51c7e70b491d9d" + + // Create a new context with the request. + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = req + ctx.Params = []gin.Param{ + {Key: "url", Value: hostAndPath}, + } + + store.PrefetchWorkers = 0 // turn off prefetching + s, err := store.NewMockStore(context.Background(), tests.NewMockRouter(make(map[string][]string))) + if err != nil { + t.Fatal(err) + } + + h := New(context.Background(), s) + + // Write the chunk file. + content := newRandomStringN(10) + + s.Cache().PutSize(expD, 200) + // nolint:errcheck + s.Cache().GetOrCreate(expD, 10, 10, func() ([]byte, error) { + return []byte(content), nil + }) + + h.Handle(ctx) + resp := recorder.Result() + + if resp.StatusCode != http.StatusPartialContent { + t.Errorf("expected %v, got %v", http.StatusOK, ctx.Writer.Status()) + } + + ret, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if string(ret) != content[2:] { + t.Errorf("expected %v, got %v", content[2:], ret) + } +} + +func TestNotFoundInP2PMode(t *testing.T) { + // Create a new request with a URL that has a query string. + req, err := http.NewRequest("GET", "http://127.0.0.1:5000/blobs/"+u, nil) + if err != nil { + t.Fatal(err) + } + expRange := fmt.Sprintf("bytes=%v-%v", files.CacheBlockSize, files.CacheBlockSize+172) + req.Header.Set("Range", expRange) + req.Header.Set(p2pcontext.P2PHeaderKey, "true") + + // Create a new context with the request. + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = req + ctx.Params = []gin.Param{ + {Key: "url", Value: hostAndPath}, + } + + store.PrefetchWorkers = 0 // turn off prefetching + s, err := store.NewFilesStore(context.Background(), tests.NewMockRouter(make(map[string][]string))) + if err != nil { + t.Fatal(err) + } + + h := New(context.Background(), s) + + h.Handle(ctx) + if ctx.Writer.Status() != http.StatusNotFound { + t.Errorf("expected %v, got %v", http.StatusNotFound, ctx.Writer.Status()) + } +} + +func TestFill(t *testing.T) { + // Create a new request with a URL that has a query string. + req, err := http.NewRequest("GET", "http://127.0.0.1:5000/blobs/"+u, nil) + if err != nil { + t.Fatal(err) + } + expRange := fmt.Sprintf("bytes=%v-%v", files.CacheBlockSize, files.CacheBlockSize+172) + req.Header.Set("Range", expRange) + req.Header.Set(p2pcontext.P2PHeaderKey, "true") + + expD := "sha256:d18c7a64c5158179bdee531a663c5b487de57ff17cff3af29a51c7e70b491d9d" + expK := fmt.Sprintf("%v_%v", expD, files.CacheBlockSize) + + // Create a new context with the request. + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = req + ctx.Params = []gin.Param{ + {Key: "url", Value: hostAndPath}, + } + + store.PrefetchWorkers = 0 // turn off prefetching + s, err := store.NewFilesStore(context.Background(), tests.NewMockRouter(make(map[string][]string))) + if err != nil { + t.Fatal(err) + } + + h := New(context.Background(), s) + + err = h.fill(ctx) + if err != nil { + t.Fatal(err) + } + if ctx.GetString(p2pcontext.FileChunkCtxKey) != expK { + t.Errorf("expected %v, got %v", expK, ctx.GetString(p2pcontext.FileChunkCtxKey)) + } + + if ctx.GetString(p2pcontext.BlobRangeCtxKey) != expRange { + t.Errorf("expected %v, got %v", expRange, ctx.GetString(p2pcontext.BlobRangeCtxKey)) + } + + if ctx.GetString(p2pcontext.BlobUrlCtxKey) != hostAndPath+query { + t.Errorf("expected %v, got %v", hostAndPath+query, ctx.GetString(p2pcontext.BlobUrlCtxKey)) + } +} diff --git a/internal/handlers/files/main_test.go b/internal/handlers/files/main_test.go new file mode 100644 index 0000000..925cc4b --- /dev/null +++ b/internal/handlers/files/main_test.go @@ -0,0 +1,42 @@ +package handlers + +import ( + "crypto/rand" + "fmt" + "os" + "testing" + + "github.com/azure/peerd/internal/files/cache" +) + +func TestMain(m *testing.M) { + setup() + code := m.Run() + err := teardown() + if code == 0 && err != nil { + code = 42 + } + os.Exit(code) +} + +func setup() { + suf := newRandomStringN(10) + cache.Path += suf +} + +// teardown removes the cache directory. +func teardown() error { + if err := os.RemoveAll(cache.Path); err != nil { + return fmt.Errorf("failed to remove cache dir: %v --- %v", cache.Path, err) + } + + return nil +} + +// newRandomStringN creates a new random string of length n. +func newRandomStringN(n int) string { + randBytes := make([]byte, n/2) + _, _ = rand.Read(randBytes) + + return fmt.Sprintf("%x", randBytes) +} diff --git a/internal/handlers/root.go b/internal/handlers/root.go new file mode 100644 index 0000000..b2e303e --- /dev/null +++ b/internal/handlers/root.go @@ -0,0 +1,107 @@ +package handlers + +import ( + "context" + "net/http" + "time" + + p2pcontext "github.com/azure/peerd/internal/context" + filesStore "github.com/azure/peerd/internal/files/store" + filesHandler "github.com/azure/peerd/internal/handlers/files" + ociHandler "github.com/azure/peerd/internal/handlers/v2" + "github.com/azure/peerd/internal/routing" + "github.com/azure/peerd/pkg/containerd" + "github.com/gin-gonic/gin" + "github.com/rs/zerolog" +) + +var fh *filesHandler.FilesHandler +var v2h *ociHandler.V2Handler + +// Server creates a new HTTP server. +func Handler(ctx context.Context, r routing.Router, containerdStore containerd.Store, fs filesStore.FilesStore) (http.Handler, error) { + var err error + fh = filesHandler.New(ctx, fs) + + v2h, err = ociHandler.New(ctx, r, containerdStore) + if err != nil { + return nil, err + } + + engine := newEngine(ctx) + + engine.HEAD("/blobs/*url", fileHandler) + engine.GET("/blobs/*url", fileHandler) + + engine.HEAD("/v2", v2Handler) + engine.GET("/v2", v2Handler) + engine.HEAD("/v2/:repo/manifests/:reference", v2Handler) + engine.GET("/v2/:repo/manifests/:reference", v2Handler) + engine.HEAD("/v2/:repo/blobs/:digest", v2Handler) + engine.GET("/v2/:repo/blobs/:digest", v2Handler) + + return engine, nil +} + +// newEngine creates a new gin engine. +func newEngine(ctx context.Context) *gin.Engine { + gin.SetMode(gin.ReleaseMode) + engine := gin.New() + + baseLog := zerolog.Ctx(ctx) + + engine.Use(func(c *gin.Context) { + p2pcontext.FillCorrelationId(c) + c.Set(p2pcontext.LoggerCtxKey, baseLog) + + l := p2pcontext.Logger(c) + l.Debug().Msg("request start") + s := time.Now() + + c.Next() + + status := c.Writer.Status() + event := l.Info() + if status >= 400 && status < 500 { + event = l.Warn() + } else if status >= 500 { + event = l.Error() + } + + if c.Errors != nil { + errs := []error{} + for _, e := range c.Errors { + errs = append(errs, e.Err) + } + event = event.Errs("error", errs) + } + + event.Dur("duration", time.Duration(time.Since(s).Seconds())).Str("method", c.Request.Method).Int("status", status).Msg("request served") + }) + + engine.Use(gin.Recovery()) + return engine +} + +// fileHandler is a handler function for the /blob API +// @Summary Get a blob by URL +// @Param url path string true "The URL of the blob" +// @Success 200 {string} string "The blob content" +// @Failure 404 {string} string "Not Found" +// @Router /blobs/{url} [get] +func fileHandler(c *gin.Context) { + fh.Handle(c) +} + +// v2Handler is a handler function for the /v2 API +// @Summary Get a manifest or a blob by repository and reference or digest +// @Param repo path string true "The repository name" +// @Param reference path string false "The reference of the manifest" +// @Param digest path string false "The digest of the blob" +// @Success 200 {object} map[string]string "The manifest or blob information" +// @Failure 404 {string} string "Not Found" +// @Router /v2/{repo}/manifests/{reference} [get] +// @Router /v2/{repo}/blobs/{digest} [get] +func v2Handler(c *gin.Context) { + v2h.Handle(c) +} diff --git a/internal/handlers/v2/handler.go b/internal/handlers/v2/handler.go new file mode 100644 index 0000000..8a35f56 --- /dev/null +++ b/internal/handlers/v2/handler.go @@ -0,0 +1,93 @@ +package handlers + +import ( + "context" + "net/http" + "path" + "time" + + p2pcontext "github.com/azure/peerd/internal/context" + "github.com/azure/peerd/internal/metrics" + "github.com/azure/peerd/internal/oci" + "github.com/azure/peerd/internal/oci/distribution" + "github.com/azure/peerd/internal/routing" + "github.com/azure/peerd/pkg/containerd" + "github.com/gin-gonic/gin" +) + +// V2Handler describes a handler for OCI content. +type V2Handler struct { + mirror *oci.Mirror + registry *oci.Registry +} + +var _ gin.HandlerFunc = (&V2Handler{}).Handle + +// Handle handles a request for a file. +func (h *V2Handler) Handle(c *gin.Context) { + l := p2pcontext.Logger(c).With().Str("blob", c.GetString(p2pcontext.BlobUrlCtxKey)).Bool("p2p", p2pcontext.IsRequestFromAPeer(c)).Logger() + l.Debug().Msg("v2 handler start") + s := time.Now() + defer func() { + dur := time.Since(s) + metrics.Global.RecordRequest(c.Request.Method, "oci", dur.Seconds()) + l.Debug().Dur("duration", dur).Str("ns", c.GetString(p2pcontext.NamespaceCtxKey)).Str("ref", c.GetString(p2pcontext.ReferenceCtxKey)).Str("digest", c.GetString(p2pcontext.DigestCtxKey)).Msg("v2 handler stop") + }() + + p := path.Clean(c.Request.URL.Path) + if p == "/v2" || p == "/v2/" { + if c.Request.Method != http.MethodGet && c.Request.Method != http.MethodHead { + c.Status(http.StatusNotFound) + return + } + c.Status(http.StatusOK) + return + } + + err := h.fill(c) + if err != nil { + l.Debug().Err(err).Msg("failed to fill context") + // nolint + c.AbortWithError(http.StatusBadRequest, err) + return + } + + if p2pcontext.IsRequestFromAPeer(c) { + h.registry.Handle(c) + return + } else { + h.mirror.Handle(c) + return + } +} + +// fill fills the context with handler specific information. +func (h *V2Handler) fill(c *gin.Context) error { + c.Set("handler", "v2") + + ns := c.Query("ns") + if ns == "" { + ns = "docker.io" + } + + c.Set(p2pcontext.NamespaceCtxKey, ns) + + ref, dgst, refType, err := distribution.ParsePathComponents(ns, c.Request.URL.Path) + if err != nil { + return err + } + + c.Set(p2pcontext.ReferenceCtxKey, ref) + c.Set(p2pcontext.DigestCtxKey, dgst.String()) + c.Set(p2pcontext.RefTypeCtxKey, refType) + + return nil +} + +// New creates a new OCI content handler. +func New(ctx context.Context, router routing.Router, containerdStore containerd.Store) (*V2Handler, error) { + return &V2Handler{ + mirror: oci.NewMirror(router), + registry: oci.NewRegistry(containerdStore), + }, nil +} diff --git a/internal/k8s/events/events.go b/internal/k8s/events/events.go new file mode 100644 index 0000000..ba1e74b --- /dev/null +++ b/internal/k8s/events/events.go @@ -0,0 +1,118 @@ +package events + +import ( + "context" + + p2pcontext "github.com/azure/peerd/internal/context" + "github.com/azure/peerd/pkg/k8s" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + typedv1core "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" +) + +type eventsRecorderCtxKeyType struct{} + +var ( + eventsRecorderCtxKey = eventsRecorderCtxKeyType{} +) + +// NewRecorder creates a new event recorder. +func NewRecorder(ctx context.Context) (EventRecorder, error) { + clientset, err := k8s.NewKubernetesInterface(p2pcontext.KubeConfigPath) + if err != nil { + return nil, err + } + + inPod := false + _, err = rest.InClusterConfig() // Assume run in a Pod or an environment with appropriate env variables set + if err == nil { + inPod = true + } + + var objRef *v1.ObjectReference + if !inPod { + node, err := clientset.CoreV1().Nodes().Get(ctx, p2pcontext.NodeName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + objRef = &v1.ObjectReference{ + Kind: "Node", + Name: node.Name, + UID: node.UID, + APIVersion: node.APIVersion, + } + } else { + pod, err := clientset.CoreV1().Pods(p2pcontext.Namespace).Get(ctx, p2pcontext.NodeName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + objRef = &v1.ObjectReference{ + Kind: "Pod", + Name: pod.Name, + UID: pod.UID, + APIVersion: pod.APIVersion, + } + } + + broadcaster := record.NewBroadcaster() + broadcaster.StartStructuredLogging(4) + broadcaster.StartRecordingToSink(&typedv1core.EventSinkImpl{Interface: clientset.CoreV1().Events("")}) + + return &eventRecorder{ + recorder: broadcaster.NewRecorder( + scheme.Scheme, + v1.EventSource{}, + ), + nodeRef: objRef, + }, nil +} + +// WithContext returns a new context with an event recorder. +func WithContext(ctx context.Context) (context.Context, error) { + er, err := NewRecorder(ctx) + if err != nil { + return nil, err + } + + return context.WithValue(ctx, eventsRecorderCtxKey, er), nil +} + +// FromContext returns the event recorder from the context. +func FromContext(ctx context.Context) EventRecorder { + return ctx.Value(eventsRecorderCtxKey).(EventRecorder) +} + +type eventRecorder struct { + recorder record.EventRecorder + nodeRef *v1.ObjectReference +} + +// Active should be called to indicate that the node is active in the cluster. +func (er *eventRecorder) Active() { + er.recorder.Eventf(er.nodeRef, v1.EventTypeNormal, "P2PActive", "P2P proxy is active on node %s", er.nodeRef.Name) +} + +// Connected should be called to indicate that the node is connected to the cluster. +func (er *eventRecorder) Connected() { + er.recorder.Eventf(er.nodeRef, v1.EventTypeNormal, "P2PConnected", "P2P proxy is connected to cluster on node %s", er.nodeRef.Name) +} + +// Disconnected should be called to indicate that the node is disconnected from the cluster. +func (er *eventRecorder) Disconnected() { + er.recorder.Eventf(er.nodeRef, v1.EventTypeWarning, "P2PDisconnected", "P2P proxy is disconnected from cluster on node %s", er.nodeRef.Name) +} + +// Failed should be called to indicate that the node has failed. +func (er *eventRecorder) Failed() { + er.recorder.Eventf(er.nodeRef, v1.EventTypeWarning, "P2PFailed", "P2P proxy failed on node %s", er.nodeRef.Name) +} + +// Initializing should be called to indicate that the node is initializing. +func (er *eventRecorder) Initializing() { + er.recorder.Eventf(er.nodeRef, v1.EventTypeNormal, "P2PInitializing", "P2P proxy is initializing on node %s", er.nodeRef.Name) +} + +var _ EventRecorder = &eventRecorder{} diff --git a/internal/k8s/events/events_test.go b/internal/k8s/events/events_test.go new file mode 100644 index 0000000..4905f00 --- /dev/null +++ b/internal/k8s/events/events_test.go @@ -0,0 +1,70 @@ +package events + +import ( + "context" + "testing" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" +) + +func TestExpectedEvents(t *testing.T) { + er := &eventRecorder{ + recorder: &testRecorder{t}, + nodeRef: &v1.ObjectReference{ + Kind: "Node", + Name: "node-name", + UID: "node.UID", + APIVersion: "node.APIVersion", + }, + } + + er.Active() + er.Connected() + er.Disconnected() + er.Initializing() + er.Failed() +} + +func TestFromContext(t *testing.T) { + er := &eventRecorder{ + recorder: &testRecorder{t}, + nodeRef: &v1.ObjectReference{ + Kind: "Node", + Name: "node-name", + UID: "node.UID", + APIVersion: "node.APIVersion", + }, + } + + ctx := context.WithValue(context.Background(), eventsRecorderCtxKey, er) + + er2 := FromContext(ctx) + if er != er2 { + t.Errorf("expected event recorders to match") + } +} + +type testRecorder struct { + t *testing.T +} + +// AnnotatedEventf implements record.EventRecorder. +func (*testRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype string, reason string, messageFmt string, args ...interface{}) { + panic("unimplemented") +} + +// Event implements record.EventRecorder. +func (*testRecorder) Event(object runtime.Object, eventtype string, reason string, message string) { + panic("unimplemented") +} + +// Eventf implements record.EventRecorder. +func (t *testRecorder) Eventf(object runtime.Object, eventtype string, reason string, messageFmt string, args ...interface{}) { + if reason != "P2PActive" && reason != "P2PConnected" && reason != "P2PDisconnected" && reason != "P2PInitializing" && reason != "P2PFailed" { + t.t.Errorf("unexpected reason: %s", reason) + } +} + +var _ record.EventRecorder = &testRecorder{} diff --git a/internal/k8s/events/interface.go b/internal/k8s/events/interface.go new file mode 100644 index 0000000..61b1c33 --- /dev/null +++ b/internal/k8s/events/interface.go @@ -0,0 +1,19 @@ +package events + +// EventRecorder can be used to record various event. +type EventRecorder interface { + // Initializing should be called to indicate that the node is initializing. + Initializing() + + // Connected should be called to indicate that the node is connected to the cluster. + Connected() + + // Active should be called to indicate that the node is active in the cluster. + Active() + + // Disconnected should be called to indicate that the node is disconnected from the cluster. + Disconnected() + + // Failed should be called to indicate that the node has failed. + Failed() +} diff --git a/internal/math/math.go b/internal/math/math.go new file mode 100644 index 0000000..186293e --- /dev/null +++ b/internal/math/math.go @@ -0,0 +1,62 @@ +package math + +import ( + "crypto/rand" + "math/big" + "sort" +) + +// PercentilesFloat64Reverse calculates the percentile of a slice of floats in reverse order. +// NOTE: The unit of each value of xs is 'bits' and the result is 'Mb'. +func PercentilesFloat64Reverse(xs []float64, ps ...float64) []float64 { + if len(xs) == 0 { + return nil + } + + // Sort in descending order + sort.Sort(ReverseFloat64Slice(xs)) + results := []float64{} + + for _, p := range ps { + if p < 0 { + p = 0 + } + if p > 1 { + p = 1 + } + + i := int(float64(len(xs)-1) * p) + results = append(results, xs[i]/1024/1024) + } + + return results +} + +// RandomizedGroups groups the given collection randomly into groups of size n +func RandomizedGroups(s []string, n int) [][]string { + groups := make([][]string, 0) + numGroups := len(s) / n + if len(s)%n != 0 { + numGroups++ + } + + // Shuffle the slice + for i := range s { + j, err := rand.Int(rand.Reader, big.NewInt(int64(i+1))) + if err != nil { + panic(err) + } + s[i], s[j.Int64()] = s[j.Int64()], s[i] + } + + // Create groups + for i := 0; i < numGroups; i++ { + group := make([]string, 0) + for j := 0; j < n && i*n+j < len(s); j++ { + group = append(group, s[i*n+j]) + } + groups = append(groups, group) + } + + return groups +} diff --git a/internal/math/math_test.go b/internal/math/math_test.go new file mode 100644 index 0000000..e3da86c --- /dev/null +++ b/internal/math/math_test.go @@ -0,0 +1,56 @@ +package math + +import "testing" + +func TestRandomizedGroups(t *testing.T) { + s := []string{"a", "b", "c", "d", "e", "f", "g", "h"} + n := 2 + + got := RandomizedGroups(s, n) + + if len(got) != 4 { + t.Errorf("expected: %v, got: %v", 4, len(got)) + } + + for _, group := range got { + if len(group) != 2 { + t.Errorf("expected: %v, got: %v", 2, len(group)) + } + + for _, item := range group { + found := false + for i, element := range s { + if item == element { + found = true + s[i] = "-1" + break + } + } + if !found { + t.Errorf("element %v not found in original slice", item) + } + } + } +} + +func TestPercentilesFloat64Reverse(t *testing.T) { + xs := []float64{5 * 1024 * 1024, 2 * 1024 * 1024, 1 * 1024 * 1024, 3 * 1024 * 1024, 4 * 1024 * 1024} + ps := []float64{0.5, 0.9, 1.0} + + got := PercentilesFloat64Reverse(xs, ps...) + if len(got) != 3 { + t.Errorf("expected length: %v, got: %v", 3, len(got)) + } + + if got[0] != 3 { + t.Errorf("expected p50: %v, got: %v ... %v", 3, got[0], got) + } + + if got[1] != 2 { + t.Errorf("expected p100: %v, got: %v ... %v", 2, got[1], got) + } + + if got[2] != 1 { + t.Errorf("expected p100: %v, got: %v ... %v", 1, got[2], got) + } +} diff --git a/internal/math/reverse.go b/internal/math/reverse.go new file mode 100644 index 0000000..61171f8 --- /dev/null +++ b/internal/math/reverse.go @@ -0,0 +1,24 @@ +package math + +import "sort" + +// ReverseFloat64Slice is a type that implements the sort.Interface interface +// so that we can sort a slice of float64 in reverse order +type ReverseFloat64Slice []float64 + +var _ sort.Interface = ReverseFloat64Slice{} + +// Len returns the length of the slice +func (r ReverseFloat64Slice) Len() int { + return len(r) +} + +// Less returns true if the element at index i is greater than the element at index j +func (r ReverseFloat64Slice) Less(i, j int) bool { + return r[i] > r[j] +} + +// Swap swaps the elements at indexes i and j +func (r ReverseFloat64Slice) Swap(i, j int) { + r[i], r[j] = r[j], r[i] +} diff --git a/internal/math/reverse_test.go b/internal/math/reverse_test.go new file mode 100644 index 0000000..8c1f79c --- /dev/null +++ b/internal/math/reverse_test.go @@ -0,0 +1,57 @@ +package math + +import ( + "sort" + "testing" +) + +func TestReverseSort(t *testing.T) { + data := []float64{1, 2, 3, 4, 5} + + sort.Sort(ReverseFloat64Slice(data)) + + for i, v := range data { + + if v != float64(5-i) { + t.Errorf("expected: %v, got: %v", float64(5-i), v) + } + } +} + +func TestLen(t *testing.T) { + data := []float64{1, 2, 3, 4, 5} + + r := ReverseFloat64Slice(data) + + if r.Len() != 5 { + t.Errorf("expected: %v, got: %v", 5, r.Len()) + } +} + +func TestLess(t *testing.T) { + data := []float64{1, 2, 3, 4, 5} + + r := ReverseFloat64Slice(data) + res := r.Less(0, 1) + + if res != false { + t.Errorf("expected: %v, got: %v", false, res) + } + + res = r.Less(1, 0) + + if res != true { + t.Errorf("expected: %v, got: %v", true, res) + } +} + +func TestSwap(t *testing.T) { + data := []float64{1, 2, 3, 4, 5} + + r := ReverseFloat64Slice(data) + r.Swap(0, 1) + + if r[0] != 2 || r[1] != 1 { + t.Errorf("expected: %v, got: %v", []float64{2, 1}, r[:2]) + } +} diff --git a/internal/metrics/interface.go b/internal/metrics/interface.go new file mode 100644 index 0000000..cbf295c --- /dev/null +++ b/internal/metrics/interface.go @@ -0,0 +1,19 @@ +package metrics + +// Metrics defines an interface to collect p2p metrics. +type Metrics interface { + // RecordRequest records the time it takes to process a request. + RecordRequest(method, handler string, duration float64) + + // RecordPeerDiscovery records the time it takes to discover a peer. + RecordPeerDiscovery(ip string, duration float64) + + // RecordPeerResponse records the time it takes for a peer to respond for a key. + RecordPeerResponse(ip, key, op string, duration float64, count int64) + + // RecordUpstreamResponse records the time it takes for an upstream to respond for a key. + RecordUpstreamResponse(hostname, key, op string, duration float64, count int64) +} + +// Global is the global metrics collector. +var Global Metrics = NewMemoryMetrics() diff --git a/internal/metrics/main_test.go b/internal/metrics/main_test.go new file mode 100644 index 0000000..bc3e2d0 --- /dev/null +++ b/internal/metrics/main_test.go @@ -0,0 +1,56 @@ +package metrics + +import ( + "crypto/rand" + "fmt" + "os" + "testing" + "time" +) + +func TestMain(m *testing.M) { + p, err := setup() + if err != nil { + fmt.Printf("failed to setup test: %v", err) + os.Exit(42) + } + code := m.Run() + err = teardown(p) + if code == 0 && err != nil { + code = 42 + } + os.Exit(code) +} + +func setup() (string, error) { + suf := newRandomStringN(10) + Path = "./" + suf + + _, err := os.Create(Path) + if err != nil { + return "", err + } + + ReportInterval = 1 * time.Second + AggregationInterval = 20 * time.Millisecond + RetentionPeriod = 2 * time.Second + + return Path, nil +} + +// teardown removes the cache directory. +func teardown(path string) error { + if err := os.RemoveAll(path); err != nil { + return fmt.Errorf("failed to remove test file: %v --- %v", path, err) + } + + return nil +} + +// newRandomStringN creates a new random string of length n. +func newRandomStringN(n int) string { + randBytes := make([]byte, n/2) + _, _ = rand.Read(randBytes) + + return fmt.Sprintf("%x", randBytes) +} diff --git a/internal/metrics/memory.go b/internal/metrics/memory.go new file mode 100644 index 0000000..adfab58 --- /dev/null +++ b/internal/metrics/memory.go @@ -0,0 +1,118 @@ +package metrics + +import ( + "os" + "syscall" + "time" + + hmetrics "github.com/hashicorp/go-metrics" +) + +var ( + // Path is the default path to write metrics. + Path = "/var/log/p2pmetrics" + + // ReportInterval is the interval to report metrics. + ReportInterval = 3 * time.Minute + + // AggregationInterval is the interval to aggregate metrics. + AggregationInterval = 2 * time.Minute + + // RetentionPeriod is the retention period of metrics. + RetentionPeriod = 10 * time.Minute +) + +// memoryMetrics is a metrics collector that stores metrics in memory. +type memoryMetrics struct { + sink *hmetrics.InmemSink + + reportingInterval time.Duration + reportFilePath string +} + +// RecordPeerDiscovery records the time it takes to discover a peer. +func (m *memoryMetrics) RecordPeerDiscovery(ip string, duration float64) { + m.recordLatency(duration, ip, "discovery") +} + +// RecordPeerResponse records the time it takes for a peer to respond for a key. +func (m *memoryMetrics) RecordPeerResponse(ip, key, op string, duration float64, count int64) { + m.recordLatency(duration, ip, op) + m.recordBytes(count, ip, op) + + if duration > 0 { + m.recordSpeed(float64(count)/duration, ip, op) + } +} + +// RecordRequest records the time it takes to process a request. +func (m *memoryMetrics) RecordRequest(method string, handler string, duration float64) { + m.recordLatency(duration, "server", method+"_"+handler) +} + +// RecordUpstreamResponse records the time it takes for an upstream to respond for a key. +func (m *memoryMetrics) RecordUpstreamResponse(hostname, key, op string, duration float64, count int64) { + m.recordLatency(duration, hostname, op) + m.recordBytes(count, hostname, op) + + if duration > 0 { + m.recordSpeed(float64(count)/duration, hostname, op) + } +} + +// recordLatency records the time it takes to perform an operation. +func (m *memoryMetrics) recordLatency(duration float64, host, op string) { + m.sink.AddSample([]string{"latency", host, op}, float32(duration)) +} + +// recordSpeed records the speed of a download from a host. +func (m *memoryMetrics) recordSpeed(speed float64, host, op string) { + m.sink.AddSample([]string{"speed", host, op}, float32(speed)) +} + +// recordBytes records the number of bytes downloaded from a host. +func (m *memoryMetrics) recordBytes(bytes int64, host, op string) { + m.sink.AddSample([]string{"bytes", host, op}, float32(bytes)) +} + +var _ Metrics = &memoryMetrics{} + +// reportPeriodically reports the current metrics to a file every 5 minutes. +func (m *memoryMetrics) reportPeriodically() { + go func() { + ticker := time.NewTicker(m.reportingInterval) + defer ticker.Stop() + for range ticker.C { + f, err := os.OpenFile(m.reportFilePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) + if err == nil { + hmetrics.NewInmemSignal(m.sink, hmetrics.DefaultSignal, f) + + _ = syscall.Kill(os.Getpid(), syscall.SIGUSR1) + + // Wait for flush. + time.Sleep(20 * time.Millisecond) + + _ = f.Sync() + f.Close() + } + } + }() +} + +// NewMemoryMetrics returns a new memory metrics collector. +func NewMemoryMetrics() Metrics { + sink := hmetrics.NewInmemSink(AggregationInterval, RetentionPeriod) + + c := hmetrics.DefaultConfig("peerd") + c.EnableRuntimeMetrics = false + + _, err := hmetrics.NewGlobal(c, sink) + if err != nil { + panic(err) + } + + m := &memoryMetrics{sink, ReportInterval, Path} + m.reportPeriodically() + + return m +} diff --git a/internal/metrics/memory_test.go b/internal/metrics/memory_test.go new file mode 100644 index 0000000..dd908a3 --- /dev/null +++ b/internal/metrics/memory_test.go @@ -0,0 +1,49 @@ +package metrics + +import ( + "os" + "strings" + "testing" + "time" +) + +func TestMetricsWritten(t *testing.T) { + m := NewMemoryMetrics() + + m.RecordPeerDiscovery("10.0.0.1", 1.0) + m.RecordPeerDiscovery("10.0.0.3", 1.2) + m.RecordPeerDiscovery("10.0.0.2", 1.0) + + m.RecordPeerResponse("10.0.0.1", "key", "pread", 1.0, 15) + m.RecordPeerResponse("10.0.0.3", "key-a", "pread", 1.2, 10) + m.RecordPeerResponse("10.0.0.2", "key-b", "pread", 1.0, 1) + + m.RecordRequest("GET", "key", 1.0) + + m.RecordUpstreamResponse("upstream-a", "key-a", "pread", 1.2, 10) + + time.Sleep(ReportInterval + 300*time.Millisecond) + + contents, err := os.ReadFile(Path) + if err != nil { + t.Fatalf("failed to read file: %v", err) + } + + if len(contents) == 0 { + t.Fatalf("file is empty") + } + + s := string(contents) + + if !strings.Contains(s, "speed") { + t.Fatalf("file does not contain speed metric") + } + + if !strings.Contains(s, "bytes") { + t.Fatalf("file does not contain bytes metric") + } + + if !strings.Contains(s, "latency") { + t.Fatalf("file does not contain latency metric") + } +} diff --git a/internal/oci/distribution/v2.go b/internal/oci/distribution/v2.go new file mode 100644 index 0000000..9081e9d --- /dev/null +++ b/internal/oci/distribution/v2.go @@ -0,0 +1,45 @@ +package distribution + +import ( + "fmt" + "regexp" + + "github.com/opencontainers/go-digest" +) + +// ReferenceType is the type of reference - manifest or blob. +type ReferenceType string + +const ( + ReferenceTypeManifest = "Manifest" + ReferenceTypeBlob = "Blob" +) + +var ( + nameRegex = regexp.MustCompile(`([a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*)`) + tagRegex = regexp.MustCompile(`([a-zA-Z0-9_][a-zA-Z0-9._-]{0,127})`) + manifestRegexTag = regexp.MustCompile(`/v2/` + nameRegex.String() + `/manifests/` + tagRegex.String() + `$`) + manifestRegexDigest = regexp.MustCompile(`/v2/` + nameRegex.String() + `/manifests/(.*)`) + blobsRegexDigest = regexp.MustCompile(`/v2/` + nameRegex.String() + `/blobs/(.*)`) +) + +// ParsePathComponents parses the registry, digest and reference type from a distribution path. +func ParsePathComponents(registry, path string) (string, digest.Digest, ReferenceType, error) { + comps := manifestRegexTag.FindStringSubmatch(path) + if len(comps) == 6 { + if registry == "" { + return "", "", "", fmt.Errorf("registry parameter needs to be set for tag references") + } + ref := fmt.Sprintf("%s/%s:%s", registry, comps[1], comps[5]) + return ref, "", ReferenceTypeManifest, nil + } + comps = manifestRegexDigest.FindStringSubmatch(path) + if len(comps) == 6 { + return "", digest.Digest(comps[5]), ReferenceTypeManifest, nil + } + comps = blobsRegexDigest.FindStringSubmatch(path) + if len(comps) == 6 { + return "", digest.Digest(comps[5]), ReferenceTypeBlob, nil + } + return "", "", "", fmt.Errorf("distribution path could not be parsed") +} diff --git a/internal/oci/distribution/v2_test.go b/internal/oci/distribution/v2_test.go new file mode 100644 index 0000000..78c20da --- /dev/null +++ b/internal/oci/distribution/v2_test.go @@ -0,0 +1,55 @@ +package distribution + +import ( + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/require" +) + +func TestParsePathComponents(t *testing.T) { + tests := []struct { + name string + registry string + path string + expectedRef string + expectedDgst digest.Digest + expectedRefType ReferenceType + }{ + { + name: "valid manifest tag", + registry: "example.com", + path: "/v2/foo/bar/manifests/hello-world", + expectedRef: "example.com/foo/bar:hello-world", + expectedDgst: "", + expectedRefType: ReferenceTypeManifest, + }, + { + name: "valid blob digest", + registry: "docker.io", + path: "/v2/library/nginx/blobs/sha256:295c7be079025306c4f1d65997fcf7adb411c88f139ad1d34b537164aa060369", + expectedRef: "", + expectedDgst: digest.Digest("sha256:295c7be079025306c4f1d65997fcf7adb411c88f139ad1d34b537164aa060369"), + expectedRefType: ReferenceTypeBlob, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ref, dgst, refType, err := ParsePathComponents(tt.registry, tt.path) + require.NoError(t, err) + require.Equal(t, tt.expectedRef, ref) + require.Equal(t, tt.expectedDgst, dgst) + require.Equal(t, tt.expectedRefType, refType) + }) + } +} + +func TestParsePathComponentsInvalidPath(t *testing.T) { + _, _, _, err := ParsePathComponents("example.com", "/v2/xenitab/spegel/v0.0.1") + require.EqualError(t, err, "distribution path could not be parsed") +} + +func TestParsePathComponentsMissingRegistry(t *testing.T) { + _, _, _, err := ParsePathComponents("", "/v2/xenitab/spegel/manifests/v0.0.1") + require.EqualError(t, err, "registry parameter needs to be set for tag references") +} diff --git a/internal/oci/mirror.go b/internal/oci/mirror.go new file mode 100644 index 0000000..a76fb5b --- /dev/null +++ b/internal/oci/mirror.go @@ -0,0 +1,130 @@ +package oci + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httputil" + "net/url" + "time" + + p2pcontext "github.com/azure/peerd/internal/context" + "github.com/azure/peerd/internal/routing" + "github.com/azure/peerd/pkg/peernet" + "github.com/gin-gonic/gin" +) + +var ( + // ResolveRetries is the number of times to attempt resolving a key before giving up. + ResolveRetries = 3 + + // ResolveTimeout is the timeout for resolving a key. + ResolveTimeout = 1 * time.Second +) + +// Mirror is a handler that handles requests to this registry Mirror. +type Mirror struct { + resolveTimeout time.Duration + router routing.Router + resolveRetries int + + n peernet.Network +} + +var _ gin.HandlerFunc = (&Mirror{}).Handle + +// Handle handles a request to this registry mirror. +func (m *Mirror) Handle(c *gin.Context) { + key := c.GetString(p2pcontext.DigestCtxKey) + if key == "" { + key = c.GetString(p2pcontext.ReferenceCtxKey) + } + + l := p2pcontext.Logger(c).With().Str("handler", "mirror").Str("ref", key).Logger() + l.Debug().Msg("mirror handler start") + s := time.Now() + defer func() { + l.Debug().Dur("duration", time.Since(s)).Msg("mirror handler stop") + }() + + // Resolve mirror with the requested key + resolveCtx, cancel := context.WithTimeout(c, m.resolveTimeout) + defer cancel() + + if key == "" { + // nolint + c.AbortWithError(http.StatusInternalServerError, errors.New("neither digest nor reference provided")) + } + + peersChan, err := m.router.Resolve(resolveCtx, key, false, m.resolveRetries) + if err != nil { + //nolint + c.AbortWithError(http.StatusInternalServerError, err) + } + + for { + select { + + case <-resolveCtx.Done(): + // Resolving mirror has timed out. + //nolint + c.AbortWithError(http.StatusNotFound, fmt.Errorf(p2pcontext.PeerNotFoundLog)) + return + + case peer, ok := <-peersChan: + // Channel closed means no more mirrors will be received and max retries has been reached. + if !ok { + //nolint + c.AbortWithError(http.StatusInternalServerError, fmt.Errorf(p2pcontext.PeerResolutionExhaustedLog)) + return + } + + succeeded := false + u, err := url.Parse(peer.Addr) + if err != nil { + //nolint + c.AbortWithError(http.StatusInternalServerError, err) + return + } + + proxy := httputil.NewSingleHostReverseProxy(u) + proxy.Director = func(r *http.Request) { + r.URL = u + r.URL.Path = c.Request.URL.Path + r.URL.RawQuery = c.Request.URL.RawQuery + p2pcontext.SetOutboundHeaders(r, c) + } + proxy.ModifyResponse = func(resp *http.Response) error { + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("expected peer to respond with 200, got: %s", resp.Status) + } + + succeeded = true + return nil + } + proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { + l.Error().Err(err).Msg("peer request failed, attempting next") + } + proxy.Transport = m.n.RoundTripperFor(peer.ID) + + proxy.ServeHTTP(c.Writer, c.Request) + if !succeeded { + break + } + + l.Info().Str("peer", u.Host).Msg("request served from peer") + return + } + } +} + +// NewMirror creates a new mirror handler. +func NewMirror(router routing.Router) *Mirror { + return &Mirror{ + resolveTimeout: ResolveTimeout, + router: router, + resolveRetries: ResolveRetries, + n: router.Net(), + } +} diff --git a/internal/oci/mirror_test.go b/internal/oci/mirror_test.go new file mode 100644 index 0000000..5bfc791 --- /dev/null +++ b/internal/oci/mirror_test.go @@ -0,0 +1,146 @@ +package oci + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + + p2pcontext "github.com/azure/peerd/internal/context" + "github.com/azure/peerd/internal/routing/tests" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +type TestResponseRecorder struct { + *httptest.ResponseRecorder + closeChannel chan bool +} + +func (r *TestResponseRecorder) CloseNotify() <-chan bool { + return r.closeChannel +} + +//nolint:unused // ignore +func (r *TestResponseRecorder) closeClient() { + r.closeChannel <- true +} + +func CreateTestResponseRecorder() *TestResponseRecorder { + return &TestResponseRecorder{ + httptest.NewRecorder(), + make(chan bool, 1), + } +} + +func TestMirrorHandler(t *testing.T) { + badSvr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + w.Header().Set("foo", "bar") + if r.Method == http.MethodGet { + //nolint:errcheck // ignore + w.Write([]byte("hello world")) + } + })) + defer badSvr.Close() + + goodSvr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("foo", "bar") + if r.Method == http.MethodGet { + //nolint:errcheck // ignore + w.Write([]byte("hello world")) + } + })) + defer goodSvr.Close() + + resolver := map[string][]string{ + "no-working-peers": {badSvr.URL, "foo", badSvr.URL}, + "first-peer": {goodSvr.URL, badSvr.URL, badSvr.URL}, + "first-peer-error": {"foo", goodSvr.URL}, + "last-peer-working": {badSvr.URL, badSvr.URL, goodSvr.URL}, + } + router := tests.NewMockRouter(resolver) + m := &Mirror{ + router: router, + resolveRetries: ResolveRetries, + resolveTimeout: ResolveTimeout, + n: router.Net(), + } + + tests := []struct { + name string + key string + expectedStatus int + expectedBody string + expectedHeaders map[string][]string + }{ + { + name: "request should timeout when no peers exists", + key: "no-peers", + expectedStatus: http.StatusNotFound, + expectedBody: "", + expectedHeaders: nil, + }, + { + name: "request should not timeout and give 500 if all peers fail", + key: "no-working-peers", + expectedStatus: http.StatusInternalServerError, + expectedBody: "", + expectedHeaders: nil, + }, + { + name: "request should work when first peer responds", + key: "first-peer", + expectedStatus: http.StatusOK, + expectedBody: "hello world", + expectedHeaders: map[string][]string{"foo": {"bar"}}, + }, + { + name: "second peer should respond when first gives error", + key: "first-peer-error", + expectedStatus: http.StatusOK, + expectedBody: "hello world", + expectedHeaders: map[string][]string{"foo": {"bar"}}, + }, + { + name: "last peer should respond when two first fail", + key: "last-peer-working", + expectedStatus: http.StatusOK, + expectedBody: "hello world", + expectedHeaders: map[string][]string{"foo": {"bar"}}, + }, + } + for _, tt := range tests { + for _, method := range []string{http.MethodGet, http.MethodHead} { + t.Run(tt.name, func(t *testing.T) { + rw := CreateTestResponseRecorder() + c, _ := gin.CreateTestContext(rw) + target := fmt.Sprintf("http://example.com/%s", tt.key) + c.Request = httptest.NewRequest(method, target, nil) + c.Set(p2pcontext.DigestCtxKey, tt.key) + m.Handle(c) + + resp := rw.Result() + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, tt.expectedStatus, resp.StatusCode) + + if method == http.MethodGet { + require.Equal(t, tt.expectedBody, string(b)) + } + if method == http.MethodHead { + require.Equal(t, "", string(b)) + } + + if tt.expectedHeaders == nil { + require.Len(t, resp.Header, 0) + } + for k, v := range tt.expectedHeaders { + require.Equal(t, v, resp.Header.Values(k)) + } + }) + } + } +} diff --git a/internal/oci/registry.go b/internal/oci/registry.go new file mode 100644 index 0000000..4deac40 --- /dev/null +++ b/internal/oci/registry.go @@ -0,0 +1,147 @@ +package oci + +import ( + "fmt" + "net/http" + "strconv" + "time" + + p2pcontext "github.com/azure/peerd/internal/context" + "github.com/azure/peerd/internal/oci/distribution" + "github.com/azure/peerd/pkg/containerd" + "github.com/gin-gonic/gin" + "github.com/opencontainers/go-digest" + "github.com/rs/zerolog" +) + +// Response headers +const ( + maxManifestSize = 4 * 1024 * 1024 + dockerContentDigestHeader = "Docker-Content-Digest" + contentLengthHeader = "Content-Length" + contentTypeHeader = "Content-Type" +) + +// Registry is a handler that handles requests to this registry. +type Registry struct { + containerdStore containerd.Store +} + +var _ gin.HandlerFunc = (&Registry{}).Handle + +// Handle handles a request to this registry. +func (r *Registry) Handle(c *gin.Context) { + dgstStr := c.GetString(p2pcontext.DigestCtxKey) + ref := c.GetString(p2pcontext.ReferenceCtxKey) + var d digest.Digest + var err error + + l := p2pcontext.Logger(c).With().Str("handler", "registry").Str("ref", ref).Str("digest", dgstStr).Logger() + l.Debug().Msg("registry handler start") + s := time.Now() + defer func() { + l.Debug().Dur("duration", time.Since(s)).Int("status", c.Writer.Status()).Str("digest", d.String()).Msg("registry handler stop") + }() + + // Serve registry endpoints. + if dgstStr == "" { + d, err = r.containerdStore.Resolve(c, ref) + if err != nil { + //nolint + c.AbortWithError(http.StatusNotFound, err) + return + } + } else { + d, err = digest.Parse(dgstStr) + if err != nil { + //nolint + c.AbortWithError(http.StatusBadRequest, err) + return + } + } + + refType, ok := c.Get(p2pcontext.RefTypeCtxKey) + if !ok { + //nolint + c.AbortWithError(http.StatusInternalServerError, fmt.Errorf("ref type not found in context")) + } + + switch refType.(distribution.ReferenceType) { + + case distribution.ReferenceTypeManifest: + r.handleManifest(c, l, d) + return + + case distribution.ReferenceTypeBlob: + r.handleBlob(c, l, d) + return + } + + // If nothing matches return 404. + c.Status(http.StatusNotFound) +} + +// handleManifest handles a manifest request. +func (r *Registry) handleManifest(c *gin.Context, l zerolog.Logger, dgst digest.Digest) { + size, err := r.containerdStore.Size(c, dgst) + if err != nil { + //nolint + c.AbortWithError(http.StatusNotFound, err) + return + } else if size >= maxManifestSize { + //nolint + c.AbortWithError(http.StatusNotFound, fmt.Errorf("refusing to serve a manifest larger than %v bytes, got: %v", maxManifestSize, size)) + return + } + + b, mediaType, err := r.containerdStore.Bytes(c, dgst) + if err != nil { + //nolint + c.AbortWithError(http.StatusNotFound, err) + return + } + + c.Header(contentTypeHeader, mediaType) + c.Header(contentLengthHeader, strconv.FormatInt(int64(len(b)), 10)) + c.Header(dockerContentDigestHeader, dgst.String()) + + if c.Request.Method == http.MethodHead { + return + } + _, err = c.Writer.Write(b) + if err != nil { + //nolint + c.AbortWithError(http.StatusNotFound, err) + return + } +} + +// handleBlob handles a blob request. +func (r *Registry) handleBlob(c *gin.Context, l zerolog.Logger, dgst digest.Digest) { + size, err := r.containerdStore.Size(c, dgst) + if err != nil { + //nolint + c.AbortWithError(http.StatusNotFound, err) + return + } + + c.Header(contentLengthHeader, strconv.FormatInt(size, 10)) + c.Header(dockerContentDigestHeader, dgst.String()) + if c.Request.Method == http.MethodHead { + return + } + + err = r.containerdStore.Write(c, c.Writer, dgst) + if err != nil { + //nolint + c.AbortWithError(http.StatusInternalServerError, err) + return + } +} + +// NewRegistry creates a new registry handler. +func NewRegistry(containerdStore containerd.Store) *Registry { + return &Registry{ + containerdStore: containerdStore, + } +} diff --git a/internal/oci/store/tests/mock.go b/internal/oci/store/tests/mock.go new file mode 100644 index 0000000..c291b8a --- /dev/null +++ b/internal/oci/store/tests/mock.go @@ -0,0 +1,53 @@ +package tests + +import ( + "context" + "io" + + "github.com/azure/peerd/pkg/containerd" + "github.com/opencontainers/go-digest" +) + +type MockContainerdStore struct { + refs []containerd.Reference +} + +var _ containerd.Store = &MockContainerdStore{} + +func NewMockContainerdStore(refs []containerd.Reference) *MockContainerdStore { + return &MockContainerdStore{ + refs: refs, + } +} + +func (m *MockContainerdStore) Verify(ctx context.Context) error { + return nil +} + +func (m *MockContainerdStore) Subscribe(ctx context.Context) (<-chan containerd.Reference, <-chan error) { + return nil, nil +} + +func (m *MockContainerdStore) List(ctx context.Context) ([]containerd.Reference, error) { + return m.refs, nil +} + +func (m *MockContainerdStore) All(ctx context.Context, ref containerd.Reference) ([]string, error) { + return []string{ref.Digest().String()}, nil +} + +func (m *MockContainerdStore) Resolve(ctx context.Context, ref string) (digest.Digest, error) { + return "", nil +} + +func (m *MockContainerdStore) Size(ctx context.Context, dgst digest.Digest) (int64, error) { + return 0, nil +} + +func (m *MockContainerdStore) Write(ctx context.Context, dst io.Writer, dgst digest.Digest) error { + return nil +} + +func (m *MockContainerdStore) Bytes(ctx context.Context, dgst digest.Digest) ([]byte, string, error) { + return nil, "", nil +} diff --git a/internal/remote/interface.go b/internal/remote/interface.go new file mode 100644 index 0000000..585d010 --- /dev/null +++ b/internal/remote/interface.go @@ -0,0 +1,25 @@ +package remote + +import ( + "net/http" + + "github.com/rs/zerolog" +) + +// Reader provides a read-only interface to a remote file. +type Reader interface { + // PreadRemote is like pread but to a remote file. + PreadRemote(buf []byte, offset int64) (int, error) + + // FstatRemote stats a remote file. + FstatRemote() (int64, error) + + // Log returns the logger with context for this reader. + Log() *zerolog.Logger +} + +// Error describes an error that occured during a remote operation. +type Error struct { + *http.Response + error +} diff --git a/internal/remote/reader.go b/internal/remote/reader.go new file mode 100644 index 0000000..02cf2aa --- /dev/null +++ b/internal/remote/reader.go @@ -0,0 +1,285 @@ +package remote + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + p2pcontext "github.com/azure/peerd/internal/context" + "github.com/azure/peerd/internal/metrics" + "github.com/azure/peerd/internal/routing" + "github.com/gin-gonic/gin" + "github.com/rs/zerolog" +) + +type operation int + +const ( + operationFstatRemote = operation(iota) + operationPreadRemote +) + +var errPeerNotFound = errors.New("peer not found") + +// reader is a Reader implementation. +type reader struct { + context *gin.Context + resolveTimeout time.Duration + + router routing.Router + resolveRetries int + defaultHttpClient *http.Client +} + +var _ Reader = &reader{} + +// Log returns the logger with context for this reader. +func (r *reader) Log() *zerolog.Logger { + l := p2pcontext.Logger(r.context) + return &l +} + +// PreadRemote is like pread but to a remote file. +func (r *reader) PreadRemote(buf []byte, offset int64) (int, error) { + key := r.context.GetString(p2pcontext.FileChunkCtxKey) + start := offset + end := int64(len(buf)) + offset - 1 + + log := r.Log().With().Str("operation", "preadremote").Str("key", key).Int64("start", start).Int64("end", end).Logger() + + count, err := r.doP2p(log, key, start, end, operationPreadRemote, buf) + if err == nil { + return int(count), nil + } + + // Could not find a peer that has this file, request origin. + startTime := time.Now() + originReq, err := r.originRequest(start, end) + if err != nil { + return -1, err + } + + count32 := int(0) + defer func() { + metrics.Global.RecordUpstreamResponse(originReq.URL.Hostname(), key, "pread", time.Since(startTime).Seconds(), int64(count32)) + }() + count32, err = r.preadRemote(log, originReq, r.defaultHttpClient, buf) + return count32, err +} + +// FstatRemote stats a remote file. +func (r *reader) FstatRemote() (int64, error) { + key := r.context.GetString(p2pcontext.FileChunkCtxKey) + start := int64(0) + end := int64(0) + + log := r.Log().With().Str("operation", "fstatremote").Int64("start", start).Int64("end", end).Str("key", key).Logger() + + startTime := time.Now() + originReq, err := r.originRequest(start, end) + if err != nil { + return -1, err + } + + var count int64 + defer func() { + metrics.Global.RecordUpstreamResponse(originReq.URL.Hostname(), key, "fstat", time.Since(startTime).Seconds(), count) + }() + count, err = r.fstatRemote(log, originReq, r.defaultHttpClient) + return count, err +} + +// doP2p tries to resolve the key in the p2p network and if successful, it will perform the operation on the peer, and return the result. +func (r *reader) doP2p(log zerolog.Logger, fileChunkKey string, start, end int64, o operation, buf []byte) (int64, error) { + if p2pcontext.IsRequestFromAPeer(r.context) { + log.Warn().Msg("refusing to propagate request from one peer to another") + return -1, errPeerNotFound + } + + log.Debug().Msg(p2pcontext.PeerResolutionStartLog) + defer log.Debug().Msg(p2pcontext.PeerResolutionStopLog) + + resolveCtx, cancel := context.WithTimeout(log.WithContext(r.context), r.resolveTimeout) + defer cancel() + + startTime := time.Now() + peerCount := 0 + peersCh, negCacheCallback, err := r.router.ResolveWithCache(resolveCtx, fileChunkKey, false, r.resolveRetries) + if err != nil { + //nolint:errcheck // ignore + log.Error().Err(err).Msg(p2pcontext.PeerRequestErrorLog) + return -1, err + } + + // Request a peer for this file. +peerLoop: + for { + select { + + case <-resolveCtx.Done(): + // Resolving mirror has timed out. + negCacheCallback() + log.Info().Msg(p2pcontext.PeerNotFoundLog) + break peerLoop + + case peer, ok := <-peersCh: + // Channel closed means no more mirrors will be received and max retries has been reached. + if !ok { + negCacheCallback() + log.Info().Msg(p2pcontext.PeerResolutionExhaustedLog) + break peerLoop + } + + if peerCount == 0 { + // Only report the time it took to discover the first peer. + metrics.Global.RecordPeerDiscovery(peer.Addr, time.Since(startTime).Seconds()) + peerCount++ + } + + peerReq, err := r.peerRequest(peer.Addr, start, end) + if err != nil { + log.Error().Err(err).Msg(p2pcontext.PeerRequestErrorLog) + // try next peer + break + } + + client := r.router.Net().HTTPClientFor(peer.ID) + + var count int64 + startTime = time.Now() + if o == operationFstatRemote { + count, err = r.fstatRemote(log, peerReq, client) + } else if o == operationPreadRemote { + var c int + c, err = r.preadRemote(log, peerReq, client, buf) + count = int64(c) + } else { + err = fmt.Errorf("unknown operation: %v", o) + } + + if err != nil { + // try next peer + log.Error().Err(err).Msg(p2pcontext.PeerRequestErrorLog) + } else { + op := "fstat" + if o == operationPreadRemote { + op = "pread" + } + metrics.Global.RecordPeerResponse(peer.Addr, fileChunkKey, op, time.Since(startTime).Seconds(), count) + return count, nil + } + } + } + + return -1, errPeerNotFound +} + +// fstatRemote stats the file. +func (r *reader) fstatRemote(log zerolog.Logger, req *http.Request, client *http.Client) (int64, error) { + log.Debug().Str("url", req.URL.String()).Str("range", req.Header.Get("Range")).Msg("reader fstatRemote start") + defer log.Debug().Msg("reader fstatRemote stop") + + resp, err := client.Do(req) + if err != nil { + log.Error().Err(err).Msg("reader fstatRemote error") + return 0, Error{resp, err} + } + defer resp.Body.Close() + + if resp.StatusCode == 200 { + return resp.ContentLength, nil + } + + if resp.StatusCode == 206 { + l := resp.ContentLength + rs := resp.Header.Get("Content-Range") + if rs == "" { + return l, nil + } + + pos := strings.LastIndexByte(rs, '/') + if pos < 0 { + return l, nil + } + + l, _ = strconv.ParseInt(rs[pos+1:], 10, 64) + return l, nil + } + + log.Error().Err(err).Int("status", resp.StatusCode).Msg("reader fstatRemote error") + return 0, Error{resp, fmt.Errorf("unexpected response code: %d", resp.StatusCode)} +} + +// preadRemote reads the file. +func (r *reader) preadRemote(log zerolog.Logger, req *http.Request, client *http.Client, buf []byte) (int, error) { + log.Debug().Str("url", req.URL.String()).Str("range", req.Header.Get("Range")).Msg("reader preadRemote start") + statusCode := -1 + s := time.Now() + defer func() { + log.Debug().Int("status", statusCode).Dur("duration", time.Since(s)).Msg("reader preadRemote stop") + }() + + resp, err := client.Do(req) + if resp != nil { + statusCode = resp.StatusCode + } + if err != nil { + detailedErr := Error{resp, err} + log.Error().Err(detailedErr).Str("url", req.URL.String()).Str("range", req.Header.Get("Range")).Msg("reader preadRemote error") + return 0, detailedErr + } + defer resp.Body.Close() + + if resp.StatusCode != 200 && resp.StatusCode != 206 { + log.Error().Err(err).Int("status", resp.StatusCode).Msg("reader preadRemote error") + return 0, Error{resp, fmt.Errorf("unexpected response code: %d", resp.StatusCode)} + } + + return io.ReadFull(resp.Body, buf) +} + +// originRequest will create a new request to origin. +func (r *reader) originRequest(start, end int64) (*http.Request, error) { + return r.remoteRequest(r.context.GetString(p2pcontext.BlobUrlCtxKey), start, end) +} + +// perRequest will create a new request to a peer. +func (r *reader) peerRequest(peer string, start, end int64) (*http.Request, error) { + return r.remoteRequest(fmt.Sprintf("%v/blobs/%v", peer, r.context.GetString(p2pcontext.BlobUrlCtxKey)), start, end) +} + +// remoteRequest creates a new HTTP request to a remote server. +func (r *reader) remoteRequest(u string, start, end int64) (*http.Request, error) { + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, err + } + + for key, vals := range r.context.Request.Header { + vals2 := make([]string, len(vals)) + copy(vals2, vals) + req.Header[key] = vals2 + } + + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + p2pcontext.SetOutboundHeaders(req, r.context) + + return req, nil +} + +// NewReader creates a new remote reader. +func NewReader(c *gin.Context, router routing.Router, resolveRetries int, resolveTimeout time.Duration) Reader { + cc := c.Copy() + return &reader{ + context: cc, + resolveTimeout: resolveTimeout, + router: router, + resolveRetries: resolveRetries, + defaultHttpClient: router.Net().HTTPClientFor(""), + } +} diff --git a/internal/remote/reader_test.go b/internal/remote/reader_test.go new file mode 100644 index 0000000..a69fa14 --- /dev/null +++ b/internal/remote/reader_test.go @@ -0,0 +1,309 @@ +package remote + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + p2pcontext "github.com/azure/peerd/internal/context" + "github.com/azure/peerd/internal/routing/tests" + "github.com/gin-gonic/gin" + "github.com/rs/zerolog" +) + +var ( + hostAndPath = "https://avtakkartest.blob.core.windows.net/d18c7a64c5158179-ff8cb2f639ff44879c12c94361a746d0-782b855128//docker/registry/v2/blobs/sha256/d1/d18c7a64c5158179bdee531a663c5b487de57ff17cff3af29a51c7e70b491d9d/data" + query = "?se=2023-09-20T01%3A14%3A49Z&sig=m4Cr%2BYTZHZQlN5LznY7nrTQ4LCIx2OqnDDM3Dpedbhs%3D&sp=r&spr=https&sr=b&sv=2018-03-28®id=01031d61e1024861afee5d512651eb9f" + u = hostAndPath + query +) + +func TestPreadRemoteUpstream(t *testing.T) { + // Setup + m := map[string][]string{} + key := "somekey" + expected := "expected-result" + peersTried := 0 + svr3 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + peersTried++ + w.WriteHeader(http.StatusUnauthorized) + })) + defer svr3.Close() + svr2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + peersTried++ + w.WriteHeader(http.StatusNotFound) + })) + defer svr2.Close() + svr1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + peersTried++ + w.WriteHeader(http.StatusBadGateway) + })) + defer svr1.Close() + val := []string{svr1.URL, svr2.URL, svr3.URL} + m[key] = val + + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if "?"+r.URL.RawQuery == query { + w.Header().Set("Content-Type", "application/octet-stream") + // nolint:errcheck + w.Write([]byte(expected)) + } else { + w.WriteHeader(http.StatusNotFound) + } + })) + defer svr.Close() + p := svr.URL + "/some-path" + u := "http://127.0.0.1:5000/blobs/" + p + query + req, err := http.NewRequest("GET", u, nil) + if err != nil { + t.Fatal(err) + } + + router := tests.NewMockRouter(m) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = req + c.Params = []gin.Param{ + {Key: "url", Value: p}, + } + c.Set(p2pcontext.BlobUrlCtxKey, p2pcontext.BlobUrl(c)) + c.Set(p2pcontext.BlobRangeCtxKey, "bytes=0-10") + c.Set(p2pcontext.FileChunkCtxKey, key) + + r := NewReader(c, router, 3, 500*time.Millisecond).(*reader) + b := make([]byte, 10) + + // Test + got, err := r.PreadRemote(b, 0) + + // Assert + if err != nil { + t.Fatal(err) + } else if got != 10 { + t.Fatalf("expected %v, got %v", 10, got) + } else if string(b) != expected[:10] { + t.Fatalf("expected %v, got %v", expected[:10], string(b)) + } else if peersTried != 3 { + t.Fatalf("expected %v, got %v", 3, peersTried) + } +} + +func TestFstatRemote(t *testing.T) { + m := map[string][]string{} + + expected := "expected-result" + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if "?"+r.URL.RawQuery == query { + w.Header().Set("Content-Type", "application/octet-stream") + // nolint:errcheck + w.Write([]byte(expected)) + } else { + w.WriteHeader(http.StatusNotFound) + } + })) + defer svr.Close() + p := svr.URL + "/some-path" + u := "http://127.0.0.1:5000/blobs/" + p + query + req, err := http.NewRequest("GET", u, nil) + if err != nil { + t.Fatal(err) + } + + router := tests.NewMockRouter(m) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = req + c.Params = []gin.Param{ + {Key: "url", Value: p}, + } + c.Set(p2pcontext.BlobUrlCtxKey, p2pcontext.BlobUrl(c)) + c.Set(p2pcontext.BlobRangeCtxKey, "bytes=0-0") + + r := NewReader(c, router, 3, 500*time.Millisecond).(*reader) + + got, err := r.FstatRemote() + if err != nil { + t.Fatal(err) + } else if got != int64(len(expected)) { + t.Fatalf("expected %v, got %v", len(expected), got) + } +} + +func TestFstatRemotePartialContent(t *testing.T) { + m := map[string][]string{} + + expected := "expected-result" + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if "?"+r.URL.RawQuery == query { + w.Header().Set("Content-Type", "application/octet-stream") + // nolint:errcheck + w.WriteHeader(http.StatusPartialContent) + w.Header().Set("Content-Range", "bytes 0-10/10") + // nolint:errcheck + w.Write([]byte(expected)) + } else { + w.WriteHeader(http.StatusNotFound) + } + })) + defer svr.Close() + p := svr.URL + "/some-path" + u := "http://127.0.0.1:5000/blobs/" + p + query + req, err := http.NewRequest("GET", u, nil) + if err != nil { + t.Fatal(err) + } + + router := tests.NewMockRouter(m) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = req + c.Params = []gin.Param{ + {Key: "url", Value: p}, + } + c.Set(p2pcontext.BlobUrlCtxKey, p2pcontext.BlobUrl(c)) + c.Set(p2pcontext.BlobRangeCtxKey, "bytes=0-0") + + r := NewReader(c, router, 3, 500*time.Millisecond).(*reader) + + got, err := r.FstatRemote() + if err != nil { + t.Fatal(err) + } else if got != int64(len(expected)) { + t.Fatalf("expected %v, got %v", len(expected), got) + } +} + +func TestP2pRetries(t *testing.T) { + l := zerolog.Nop() + m := map[string][]string{} + key := "somekey" + expected := "expected-result" + svr3 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + // nolint:errcheck + w.Write([]byte(expected)) + })) + defer svr3.Close() + svr2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer svr2.Close() + svr1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadGateway) + })) + defer svr1.Close() + val := []string{svr1.URL, svr2.URL, svr3.URL} + m[key] = val + + req, err := http.NewRequest("GET", "http://127.0.0.1:5000/blobs/"+u, nil) + if err != nil { + t.Fatal(err) + } + + router := tests.NewMockRouter(m) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = req + r := NewReader(c, router, 3, 500*time.Millisecond).(*reader) + b := make([]byte, 10) + + got, err := r.doP2p(l, key, 0, 10, operationPreadRemote, b) + if err != nil { + t.Fatal(err) + } + + if got != 10 { + t.Fatalf("expected %v, got %v", 10, got) + } else if string(b) != expected[:10] { + t.Fatalf("expected %v, got %v", expected[:10], string(b)) + } +} + +func TestP2pSuccess(t *testing.T) { + l := zerolog.Nop() + m := map[string][]string{} + key := "somekey" + expected := "expected-result" + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + // nolint:errcheck + w.Write([]byte(expected)) + })) + defer svr.Close() + val := []string{svr.URL} + m[key] = val + + req, err := http.NewRequest("GET", "http://127.0.0.1:5000/blobs/"+u, nil) + if err != nil { + t.Fatal(err) + } + + router := tests.NewMockRouter(m) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = req + r := NewReader(c, router, 3, 500*time.Millisecond).(*reader) + b := make([]byte, 10) + + got, err := r.doP2p(l, key, 0, 10, operationPreadRemote, b) + if err != nil { + t.Fatal(err) + } + + if got != 10 { + t.Fatalf("expected %v, got %v", 10, got) + } else if string(b) != expected[:10] { + t.Fatalf("expected %v, got %v", expected[:10], string(b)) + } +} + +func TestP2pPeerNotFound(t *testing.T) { + l := zerolog.Nop() + m := map[string][]string{} + + req, err := http.NewRequest("GET", "http://127.0.0.1:5000/blobs/"+u, nil) + if err != nil { + t.Fatal(err) + } + + router := tests.NewMockRouter(m) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = req + + r := NewReader(c, router, 3, 500*time.Millisecond).(*reader) + + b := make([]byte, 10) + _, err = r.doP2p(l, "key", 0, 10, operationPreadRemote, b) + if err == nil { + t.Fatal("expected error") + } + + if err != errPeerNotFound { + t.Fatalf("expected %v, got %v", errPeerNotFound, err) + } +} + +func TestP2pNoInfiniteLoops(t *testing.T) { + l := zerolog.Nop() + m := map[string][]string{} + key := "some-key" + val := []string{"http://localhost"} + m[key] = val + + req, err := http.NewRequest("GET", "http://127.0.0.1:5000/blobs/"+u, nil) + if err != nil { + t.Fatal(err) + } + + router := tests.NewMockRouter(m) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = req + c.Request.Header.Add(p2pcontext.P2PHeaderKey, "true") + + r := NewReader(c, router, 3, 500*time.Millisecond).(*reader) + + b := make([]byte, 10) + _, err = r.doP2p(l, key, 0, 10, operationPreadRemote, b) + if err == nil { + t.Fatal("expected error") + } + + if err != errPeerNotFound { + t.Fatalf("expected %v, got %v", errPeerNotFound, err) + } +} diff --git a/internal/remote/tests/mockreader.go b/internal/remote/tests/mockreader.go new file mode 100644 index 0000000..92fee1c --- /dev/null +++ b/internal/remote/tests/mockreader.go @@ -0,0 +1,37 @@ +package tests + +import ( + "github.com/azure/peerd/internal/remote" + "github.com/rs/zerolog" +) + +var l = zerolog.Nop() + +type mockReader struct { + data []byte +} + +var _ remote.Reader = &mockReader{} + +// FstatRemote implements remote.Reader. +func (m *mockReader) FstatRemote() (int64, error) { + return int64(len(m.data)), nil +} + +// Log implements remote.Reader. +func (*mockReader) Log() *zerolog.Logger { + return &l +} + +// PreadRemote implements remote.Reader. +func (m *mockReader) PreadRemote(buf []byte, offset int64) (int, error) { + if offset >= int64(len(m.data)) { + return 0, nil + } + return copy(buf, m.data[offset:]), nil +} + +// NewMockReader creates a new mock reader for testing purposes. +func NewMockReader(data []byte) remote.Reader { + return &mockReader{data: data} +} diff --git a/internal/routing/interface.go b/internal/routing/interface.go new file mode 100644 index 0000000..2cde9a1 --- /dev/null +++ b/internal/routing/interface.go @@ -0,0 +1,32 @@ +package routing + +import ( + "context" + + "github.com/azure/peerd/pkg/peernet" + "github.com/libp2p/go-libp2p/core/peer" +) + +// Router provides an interface to a peered network. +type Router interface { + // Net returns the network interface. + Net() peernet.Network + + // Resolve resolves the given key to a peer address. + Resolve(ctx context.Context, key string, allowSelf bool, count int) (<-chan PeerInfo, error) + + // ResolveWithCache is like Resolve but it also returns a function callback that can be used to cache that a key could not be resolved. + ResolveWithCache(ctx context.Context, key string, allowSelf bool, count int) (<-chan PeerInfo, func(), error) + + // Advertise advertises the given keys to the network. + Advertise(ctx context.Context, keys []string) error + + // Close closes the router. + Close() error +} + +// PeerInfo describes a peer. +type PeerInfo struct { + peer.ID + Addr string +} diff --git a/internal/routing/router.go b/internal/routing/router.go new file mode 100644 index 0000000..6aeb63f --- /dev/null +++ b/internal/routing/router.go @@ -0,0 +1,237 @@ +package routing + +import ( + "context" + "fmt" + "net" + "sync/atomic" + + p2pcontext "github.com/azure/peerd/internal/context" + "github.com/azure/peerd/internal/k8s/events" + "github.com/azure/peerd/pkg/k8s/election" + "github.com/azure/peerd/pkg/peernet" + "github.com/dgraph-io/ristretto" + cid "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p" + dht "github.com/libp2p/go-libp2p-kad-dht" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/p2p/discovery/routing" + "github.com/multiformats/go-multiaddr" + mc "github.com/multiformats/go-multicodec" + mh "github.com/multiformats/go-multihash" + "github.com/rs/zerolog" +) + +type router struct { + p2pnet peernet.Network + host host.Host + rd *routing.RoutingDiscovery + port string + lookupCache *ristretto.Cache + + active atomic.Bool +} + +// PeerNotFoundError indicates that no peer could be found for the given key. +type PeerNotFoundError struct { + error + key string +} + +// NewRouter creates a new router. +func NewRouter(ctx context.Context, routerAddr, serverPort string) (Router, error) { + log := zerolog.Ctx(ctx).With().Str("component", "router").Logger() + + h, p, err := net.SplitHostPort(routerAddr) + if err != nil { + return nil, err + } + + multiAddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%s", h, p)) + if err != nil { + return nil, fmt.Errorf("could not create host multi address: %w", err) + } + + factory := libp2p.AddrsFactory(func(addrs []multiaddr.Multiaddr) []multiaddr.Multiaddr { + for _, addr := range addrs { + v, err := addr.ValueForProtocol(multiaddr.P_IP4) + if err != nil { + continue + } + if v == "" { + continue + } + if v == "127.0.0.1" { + continue + } + return []multiaddr.Multiaddr{addr} + } + return nil + }) + + host, err := libp2p.New(libp2p.ListenAddrs(multiAddr), factory) + if err != nil { + return nil, fmt.Errorf("could not create host: %w", err) + } + + self := fmt.Sprintf("%s/p2p/%s", host.Addrs()[0].String(), host.ID().String()) + log.Info().Str("id", self).Msg("starting p2p router") + + leaderElection := election.New("peerd-ns", "peerd-leader-election", p2pcontext.KubeConfigPath) + + err = leaderElection.RunOrDie(ctx, self) + if err != nil { + return nil, err + } + + // TODO avtakkar: reconsider the max record age for cached files. Or, ensure that the cached list is periodically advertised. + dhtOpts := []dht.Option{dht.Mode(dht.ModeServer), dht.ProtocolPrefix("/microsoft"), dht.DisableValues(), dht.MaxRecordAge(p2pcontext.KeyTTL)} + bootstrapPeerOpt := dht.BootstrapPeersFunc(func() []peer.AddrInfo { + addr, err := leaderElection.Leader() + if err != nil { + events.FromContext(ctx).Disconnected() + log.Error().Err(err).Msg("could not get leader") + return nil + } + + addrInfo, err := peer.AddrInfoFromP2pAddr(addr) + if err != nil { + log.Error().Err(err).Msg("could not get leader") + return nil + } + + defer func() { + events.FromContext(ctx).Connected() + }() + + if addrInfo.ID == host.ID() { + log.Info().Msg("leader is self, skipping connection to bootstrap node") + return nil + } + + log.Info().Str("node", addrInfo.ID.String()).Msg("bootstrap node found") + return []peer.AddrInfo{*addrInfo} + }) + + dhtOpts = append(dhtOpts, bootstrapPeerOpt) + kdht, err := dht.New(ctx, host, dhtOpts...) + if err != nil { + return nil, fmt.Errorf("could not create distributed hash table: %w", err) + } + if err = kdht.Bootstrap(ctx); err != nil { + return nil, fmt.Errorf("could not boostrap distributed hash table: %w", err) + } + rd := routing.NewRoutingDiscovery(kdht) + + c, err := ristretto.NewCache(&ristretto.Config{NumCounters: 1e7, MaxCost: 1073741824, BufferItems: 64}) + if err != nil { + return nil, err + } + + n, err := peernet.New(host) + if err != nil { + return nil, err + } + + return &router{ + p2pnet: n, + host: host, + rd: rd, + port: serverPort, + lookupCache: c, + }, nil +} + +// Transport returns the transport. +func (r *router) Net() peernet.Network { + return r.p2pnet +} + +// Close closes the router. +func (r *router) Close() error { + return r.host.Close() +} + +// ResolveWithCache is like Resolve but it also returns a function callback that can be used to cache that a key could not be resolved. +func (r *router) ResolveWithCache(ctx context.Context, key string, allowSelf bool, count int) (<-chan PeerInfo, func(), error) { + if val, ok := r.lookupCache.Get(key); ok && val.(string) == p2pcontext.P2pLookupNotFoundValue { + // TODO avtakkar: currently only doing a negative cache, this could maybe become a positive cache as well. + return nil, nil, PeerNotFoundError{key: key, error: fmt.Errorf("(cached) peer not found for key")} + } + peerCh, err := r.Resolve(ctx, key, allowSelf, count) + return peerCh, func() { + r.lookupCache.SetWithTTL(key, p2pcontext.P2pLookupNotFoundValue, 1, p2pcontext.P2pLookupCacheTtl) + }, err +} + +// Resolve resolves the given key to a peer address. +func (r *router) Resolve(ctx context.Context, key string, allowSelf bool, count int) (<-chan PeerInfo, error) { + log := zerolog.Ctx(ctx).With().Str("host", r.host.ID().String()).Str("key", key).Logger() + c, err := createCid(key) + if err != nil { + return nil, err + } + addrCh := r.rd.FindProvidersAsync(ctx, c, count) + peerCh := make(chan PeerInfo, count) + go func() { + for info := range addrCh { + if !allowSelf && info.ID == r.host.ID() { + continue + } + if len(info.Addrs) != 1 { + log.Info().Msg("expected address list to only contain a single item") + continue + } + + v, err := info.Addrs[0].ValueForProtocol(multiaddr.P_IP4) + if err != nil { + log.Error().Err(err).Msg("could not get IPV4 address") + continue + } + + // Combine peer with registry port to create mirror endpoint. + peerCh <- PeerInfo{info.ID, fmt.Sprintf("https://%s:%s", v, r.port)} + + if r.active.CompareAndSwap(false, true) { + er, err := events.NewRecorder(ctx) + if err != nil { + log.Error().Err(err).Msg("could not create event recorder") + } else { + er.Active() // Report that p2p is active. + } + } + } + }() + return peerCh, nil +} + +// Advertise advertises the given keys to the network. +func (r *router) Advertise(ctx context.Context, keys []string) error { + zerolog.Ctx(ctx).Debug().Str("host", r.host.ID().String()).Strs("keys", keys).Msg("advertising keys") + for _, key := range keys { + c, err := createCid(key) + if err != nil { + return err + } + err = r.rd.Provide(ctx, c, true) + if err != nil { + return err + } + } + return nil +} + +func createCid(key string) (cid.Cid, error) { + pref := cid.Prefix{ + Version: 1, + Codec: uint64(mc.Raw), + MhType: mh.SHA2_256, + MhLength: -1, + } + c, err := pref.Sum([]byte(key)) + if err != nil { + return cid.Cid{}, err + } + return c, nil +} diff --git a/internal/routing/router_test.go b/internal/routing/router_test.go new file mode 100644 index 0000000..1d6a246 --- /dev/null +++ b/internal/routing/router_test.go @@ -0,0 +1,250 @@ +package routing + +import ( + "context" + "errors" + "testing" + "time" + + p2pcontext "github.com/azure/peerd/internal/context" + "github.com/dgraph-io/ristretto" + cid "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p/core/connmgr" + "github.com/libp2p/go-libp2p/core/event" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/peerstore" + "github.com/libp2p/go-libp2p/core/protocol" + corerouting "github.com/libp2p/go-libp2p/core/routing" + "github.com/libp2p/go-libp2p/p2p/discovery/routing" + multiaddr "github.com/multiformats/go-multiaddr" +) + +func TestResolveWithCache(t *testing.T) { + c, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e7, + MaxCost: 1000, + BufferItems: 64, + }) + if err != nil { + t.Fatal(err) + } + + h := &testHost{"host-id"} + key := "some-key" + + tcr := &testCr{ + m: map[string][]string{}, + } + + r := &router{ + host: h, + port: "5000", + lookupCache: c, + rd: routing.NewRoutingDiscovery(tcr), + } + + ctx := context.Background() + _, negCacheCallback, err := r.ResolveWithCache(ctx, key, false, 2) + if err != nil { + t.Fatal(err) + } + + negCacheCallback() + time.Sleep(250 * time.Millisecond) // allow cache to flush + + if val, ok := r.lookupCache.Get(key); !ok || val != p2pcontext.P2pLookupNotFoundValue { + t.Errorf("expected key to be %s, got %s", p2pcontext.P2pLookupNotFoundValue, val) + } +} + +func TestResolve(t *testing.T) { + c, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e7, + MaxCost: 1000, + BufferItems: 64, + }) + if err != nil { + t.Fatal(err) + } + + h := &testHost{"host-id"} + key := "some-key" + contentId, err := createCid(key) + if err != nil { + t.Fatal(err) + } + + r := &router{ + host: h, + port: "5000", + lookupCache: c, + rd: routing.NewRoutingDiscovery(&testCr{ + m: map[string][]string{ + contentId.String(): {"10.0.0.1", "10.0.0.2"}, + }, + }), + } + + ctx := context.Background() + got, err := r.Resolve(ctx, key, false, 2) + if err != nil { + t.Fatal(err) + } + + count := 0 + for info := range got { + if info.Addr == "https://10.0.0.1:5000" || info.Addr == "https://10.0.0.2:5000" { + count++ + } else { + t.Errorf("expected peer1 or peer2, got %s", info) + } + + if count == 2 { + break + } + } + + if count != 2 { + t.Errorf("expected 2 addresses, got %d", count) + } +} + +func TestProvide(t *testing.T) { + c, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e7, + MaxCost: 1000, + BufferItems: 64, + }) + if err != nil { + t.Fatal(err) + } + + h := &testHost{"host-id"} + key := "some-key" + contentId, err := createCid(key) + if err != nil { + t.Fatal(err) + } + tcr := &testCr{ + m: map[string][]string{}, + } + + r := &router{ + host: h, + port: "5000", + lookupCache: c, + rd: routing.NewRoutingDiscovery(tcr), + } + + ctx := context.Background() + err = r.Advertise(ctx, []string{key}) + if err != nil { + t.Fatal(err) + } + + if len(tcr.provided) != 1 { + t.Errorf("expected 1 cid to be provided, got %d", len(tcr.provided)) + } else if tcr.provided[0] != contentId { + t.Errorf("expected cid %s to be provided, got %s", contentId, tcr.provided[0]) + } +} + +type testCr struct { + m map[string][]string + provided []cid.Cid +} + +// FindProvidersAsync implements routing.ContentRouting. +func (t *testCr) FindProvidersAsync(ctx context.Context, c cid.Cid, count int) <-chan peer.AddrInfo { + ch := make(chan peer.AddrInfo, count) + if val, ok := t.m[c.String()]; ok { + for _, addr := range val { + ch <- peer.AddrInfo{ID: peer.ID(addr), Addrs: []multiaddr.Multiaddr{multiaddr.StringCast("/ip4/" + addr + "/tcp/5005")}} + } + } + return ch +} + +// Provide implements routing.ContentRouting. +func (t *testCr) Provide(ctx context.Context, c cid.Cid, advertise bool) error { + if !advertise { + return errors.New("advertise must be true") + } + t.provided = append(t.provided, c) + return nil +} + +var _ corerouting.ContentRouting = &testCr{} + +type testHost struct { + id peer.ID +} + +// Addrs implements host.Host. +func (*testHost) Addrs() []multiaddr.Multiaddr { + panic("unimplemented") +} + +// Close implements host.Host. +func (*testHost) Close() error { + panic("unimplemented") +} + +// ConnManager implements host.Host. +func (*testHost) ConnManager() connmgr.ConnManager { + panic("unimplemented") +} + +// Connect implements host.Host. +func (*testHost) Connect(ctx context.Context, pi peer.AddrInfo) error { + panic("unimplemented") +} + +// EventBus implements host.Host. +func (*testHost) EventBus() event.Bus { + panic("unimplemented") +} + +// ID implements host.Host. +func (th *testHost) ID() peer.ID { + return th.id +} + +// Mux implements host.Host. +func (*testHost) Mux() protocol.Switch { + panic("unimplemented") +} + +// Network implements host.Host. +func (*testHost) Network() network.Network { + panic("unimplemented") +} + +// NewStream implements host.Host. +func (*testHost) NewStream(ctx context.Context, p peer.ID, pids ...protocol.ID) (network.Stream, error) { + panic("unimplemented") +} + +// Peerstore implements host.Host. +func (*testHost) Peerstore() peerstore.Peerstore { + panic("unimplemented") +} + +// RemoveStreamHandler implements host.Host. +func (*testHost) RemoveStreamHandler(pid protocol.ID) { + panic("unimplemented") +} + +// SetStreamHandler implements host.Host. +func (*testHost) SetStreamHandler(pid protocol.ID, handler network.StreamHandler) { + panic("unimplemented") +} + +// SetStreamHandlerMatch implements host.Host. +func (*testHost) SetStreamHandlerMatch(protocol.ID, func(protocol.ID) bool, network.StreamHandler) { + panic("unimplemented") +} + +var _ host.Host = &testHost{} diff --git a/internal/routing/tests/mock.go b/internal/routing/tests/mock.go new file mode 100644 index 0000000..43f241f --- /dev/null +++ b/internal/routing/tests/mock.go @@ -0,0 +1,89 @@ +package tests + +import ( + "context" + "sync" + + "github.com/azure/peerd/internal/routing" + "github.com/azure/peerd/pkg/mocks" + "github.com/azure/peerd/pkg/peernet" + "github.com/libp2p/go-libp2p/core/peer" +) + +type MockRouter struct { + net peernet.Network + mx sync.RWMutex + resolver map[string][]string + + negCache map[string]struct{} +} + +// Net implements routing.Router. +func (m *MockRouter) Net() peernet.Network { + return m.net +} + +// ResolveWithCache implements Router. +func (m *MockRouter) ResolveWithCache(ctx context.Context, key string, allowSelf bool, count int) (<-chan routing.PeerInfo, func(), error) { + c, e := m.Resolve(ctx, key, allowSelf, count) + return c, func() { + m.mx.Lock() + defer m.mx.Unlock() + m.negCache[key] = struct{}{} + }, e +} + +var _ routing.Router = &MockRouter{} + +func NewMockRouter(resolver map[string][]string) *MockRouter { + n, err := peernet.New(&mocks.MockHost{PeerStore: &mocks.MockPeerstore{}}) + if err != nil { + panic(err) + } + + return &MockRouter{ + net: n, + resolver: resolver, + negCache: map[string]struct{}{}, + } +} + +func (m *MockRouter) Close() error { + return nil +} + +func (m *MockRouter) Resolve(ctx context.Context, key string, allowSelf bool, count int) (<-chan routing.PeerInfo, error) { + peerCh := make(chan routing.PeerInfo, count) + peers, ok := m.resolver[key] + // Not found will look forever until timeout. + if !ok { + return peerCh, nil + } + + go func() { + m.mx.RLock() + defer m.mx.RUnlock() + for _, p := range peers { + peerCh <- routing.PeerInfo{ID: peer.ID(p), Addr: p} + } + close(peerCh) + }() + + return peerCh, nil +} + +func (m *MockRouter) Advertise(ctx context.Context, keys []string) error { + m.mx.Lock() + defer m.mx.Unlock() + for _, key := range keys { + m.resolver[key] = []string{"localhost"} + } + return nil +} + +func (m *MockRouter) LookupKey(key string) ([]string, bool) { + m.mx.RLock() + defer m.mx.RUnlock() + v, ok := m.resolver[key] + return v, ok +} diff --git a/internal/state/state.go b/internal/state/state.go new file mode 100644 index 0000000..0bfe776 --- /dev/null +++ b/internal/state/state.go @@ -0,0 +1,124 @@ +package state + +import ( + "context" + "errors" + "fmt" + "time" + + p2pcontext "github.com/azure/peerd/internal/context" + "github.com/azure/peerd/internal/routing" + "github.com/azure/peerd/pkg/containerd" + "github.com/rs/zerolog" +) + +// Advertise advertises images and files to the given routing.Router. +// It listens for events from the containerd.Store and filesChan channel to trigger the advertisement. +// The function runs until the context is done or an error occurs. +// +// Parameters: +// - ctx: The context.Context used for cancellation and deadline propagation. +// - r: The routing.Router used for advertising files. +// - containerdStore: The containerd.Store used for subscribing to events and advertising images. +// - filesChan: The channel that provides the files to be advertised. +// +// Returns: None. +func Advertise(ctx context.Context, r routing.Router, containerdStore containerd.Store, filesChan <-chan string) { + l := zerolog.Ctx(ctx).With().Str("component", "state").Logger() + l.Debug().Msg("advertising start") + s := time.Now() + defer func() { + l.Debug().Dur("duration", time.Since(s)).Msg("advertising stop") + }() + + eventCh, errCh := containerdStore.Subscribe(ctx) + + immediate := make(chan time.Time, 1) + immediate <- time.Now() + + expirationTicker := time.NewTicker(p2pcontext.KeyTTL - time.Minute) + defer expirationTicker.Stop() + + ticker := p2pcontext.Merge(immediate, expirationTicker.C) + + for { + select { + + case <-ctx.Done(): + return + + case <-ticker: + l.Info().Msg("scheduled advertisement") + err := advertiseAll(ctx, l, containerdStore, r) + if err != nil { + l.Error().Err(err).Msg("schedule: error advertising") + continue + } + + case ref := <-eventCh: + l.Debug().Str("image", ref.Name()).Str("digest", ref.Digest().String()).Msg("advertising image") + _, err := advertiseRef(ctx, l, containerdStore, r, ref) + if err != nil { + l.Error().Err(err).Msg("image: advertising error") + continue + } + + case blob := <-filesChan: + l.Debug().Str("blob", blob).Msg("advertising file") + err := r.Advertise(ctx, []string{blob}) + if err != nil { + l.Error().Err(err).Str("blob", blob).Msg("file: advertising error") + continue + } + + case err := <-errCh: + l.Error().Err(err).Msg("channel error") + continue + } + } +} + +// advertiseAll advertises all references in the containerd store using the provided logger and router. +// It returns an error if any error occurs during the advertisement process. +func advertiseAll(ctx context.Context, l zerolog.Logger, containerdStore containerd.Store, router routing.Router) error { + refs, err := containerdStore.List(ctx) + if err != nil { + return err + } + + errs := []error{} + for _, ref := range refs { + _, err := advertiseRef(ctx, l, containerdStore, router, ref) + if err != nil { + errs = append(errs, err) + continue + } + } + + return errors.Join(errs...) +} + +// advertiseRef advertises the given containerd reference by extracting its digest and tags, +// retrieving additional digests from the containerd store, and advertising all the keys to the router. +// It returns the number of keys advertised and any error encountered. +func advertiseRef(ctx context.Context, l zerolog.Logger, containerdStore containerd.Store, router routing.Router, ref containerd.Reference) (int, error) { + keys := []string{} + keys = append(keys, ref.Digest().String()) + if ref.Tag() != "" { + keys = append(keys, ref.String()) + } + + dgsts, err := containerdStore.All(ctx, ref) + if err != nil { + l.Error().Err(err).Str("image", ref.Name()).Str("digest", ref.Digest().String()).Msg("could not get digests for image") + } else { + keys = append(keys, dgsts...) + } + + err = router.Advertise(ctx, keys) + if err != nil { + return 0, fmt.Errorf("could not advertise image %v: %w", ref, err) + } + + return len(keys), nil +} diff --git a/internal/state/state_test.go b/internal/state/state_test.go new file mode 100644 index 0000000..438f8d5 --- /dev/null +++ b/internal/state/state_test.go @@ -0,0 +1,54 @@ +package state + +import ( + "context" + "testing" + "time" + + ocitests "github.com/azure/peerd/internal/oci/store/tests" + "github.com/azure/peerd/internal/routing/tests" + "github.com/azure/peerd/pkg/containerd" + "github.com/stretchr/testify/require" +) + +// TestContainerdStoreAds is a unit test function that tests the basic functionality of the Advertise function. +// It creates a list of container image references, initializes a mock containerd store and a mock router, +// and then calls the Advertise function with the created context, router, containerd store, and an empty file channel. +// After that, it verifies that the router correctly looks up the peers for each reference. +func TestContainerdStoreAds(t *testing.T) { + refsStr := []string{ + "docker.io/library/ubuntu:latest@sha256:b060fffe8e1561c9c3e6dea6db487b900100fc26830b9ea2ec966c151ab4c020", + "ghcr.io/xenitab/spegel:v0.0.9@sha256:fa32bd3bcd49a45a62cfc1b0fed6a0b63bf8af95db5bad7ec22865aee0a4b795", + "docker.io/library/alpine@sha256:25fad2a32ad1f6f510e528448ae1ec69a28ef81916a004d3629874104f8a7f70", + } + + refs := []containerd.Reference{} + for _, refStr := range refsStr { + img, err := containerd.ParseReference(refStr, "") + require.NoError(t, err) + refs = append(refs, img) + } + + containerdStore := ocitests.NewMockContainerdStore(refs) + router := tests.NewMockRouter(map[string][]string{}) + + ctx, cancel := context.WithCancel(context.TODO()) + go func() { + time.Sleep(2 * time.Second) + cancel() + }() + + Advertise(ctx, router, containerdStore, make(<-chan string)) // TODO avtakkar: add tests for file chan + + for _, ref := range refs { + peers, ok := router.LookupKey(ref.Digest().String()) + require.True(t, ok) + require.Len(t, peers, 1) + + if ref.Tag() != "" { + peers, ok = router.LookupKey(ref.String()) + require.True(t, ok) + require.Len(t, peers, 1) + } + } +} diff --git a/pkg/containerd/reference.go b/pkg/containerd/reference.go new file mode 100644 index 0000000..53e4fce --- /dev/null +++ b/pkg/containerd/reference.go @@ -0,0 +1,155 @@ +package containerd + +import ( + "fmt" + "net/url" + "regexp" + "strings" + + "github.com/opencontainers/go-digest" +) + +var separator = regexp.MustCompile(`[:@]`) + +// Reference is a reference to an OCI artifact in the content store. +type Reference interface { + // Name is the name of the artifact. + Name() string + + // Digest is the digest of the artifact. + Digest() digest.Digest + + // Tag is the tag of the artifact. + Tag() string + + // Repository is the repository of the artifact. + Repository() string + + // Host is the host of the artifact. + Host() string + + // String returns the string representation of the reference. + // docker.io/library/ubuntu:latest + // docker.io/library/ubuntu@sha256:abcdef + String() string +} + +// Reference is a reference to an OCI artifact. +type reference struct { + // name is the name of the artifact. + name string + + // dgst is the digest of the artifact. + dgst digest.Digest + + // tag is the tag of the artifact. + tag string + + // repo is the repository of the artifact. + repo string + + // host is the host of the artifact. + host string +} + +var _ Reference = &reference{} + +// Name is the name of the artifact. +func (r *reference) Name() string { + return r.name +} + +// Digest is the digest of the artifact. +func (r *reference) Digest() digest.Digest { + return r.dgst +} + +// Tag is the tag of the artifact. +func (r *reference) Tag() string { + return r.tag +} + +// Repository is the repository of the artifact. +func (r *reference) Repository() string { + return r.repo +} + +// Host is the host of the artifact. +func (r *reference) Host() string { + return r.host +} + +// String returns the string representation of the reference. +func (r *reference) String() string { + if r.tag != "" { + return fmt.Sprintf("%s/%s:%s", r.Host(), r.Repository(), r.Tag()) + } + return fmt.Sprintf("%s/%s@%s", r.Host(), r.Repository(), r.Digest()) +} + +// ParseReference parses the given name into a reference. +// targetDigest is obtained from the containerd interface, and is used to verify the parsed digest, or to set the digest if it is not present. +func ParseReference(name string, targetDigest digest.Digest) (Reference, error) { + if strings.Contains(name, "://") { + return nil, fmt.Errorf("invalid reference") + } + + u, err := url.Parse("localhost://" + name) + if err != nil { + return nil, err + } + + if u.Scheme != "localhost" { + return nil, fmt.Errorf("invalid reference") + } + + if u.Host == "" { + return nil, fmt.Errorf("hostname required") + } + + var obj string + if idx := separator.FindStringIndex(u.Path); idx != nil { + // This allows us to retain the @ to signify digests or shortened digests in the object. + obj = u.Path[idx[0]:] + if obj[:1] == ":" { + obj = obj[1:] + } + u.Path = u.Path[:idx[0]] + } + + tag, dgst := splitTagAndDigest(obj) + tag, _, _ = strings.Cut(tag, "@") + repository := strings.TrimPrefix(u.Path, "/") + + if dgst == "" { + dgst = targetDigest + } + + if targetDigest != "" && dgst != targetDigest { + return nil, fmt.Errorf("invalid digest, target does not match parsed digest: %v %v", name, dgst) + } + + if repository == "" { + return nil, fmt.Errorf("invalid repository: %v", repository) + } + + if dgst == "" { + return nil, fmt.Errorf("invalid digest: %v", dgst) + } + + return &reference{ + name: name, + host: u.Host, + tag: tag, + dgst: dgst, + repo: repository, + }, nil +} + +func splitTagAndDigest(obj string) (tag string, dgst digest.Digest) { + parts := strings.SplitAfterN(obj, "@", 2) + if len(parts) < 2 { + return parts[0], "" + } + return parts[0], digest.Digest(parts[1]) +} diff --git a/pkg/containerd/reference_test.go b/pkg/containerd/reference_test.go new file mode 100644 index 0000000..dd7ca3c --- /dev/null +++ b/pkg/containerd/reference_test.go @@ -0,0 +1,92 @@ +package containerd + +import ( + "fmt" + "testing" + + digest "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/require" +) + +func TestParseReference(t *testing.T) { + tests := []struct { + name string + image string + digestInImage bool + expectedRepository string + expectedTag string + expectedDigest digest.Digest + }{ + { + name: "Latest tag", + image: "library/ubuntu:latest", + digestInImage: false, + expectedRepository: "library/ubuntu", + expectedTag: "latest", + expectedDigest: digest.Digest("sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda"), + }, + { + name: "Only tag", + image: "library/alpine:3.18.0", + digestInImage: false, + expectedRepository: "library/alpine", + expectedTag: "3.18.0", + expectedDigest: digest.Digest("sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda"), + }, + { + name: "Tag and digest", + image: "jetstack/cert-manager-controller:3.18.0@sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda", + digestInImage: true, + expectedRepository: "jetstack/cert-manager-controller", + expectedTag: "3.18.0", + expectedDigest: digest.Digest("sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda"), + }, + { + name: "Only digest", + image: "fluxcd/helm-controller@sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda", + digestInImage: true, + expectedRepository: "fluxcd/helm-controller", + expectedTag: "", + expectedDigest: digest.Digest("sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda"), + }, + } + registries := []string{"docker.io", "quay.io", "ghcr.com", "127.0.0.1"} + for _, registry := range registries { + for _, tt := range tests { + t.Run(fmt.Sprintf("%s_%s", tt.name, registry), func(t *testing.T) { + for _, targetDigest := range []string{tt.expectedDigest.String(), ""} { + ref, err := ParseReference(fmt.Sprintf("%s/%s", registry, tt.image), digest.Digest(targetDigest)) + if !tt.digestInImage && targetDigest == "" { + require.EqualError(t, err, "invalid digest: ") + continue + } + require.NoError(t, err) + require.Equal(t, registry, ref.Host()) + require.Equal(t, tt.expectedRepository, ref.Repository()) + require.Equal(t, tt.expectedTag, ref.Tag()) + require.Equal(t, tt.expectedDigest, ref.Digest()) + } + }) + + } + } +} + +func TestParseImageDigestDoesNotMatch(t *testing.T) { + _, err := ParseReference("quay.io/jetstack/cert-manager-webhook@sha256:13fd9eaadb4e491ef0e1d82de60cb199f5ad2ea5a3f8e0c19fdf31d91175b9cb", digest.Digest("sha256:ec4306b243d98cce7c3b1f994f2dae660059ef521b2b24588cfdc950bd816d4c")) + require.EqualError(t, err, "invalid digest, target does not match parsed digest: quay.io/jetstack/cert-manager-webhook@sha256:13fd9eaadb4e491ef0e1d82de60cb199f5ad2ea5a3f8e0c19fdf31d91175b9cb sha256:13fd9eaadb4e491ef0e1d82de60cb199f5ad2ea5a3f8e0c19fdf31d91175b9cb") +} + +func TestParseImageNoTagOrDigest(t *testing.T) { + _, err := ParseReference("ghcr.io/xenitab/spegel", digest.Digest("")) + require.EqualError(t, err, "invalid digest: ") +} + +func TestString(t *testing.T) { + got, err := ParseReference("jetstack/cert-manager-controller:3.18.0@sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda", digest.Digest("sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda")) + if err != nil { + t.Fatal(err) + } + require.Equal(t, "jetstack/cert-manager-controller:3.18.0", got.String()) + require.Equal(t, "jetstack/cert-manager-controller:3.18.0@sha256:c0669ef34cdc14332c0f1ab0c2c01acb91d96014b172f1a76f3a39e63d1f0bda", got.Name()) +} diff --git a/pkg/containerd/store.go b/pkg/containerd/store.go new file mode 100644 index 0000000..2f1a451 --- /dev/null +++ b/pkg/containerd/store.go @@ -0,0 +1,350 @@ +package containerd + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/url" + "sort" + "strings" + + "github.com/containerd/containerd" + eventtypes "github.com/containerd/containerd/api/events" + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/platforms" + "github.com/containerd/typeurl/v2" + "github.com/distribution/distribution/manifest" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +const ( + // DefaultSock is the default containerd socket path. + DefaultSock = "/run/containerd/containerd.sock" + + // DefaultNamespace is the default containerd namespace for this client. + DefaultNamespace = "k8s.io" +) + +// Store is the interface for all containerd content store artifacts. +type Store interface { + // Subscribe returns a channel of artifacts and a channel of errors. + // Artifacts are sent on the channel as they are discovered. + Subscribe(ctx context.Context) (<-chan Reference, <-chan error) + + // List returns a list of artifacts. + List(ctx context.Context) ([]Reference, error) + + // Resolve returns the digest for an existing artifact. + Resolve(ctx context.Context, ref string) (digest.Digest, error) + + // Size returns the size of the artifact. + Size(ctx context.Context, dgst digest.Digest) (int64, error) + + // Bytes returns the artifact bytes. + Bytes(ctx context.Context, dgst digest.Digest) ([]byte, string, error) + + // Write writes the artifact bytes to the writer. + Write(ctx context.Context, dst io.Writer, dgst digest.Digest) error + + // Verify will verify that the client status is healthy. + Verify(ctx context.Context) error + + // All returns a list of digests of all resources referenced in ref. + All(ctx context.Context, ref Reference) ([]string, error) +} + +// store provides an interface to the containerd content store. +type store struct { + client *containerd.Client + platform platforms.MatchComparer + + // Filters for list and event subscriptions. + // The syntax of these filters is defined here: https://github.com/containerd/containerd/blob/main/filters/filter.go + listFilter string + eventFilter string +} + +var _ Store = &store{} + +// NewDefaultStore creates a new Store with default values for containerd socket, namespace and hosts configuration path. +func NewDefaultStore(hosts []string) (Store, error) { + return NewStore(DefaultSock, DefaultNamespace, hosts) +} + +// NewStore creates a new Store. +func NewStore(sock, ns string, hosts []string) (Store, error) { + if sock == "" { + return nil, fmt.Errorf("containerd socket path cannot be empty") + } + + if ns == "" { + return nil, fmt.Errorf("containerd namespace cannot be empty") + } + + client, err := containerd.New(sock, containerd.WithDefaultNamespace(ns)) + if err != nil { + return nil, fmt.Errorf("could not create containerd client: %w", err) + } + + return newStore(hosts, client) +} + +func newStore(hosts []string, client *containerd.Client) (*store, error) { + for _, host := range hosts { + _, err := url.Parse(host) + if err != nil { + return nil, err + } + } + + return &store{ + client: client, + platform: platforms.Default(), + listFilter: getListFilter(hosts), + eventFilter: getEventFilter(hosts), + }, nil +} + +// Verify will verify that the containerd service is serving at the configured socket. +func (c *store) Verify(ctx context.Context) error { + ok, err := c.client.IsServing(ctx) + if err != nil { + return err + } else if !ok { + return fmt.Errorf("could not reach containerd service") + } + + return nil +} + +// Subscribe provides a subscription to containerd events on the configured hosts artifacts. +// It also returns a channel of errors. +func (c *store) Subscribe(ctx context.Context) (<-chan Reference, <-chan error) { + refChan := make(chan Reference) + errChan := make(chan error) + + eventsChan, eventsErrChan := c.client.EventService().Subscribe(ctx, c.eventFilter) + go func() { + for event := range eventsChan { + name, err := getEventImageName(event.Event) + if err != nil { + errChan <- err + continue + } + + image, err := c.client.GetImage(ctx, name) + if err != nil { + errChan <- err + continue + } + + ref, err := ParseReference(image.Name(), image.Target().Digest) + if err != nil { + errChan <- err + } else { + refChan <- ref + } + } + }() + + go func() { + for err := range eventsErrChan { + errChan <- err + } + }() + + return refChan, errChan +} + +// List returns the list of locally found images. +func (c *store) List(ctx context.Context) ([]Reference, error) { + imgs, err := c.client.ListImages(ctx, c.listFilter) + if err != nil { + return nil, err + } + + refs := []Reference{} + for _, img := range imgs { + ref, err := ParseReference(img.Name(), img.Target().Digest) + if err != nil { + return nil, err + } + refs = append(refs, ref) + } + return refs, nil +} + +// All returns a list of digests of all resources referenced in ref. +func (c *store) All(ctx context.Context, ref Reference) ([]string, error) { + img, err := c.client.ImageService().Get(ctx, ref.Name()) + if err != nil { + return nil, err + } + + keys := []string{} + + err = images.Walk(ctx, images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + keys = append(keys, desc.Digest.String()) + + switch desc.MediaType { + case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: + var idx ocispec.Index + + b, err := content.ReadBlob(ctx, c.client.ContentStore(), desc) + if err != nil { + return nil, err + } + + if err := json.Unmarshal(b, &idx); err != nil { + return nil, err + } + + var descs []ocispec.Descriptor + for _, m := range idx.Manifests { + if !c.platform.Match(*m.Platform) { + continue + } + descs = append(descs, m) + } + if len(descs) == 0 { + return nil, fmt.Errorf("could not find platform architecture in manifest: %v", desc.Digest) + } + + // Platform matching is a bit weird in that multiple platforms can match. + // There is however a "best" match that should be used. + // This logic is used by Containerd to determine which layer to pull so we should use the same logic. + sort.SliceStable(descs, func(i, j int) bool { + if descs[i].Platform == nil { + return false + } + if descs[j].Platform == nil { + return true + } + return c.platform.Less(*descs[i].Platform, *descs[j].Platform) + }) + return []ocispec.Descriptor{descs[0]}, nil + + case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: + var manifest ocispec.Manifest + b, err := content.ReadBlob(ctx, c.client.ContentStore(), desc) + if err != nil { + return nil, err + } + + if err := json.Unmarshal(b, &manifest); err != nil { + return nil, err + } + keys = append(keys, manifest.Config.Digest.String()) + for _, layer := range manifest.Layers { + keys = append(keys, layer.Digest.String()) + } + return nil, nil + + default: + return nil, fmt.Errorf("unexpected media type %v for digest: %v", desc.MediaType, desc.Digest) + } + }), img.Target) + if err != nil { + return nil, fmt.Errorf("failed to walk image manifests: %w", err) + } + + if len(keys) == 0 { + return nil, fmt.Errorf("no image digests found") + } + + return keys, nil +} + +// Resolve returns the digest for an existing artifact. +func (c *store) Resolve(ctx context.Context, ref string) (digest.Digest, error) { + cImg, err := c.client.GetImage(ctx, ref) + if err != nil { + return "", err + } + return cImg.Target().Digest, nil +} + +// Size returns the size of the artifact. +func (c *store) Size(ctx context.Context, dgst digest.Digest) (int64, error) { + info, err := c.client.ContentStore().Info(ctx, dgst) + if err != nil { + return 0, err + } + return info.Size, nil +} + +// Bytes returns the artifact bytes. This method should only be used for manifests. +func (c *store) Bytes(ctx context.Context, dgst digest.Digest) ([]byte, string, error) { + b, err := content.ReadBlob(ctx, c.client.ContentStore(), ocispec.Descriptor{Digest: dgst}) + if err != nil { + return nil, "", err + } + var m manifest.Versioned + if err := json.Unmarshal(b, &m); err != nil { + return nil, "", err + } + + return b, m.MediaType, nil +} + +// Write writes the blob bytes to the writer. +func (c *store) Write(ctx context.Context, dst io.Writer, dgst digest.Digest) error { + ra, err := c.client.ContentStore().ReaderAt(ctx, ocispec.Descriptor{Digest: dgst}) + if err != nil { + return err + } + defer ra.Close() + + _, err = io.Copy(dst, content.NewReader(ra)) + if err != nil { + return err + } + + return nil +} + +// getEventImageName will get the image name from an event. +func getEventImageName(e typeurl.Any) (string, error) { + evt, err := typeurl.UnmarshalAny(e) + if err != nil { + return "", fmt.Errorf("failed to unmarshal any: %w", err) + } + + switch e := evt.(type) { + case *eventtypes.ImageCreate: + return e.Name, nil + case *eventtypes.ImageUpdate: + return e.Name, nil + default: + return "", fmt.Errorf("unsupported event: %v", e) + } +} + +func getListFilter(hosts []string) string { + return fmt.Sprintf(`name~="%s"`, strings.Join(getHostNames(hosts), "|")) +} + +func getEventFilter(hosts []string) string { + return fmt.Sprintf(`topic~="/images/create|/images/update",event.name~="%s"`, strings.Join(getHostNames(hosts), "|")) +} + +func getHostNames(hosts []string) []string { + names := []string{} + for _, host := range hosts { + if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") { + host = "http://" + host + } + + u, err := url.Parse(host) + if err != nil { + // Use the host as is. + names = append(names, host) + } else { + names = append(names, u.Host) + } + } + return names +} diff --git a/pkg/containerd/store_test.go b/pkg/containerd/store_test.go new file mode 100644 index 0000000..23930c8 --- /dev/null +++ b/pkg/containerd/store_test.go @@ -0,0 +1,658 @@ +package containerd + +import ( + "bytes" + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/azure/peerd/pkg/mocks" + "github.com/containerd/containerd" + eventtypes "github.com/containerd/containerd/api/events" + "github.com/containerd/containerd/events" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/platforms" + "github.com/containerd/typeurl/v2" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/require" +) + +func TestCreateFilter(t *testing.T) { + tests := []struct { + name string + hosts []string + expectedListFilter string + expectedEventFilter string + }{ + { + name: "only registries", + hosts: []string{"https://docker.io", "https://gcr.io"}, + expectedListFilter: `name~="docker.io|gcr.io"`, + expectedEventFilter: `topic~="/images/create|/images/update",event.name~="docker.io|gcr.io"`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expectedListFilter, getListFilter(tt.hosts)) + require.Equal(t, tt.expectedEventFilter, getEventFilter(tt.hosts)) + }) + } +} + +func TestAllNoPlatform(t *testing.T) { + cs := &mocks.MockContentStore{ + Data: map[string]string{ + // Index + "sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a": `{ "mediaType": "application/vnd.oci.image.index.v1+json", "schemaVersion": 2, "manifests": [ { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", "size": 2372, "platform": { "architecture": "amd64", "os": "linux" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b", "size": 2372, "platform": { "architecture": "arm", "os": "linux", "variant": "v7" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53", "size": 2372, "platform": { "architecture": "arm64", "os": "linux" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:73af5483f4d2d636275dcef14d5443ff96d7347a0720ca5a73a32c73855c4aac", "size": 566, "annotations": { "vnd.docker.reference.digest": "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", "vnd.docker.reference.type": "attestation-manifest" }, "platform": { "architecture": "unknown", "os": "unknown" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:36e11bf470af256febbdfad9d803e60b7290b0268218952991b392be9e8153bd", "size": 566, "annotations": { "vnd.docker.reference.digest": "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b", "vnd.docker.reference.type": "attestation-manifest" }, "platform": { "architecture": "unknown", "os": "unknown" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:42d1c43f2285e8e3d39f80b8eed8e4c5c28b8011c942b5413ecc6a0050600609", "size": 566, "annotations": { "vnd.docker.reference.digest": "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53", "vnd.docker.reference.type": "attestation-manifest" }, "platform": { "architecture": "unknown", "os": "unknown" } } ] }`, + }, + } + + is := &mocks.MockImageStore{ + Data: map[string]images.Image{ + "ghcr.io/distribution/distribution:v0.0.8": { + Target: ocispec.Descriptor{MediaType: "application/vnd.oci.image.index.v1+json", Digest: digest.Digest("sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a")}, + }, + }, + } + + client, err := containerd.New("", containerd.WithServices(containerd.WithImageStore(is), containerd.WithContentStore(cs))) + require.NoError(t, err) + s := store{ + client: client, + platform: platforms.Only(platforms.MustParse("darwin/arm64")), + } + img, err := ParseReference("ghcr.io/distribution/distribution:v0.0.8", digest.Digest("sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a")) + require.NoError(t, err) + + _, err = s.All(context.TODO(), img) + require.EqualError(t, err, "failed to walk image manifests: could not find platform architecture in manifest: sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a") +} + +func TestAll(t *testing.T) { + tests := []struct { + platformStr string + imageName string + imageDigest string + expectedKeys []string + }{ + { + platformStr: "linux/amd64", + imageName: "ghcr.io/distribution/distribution:v0.0.8-with-media-type", + imageDigest: "sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a", + expectedKeys: []string{ + "sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a", + "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", + "sha256:d715ba0d85ee7d37da627d0679652680ed2cb23dde6120f25143a0b8079ee47e", + "sha256:a7ca0d9ba68fdce7e15bc0952d3e898e970548ca24d57698725836c039086639", + "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", + "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", + "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", + "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", + "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", + "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", + "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", + "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", + "sha256:76f3a495ffdc00c612747ba0c59fc56d0a2610d2785e80e9edddbf214c2709ef", + "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", + }, + }, + { + platformStr: "linux/amd64", + imageName: "ghcr.io/distribution/distribution:v0.0.8-without-media-type", + imageDigest: "sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a", + expectedKeys: []string{ + "sha256:e2db0e6787216c5abfc42ea8ec82812e41782f3bc6e3b5221d5ef9c800e6c507", + "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", + "sha256:d715ba0d85ee7d37da627d0679652680ed2cb23dde6120f25143a0b8079ee47e", + "sha256:a7ca0d9ba68fdce7e15bc0952d3e898e970548ca24d57698725836c039086639", + "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", + "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", + "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", + "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", + "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", + "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", + "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", + "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", + "sha256:76f3a495ffdc00c612747ba0c59fc56d0a2610d2785e80e9edddbf214c2709ef", + "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", + }, + }, + { + platformStr: "linux/arm64", + imageName: "ghcr.io/distribution/distribution:v0.0.8-with-media-type", + imageDigest: "sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a", + expectedKeys: []string{ + "sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a", + "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53", + "sha256:c73129c9fb699b620aac2df472196ed41797fd0f5a90e1942bfbf19849c4a1c9", + "sha256:0b41f743fd4d78cb50ba86dd3b951b51458744109e1f5063a76bc5a792c3d8e7", + "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", + "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", + "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", + "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", + "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", + "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", + "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", + "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", + "sha256:0dc769edeab7d9f622b9703579f6c89298a4cf45a84af1908e26fffca55341e1", + "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", + }, + }, + { + platformStr: "linux/arm", + imageName: "ghcr.io/distribution/distribution:v0.0.8-with-media-type", + imageDigest: "sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a", + expectedKeys: []string{ + "sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a", + "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b", + "sha256:1079836371d57a148a0afa5abfe00bd91825c869fcc6574a418f4371d53cab4c", + "sha256:b437b30b8b4cc4e02865517b5ca9b66501752012a028e605da1c98beb0ed9f50", + "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", + "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", + "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", + "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", + "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", + "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", + "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", + "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", + "sha256:01d28554416aa05390e2827a653a1289a2a549e46cc78d65915a75377c6008ba", + "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", + }, + }, + } + + cs := &mocks.MockContentStore{ + Data: map[string]string{ + // Index with media type + "sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a": `{ "mediaType": "application/vnd.oci.image.index.v1+json", "schemaVersion": 2, "manifests": [ { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", "size": 2372, "platform": { "architecture": "amd64", "os": "linux" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b", "size": 2372, "platform": { "architecture": "arm", "os": "linux", "variant": "v7" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53", "size": 2372, "platform": { "architecture": "arm64", "os": "linux" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:73af5483f4d2d636275dcef14d5443ff96d7347a0720ca5a73a32c73855c4aac", "size": 566, "annotations": { "vnd.docker.reference.digest": "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", "vnd.docker.reference.type": "attestation-manifest" }, "platform": { "architecture": "unknown", "os": "unknown" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:36e11bf470af256febbdfad9d803e60b7290b0268218952991b392be9e8153bd", "size": 566, "annotations": { "vnd.docker.reference.digest": "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b", "vnd.docker.reference.type": "attestation-manifest" }, "platform": { "architecture": "unknown", "os": "unknown" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:42d1c43f2285e8e3d39f80b8eed8e4c5c28b8011c942b5413ecc6a0050600609", "size": 566, "annotations": { "vnd.docker.reference.digest": "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53", "vnd.docker.reference.type": "attestation-manifest" }, "platform": { "architecture": "unknown", "os": "unknown" } } ] }`, + // Index without media type + "sha256:e2db0e6787216c5abfc42ea8ec82812e41782f3bc6e3b5221d5ef9c800e6c507": `{ "schemaVersion": 2, "manifests": [ { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", "size": 2372, "platform": { "architecture": "amd64", "os": "linux" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b", "size": 2372, "platform": { "architecture": "arm", "os": "linux", "variant": "v7" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53", "size": 2372, "platform": { "architecture": "arm64", "os": "linux" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:73af5483f4d2d636275dcef14d5443ff96d7347a0720ca5a73a32c73855c4aac", "size": 566, "annotations": { "vnd.docker.reference.digest": "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", "vnd.docker.reference.type": "attestation-manifest" }, "platform": { "architecture": "unknown", "os": "unknown" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:36e11bf470af256febbdfad9d803e60b7290b0268218952991b392be9e8153bd", "size": 566, "annotations": { "vnd.docker.reference.digest": "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b", "vnd.docker.reference.type": "attestation-manifest" }, "platform": { "architecture": "unknown", "os": "unknown" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:42d1c43f2285e8e3d39f80b8eed8e4c5c28b8011c942b5413ecc6a0050600609", "size": 566, "annotations": { "vnd.docker.reference.digest": "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53", "vnd.docker.reference.type": "attestation-manifest" }, "platform": { "architecture": "unknown", "os": "unknown" } } ] }`, + // AMD64 + "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355": `{ "mediaType": "application/vnd.oci.image.manifest.v1+json", "schemaVersion": 2, "config": { "mediaType": "application/vnd.oci.image.config.v1+json", "digest": "sha256:d715ba0d85ee7d37da627d0679652680ed2cb23dde6120f25143a0b8079ee47e", "size": 2842 }, "layers": [ { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:a7ca0d9ba68fdce7e15bc0952d3e898e970548ca24d57698725836c039086639", "size": 103732 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", "size": 21202 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", "size": 716491 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", "size": 317 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", "size": 198 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", "size": 113 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", "size": 385 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", "size": 355 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", "size": 130562 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:76f3a495ffdc00c612747ba0c59fc56d0a2610d2785e80e9edddbf214c2709ef", "size": 36529876 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", "size": 32 } ] }`, + // ARM64 + "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53": `{ "mediaType": "application/vnd.oci.image.manifest.v1+json", "schemaVersion": 2, "config": { "mediaType": "application/vnd.oci.image.config.v1+json", "digest": "sha256:c73129c9fb699b620aac2df472196ed41797fd0f5a90e1942bfbf19849c4a1c9", "size": 2842 }, "layers": [ { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:0b41f743fd4d78cb50ba86dd3b951b51458744109e1f5063a76bc5a792c3d8e7", "size": 103732 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", "size": 21202 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", "size": 716491 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", "size": 317 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", "size": 198 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", "size": 113 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", "size": 385 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", "size": 355 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", "size": 130562 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:0dc769edeab7d9f622b9703579f6c89298a4cf45a84af1908e26fffca55341e1", "size": 34168923 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", "size": 32 } ] }`, + // ARM + "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b": `{ "mediaType": "application/vnd.oci.image.manifest.v1+json", "schemaVersion": 2, "config": { "mediaType": "application/vnd.oci.image.config.v1+json", "digest": "sha256:1079836371d57a148a0afa5abfe00bd91825c869fcc6574a418f4371d53cab4c", "size": 2855 }, "layers": [ { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:b437b30b8b4cc4e02865517b5ca9b66501752012a028e605da1c98beb0ed9f50", "size": 103732 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", "size": 21202 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", "size": 716491 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", "size": 317 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", "size": 198 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", "size": 113 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", "size": 385 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", "size": 355 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", "size": 130562 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:01d28554416aa05390e2827a653a1289a2a549e46cc78d65915a75377c6008ba", "size": 34318536 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", "size": 32 } ] }`, + }, + } + is := &mocks.MockImageStore{ + Data: map[string]images.Image{ + "ghcr.io/distribution/distribution:v0.0.8-with-media-type": { + Target: ocispec.Descriptor{MediaType: "application/vnd.oci.image.index.v1+json", Digest: digest.Digest("sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a")}, + }, + "ghcr.io/distribution/distribution:v0.0.8-without-media-type": { + Target: ocispec.Descriptor{MediaType: "application/vnd.oci.image.index.v1+json", Digest: digest.Digest("sha256:e2db0e6787216c5abfc42ea8ec82812e41782f3bc6e3b5221d5ef9c800e6c507")}, + }, + }, + } + client, err := containerd.New("", containerd.WithServices(containerd.WithImageStore(is), containerd.WithContentStore(cs))) + require.NoError(t, err) + + for _, tt := range tests { + t.Run(strings.Join([]string{tt.platformStr, tt.imageName}, "-"), func(t *testing.T) { + s := store{ + client: client, + platform: platforms.Only(platforms.MustParse(tt.platformStr)), + } + img, err := ParseReference(tt.imageName, digest.Digest(tt.imageDigest)) + require.NoError(t, err) + + keys, err := s.All(context.TODO(), img) + require.NoError(t, err) + require.Equal(t, tt.expectedKeys, keys) + }) + } +} + +func TestSize(t *testing.T) { + cs := &mocks.MockContentStore{ + Data: map[string]string{ + "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355": `{ "mediaType": "application/vnd.oci.image.manifest.v1+json", "schemaVersion": 2, "config": { "mediaType": "application/vnd.oci.image.config.v1+json", "digest": "sha256:d715ba0d85ee7d37da627d0679652680ed2cb23dde6120f25143a0b8079ee47e", "size": 2842 }, "layers": [ { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:a7ca0d9ba68fdce7e15bc0952d3e898e970548ca24d57698725836c039086639", "size": 103732 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", "size": 21202 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", "size": 716491 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", "size": 317 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", "size": 198 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", "size": 113 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", "size": 385 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", "size": 355 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", "size": 130562 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:76f3a495ffdc00c612747ba0c59fc56d0a2610d2785e80e9edddbf214c2709ef", "size": 36529876 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", "size": 32 } ] }`, + }, + } + is := &mocks.MockImageStore{ + Data: map[string]images.Image{ + "ghcr.io/distribution/distribution:v0.0.8-with-media-type": { + Target: ocispec.Descriptor{MediaType: "application/vnd.oci.image.index.v1+json", Digest: digest.Digest("sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355")}, + }, + }, + } + + client, err := containerd.New("", containerd.WithServices(containerd.WithImageStore(is), containerd.WithContentStore(cs))) + require.NoError(t, err) + + for _, tt := range []struct { + d digest.Digest + s int64 + errExpected bool + }{ + { + "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", + 2062, + false, + }, + { + "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4353", + 0, + true, + }, + } { + t.Run(fmt.Sprintf("%v-%v", tt.d.String(), tt.errExpected), func(t *testing.T) { + s := store{ + client: client, + platform: platforms.Only(platforms.MustParse("linux/amd64")), + } + + size, err := s.Size(context.Background(), tt.d) + if tt.errExpected { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tt.s, size) + } + }) + } +} + +func TestResolve(t *testing.T) { + cs := &mocks.MockContentStore{ + Data: map[string]string{ + "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355": `{ "mediaType": "application/vnd.oci.image.manifest.v1+json", "schemaVersion": 2, "config": { "mediaType": "application/vnd.oci.image.config.v1+json", "digest": "sha256:d715ba0d85ee7d37da627d0679652680ed2cb23dde6120f25143a0b8079ee47e", "size": 2842 }, "layers": [ { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:a7ca0d9ba68fdce7e15bc0952d3e898e970548ca24d57698725836c039086639", "size": 103732 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", "size": 21202 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", "size": 716491 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", "size": 317 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", "size": 198 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", "size": 113 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", "size": 385 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", "size": 355 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", "size": 130562 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:76f3a495ffdc00c612747ba0c59fc56d0a2610d2785e80e9edddbf214c2709ef", "size": 36529876 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", "size": 32 } ] }`, + }, + } + is := &mocks.MockImageStore{ + Data: map[string]images.Image{ + "ghcr.io/distribution/distribution:v0.0.8": { + Target: ocispec.Descriptor{MediaType: "application/vnd.oci.image.index.v1+json", Digest: digest.Digest("sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355")}, + }, + }, + } + + client, err := containerd.New("", containerd.WithServices(containerd.WithImageStore(is), containerd.WithContentStore(cs))) + require.NoError(t, err) + + for _, tt := range []struct { + ref string + expected digest.Digest + errExpected bool + }{ + { + "ghcr.io/distribution/distribution:v0.0.8", + "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", + false, + }, + { + "ghcr.io/distribution/distribution:latest", + "", + true, + }, + } { + t.Run(fmt.Sprintf("%v-%v", tt.ref, tt.errExpected), func(t *testing.T) { + s := store{ + client: client, + platform: platforms.Only(platforms.MustParse("linux/amd64")), + } + + got, err := s.Resolve(context.Background(), tt.ref) + if tt.errExpected { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tt.expected, got) + } + }) + } +} + +func TestBytes(t *testing.T) { + man := `{ "mediaType": "application/vnd.oci.image.manifest.v1+json", "schemaVersion": 2, "config": { "mediaType": "application/vnd.oci.image.config.v1+json", "digest": "sha256:d715ba0d85ee7d37da627d0679652680ed2cb23dde6120f25143a0b8079ee47e", "size": 2842 }, "layers": [ { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:a7ca0d9ba68fdce7e15bc0952d3e898e970548ca24d57698725836c039086639", "size": 103732 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", "size": 21202 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", "size": 716491 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", "size": 317 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", "size": 198 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", "size": 113 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", "size": 385 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", "size": 355 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", "size": 130562 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:76f3a495ffdc00c612747ba0c59fc56d0a2610d2785e80e9edddbf214c2709ef", "size": 36529876 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", "size": 32 } ] }` + + cs := &mocks.MockContentStore{ + Data: map[string]string{ + "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355": man, + }, + } + is := &mocks.MockImageStore{ + Data: map[string]images.Image{ + "ghcr.io/distribution/distribution:v0.0.8": { + Target: ocispec.Descriptor{MediaType: "application/vnd.oci.image.index.v1+json", Digest: digest.Digest("sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355")}, + }, + }, + } + + client, err := containerd.New("", containerd.WithServices(containerd.WithImageStore(is), containerd.WithContentStore(cs))) + require.NoError(t, err) + + for _, tt := range []struct { + d digest.Digest + expected string + mt string + errExpected bool + }{ + { + "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", + man, + "application/vnd.oci.image.manifest.v1+json", + false, + }, + { + "ghcr.io/distribution/distribution:latest", + "", + "", + true, + }, + } { + t.Run(fmt.Sprintf("%v-%v", tt.d, tt.errExpected), func(t *testing.T) { + s := store{ + client: client, + platform: platforms.Only(platforms.MustParse("linux/amd64")), + } + + got, mt, err := s.Bytes(context.Background(), tt.d) + if tt.errExpected { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tt.expected, string(got)) + require.Equal(t, tt.mt, mt) + } + }) + } +} + +func TestGetHostNames(t *testing.T) { + for _, tt := range []struct { + hosts []string + expected []string + }{ + { + hosts: []string{"ghcr.io"}, + expected: []string{"ghcr.io"}, + }, + { + hosts: []string{"ghcr.io", "docker.io", "mcr.microsoft.com", "localhost:5000"}, + expected: []string{"ghcr.io", "docker.io", "mcr.microsoft.com", "localhost:5000"}, + }, + { + hosts: []string{"https://k8s.io", "https://registry-1.docker.io"}, + expected: []string{"k8s.io", "registry-1.docker.io"}, + }, + } { + t.Run(fmt.Sprintf("%v", tt.hosts), func(t *testing.T) { + got := getHostNames(tt.hosts) + require.Equal(t, tt.expected, got) + }) + } +} + +func TestList(t *testing.T) { + cs := &mocks.MockContentStore{ + Data: map[string]string{ + // Index with media type + "sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a": `{ "mediaType": "application/vnd.oci.image.index.v1+json", "schemaVersion": 2, "manifests": [ { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", "size": 2372, "platform": { "architecture": "amd64", "os": "linux" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b", "size": 2372, "platform": { "architecture": "arm", "os": "linux", "variant": "v7" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53", "size": 2372, "platform": { "architecture": "arm64", "os": "linux" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:73af5483f4d2d636275dcef14d5443ff96d7347a0720ca5a73a32c73855c4aac", "size": 566, "annotations": { "vnd.docker.reference.digest": "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", "vnd.docker.reference.type": "attestation-manifest" }, "platform": { "architecture": "unknown", "os": "unknown" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:36e11bf470af256febbdfad9d803e60b7290b0268218952991b392be9e8153bd", "size": 566, "annotations": { "vnd.docker.reference.digest": "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b", "vnd.docker.reference.type": "attestation-manifest" }, "platform": { "architecture": "unknown", "os": "unknown" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:42d1c43f2285e8e3d39f80b8eed8e4c5c28b8011c942b5413ecc6a0050600609", "size": 566, "annotations": { "vnd.docker.reference.digest": "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53", "vnd.docker.reference.type": "attestation-manifest" }, "platform": { "architecture": "unknown", "os": "unknown" } } ] }`, + // Index without media type + "sha256:e2db0e6787216c5abfc42ea8ec82812e41782f3bc6e3b5221d5ef9c800e6c507": `{ "schemaVersion": 2, "manifests": [ { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", "size": 2372, "platform": { "architecture": "amd64", "os": "linux" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b", "size": 2372, "platform": { "architecture": "arm", "os": "linux", "variant": "v7" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53", "size": 2372, "platform": { "architecture": "arm64", "os": "linux" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:73af5483f4d2d636275dcef14d5443ff96d7347a0720ca5a73a32c73855c4aac", "size": 566, "annotations": { "vnd.docker.reference.digest": "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", "vnd.docker.reference.type": "attestation-manifest" }, "platform": { "architecture": "unknown", "os": "unknown" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:36e11bf470af256febbdfad9d803e60b7290b0268218952991b392be9e8153bd", "size": 566, "annotations": { "vnd.docker.reference.digest": "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b", "vnd.docker.reference.type": "attestation-manifest" }, "platform": { "architecture": "unknown", "os": "unknown" } }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:42d1c43f2285e8e3d39f80b8eed8e4c5c28b8011c942b5413ecc6a0050600609", "size": 566, "annotations": { "vnd.docker.reference.digest": "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53", "vnd.docker.reference.type": "attestation-manifest" }, "platform": { "architecture": "unknown", "os": "unknown" } } ] }`, + // AMD64 + "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355": `{ "mediaType": "application/vnd.oci.image.manifest.v1+json", "schemaVersion": 2, "config": { "mediaType": "application/vnd.oci.image.config.v1+json", "digest": "sha256:d715ba0d85ee7d37da627d0679652680ed2cb23dde6120f25143a0b8079ee47e", "size": 2842 }, "layers": [ { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:a7ca0d9ba68fdce7e15bc0952d3e898e970548ca24d57698725836c039086639", "size": 103732 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", "size": 21202 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", "size": 716491 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", "size": 317 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", "size": 198 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", "size": 113 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", "size": 385 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", "size": 355 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", "size": 130562 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:76f3a495ffdc00c612747ba0c59fc56d0a2610d2785e80e9edddbf214c2709ef", "size": 36529876 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", "size": 32 } ] }`, + // ARM64 + "sha256:dce623533c59af554b85f859e91fc1cbb7f574e873c82f36b9ea05a09feb0b53": `{ "mediaType": "application/vnd.oci.image.manifest.v1+json", "schemaVersion": 2, "config": { "mediaType": "application/vnd.oci.image.config.v1+json", "digest": "sha256:c73129c9fb699b620aac2df472196ed41797fd0f5a90e1942bfbf19849c4a1c9", "size": 2842 }, "layers": [ { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:0b41f743fd4d78cb50ba86dd3b951b51458744109e1f5063a76bc5a792c3d8e7", "size": 103732 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", "size": 21202 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", "size": 716491 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", "size": 317 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", "size": 198 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", "size": 113 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", "size": 385 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", "size": 355 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", "size": 130562 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:0dc769edeab7d9f622b9703579f6c89298a4cf45a84af1908e26fffca55341e1", "size": 34168923 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", "size": 32 } ] }`, + // ARM + "sha256:0ad7c556c55464fa44d4c41e5236715e015b0266daced62140fb5c6b983c946b": `{ "mediaType": "application/vnd.oci.image.manifest.v1+json", "schemaVersion": 2, "config": { "mediaType": "application/vnd.oci.image.config.v1+json", "digest": "sha256:1079836371d57a148a0afa5abfe00bd91825c869fcc6574a418f4371d53cab4c", "size": 2855 }, "layers": [ { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:b437b30b8b4cc4e02865517b5ca9b66501752012a028e605da1c98beb0ed9f50", "size": 103732 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", "size": 21202 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", "size": 716491 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", "size": 317 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", "size": 198 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", "size": 113 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", "size": 385 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", "size": 355 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", "size": 130562 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:01d28554416aa05390e2827a653a1289a2a549e46cc78d65915a75377c6008ba", "size": 34318536 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", "size": 32 } ] }`, + }, + } + is := &mocks.MockImageStore{ + Data: map[string]images.Image{ + "ghcr.io/distribution/distribution:v0.0.8-with-media-type": { + Target: ocispec.Descriptor{MediaType: "application/vnd.oci.image.index.v1+json", Digest: digest.Digest("sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a")}, + Name: "ghcr.io/distribution/distribution:v0.0.8-with-media-type", + }, + "mcr.microsoft.com/distribution/distribution:v0.0.8-without-media-type": { + Target: ocispec.Descriptor{MediaType: "application/vnd.oci.image.index.v1+json", Digest: digest.Digest("sha256:e2db0e6787216c5abfc42ea8ec82812e41782f3bc6e3b5221d5ef9c800e6c507")}, + Name: "mcr.microsoft.com/distribution/distribution:v0.0.8-without-media-type", + }, + }, + } + + client, err := containerd.New("", containerd.WithServices(containerd.WithImageStore(is), containerd.WithContentStore(cs))) + require.NoError(t, err) + + for _, tt := range []struct { + hosts []string + expected []Reference + errExpected bool + }{ + { + []string{"ghcr.io"}, + []Reference{ + &reference{ + name: "ghcr.io/distribution/distribution:v0.0.8-with-media-type", + host: "ghcr.io", + tag: "v0.0.8-with-media-type", + dgst: "sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a", + repo: "distribution/distribution", + }, + }, + false, + }, + { + []string{"mcr.microsoft.com"}, + []Reference{ + &reference{ + name: "mcr.microsoft.com/distribution/distribution:v0.0.8-without-media-type", + host: "mcr.microsoft.com", + tag: "v0.0.8-without-media-type", + dgst: "sha256:e2db0e6787216c5abfc42ea8ec82812e41782f3bc6e3b5221d5ef9c800e6c507", + repo: "distribution/distribution", + }, + }, + false, + }, + { + []string{"ghcr.io", "mcr.microsoft.com"}, + []Reference{ + &reference{ + name: "ghcr.io/distribution/distribution:v0.0.8-with-media-type", + host: "ghcr.io", + tag: "v0.0.8-with-media-type", + dgst: "sha256:e80e36564e9617f684eb5972bf86dc9e9e761216e0d40ff78ca07741ec70725a", + repo: "distribution/distribution", + }, + &reference{ + name: "mcr.microsoft.com/distribution/distribution:v0.0.8-without-media-type", + host: "mcr.microsoft.com", + tag: "v0.0.8-without-media-type", + dgst: "sha256:e2db0e6787216c5abfc42ea8ec82812e41782f3bc6e3b5221d5ef9c800e6c507", + repo: "distribution/distribution", + }, + }, + false, + }, + } { + t.Run(fmt.Sprintf("%v-%v", tt.hosts, tt.errExpected), func(t *testing.T) { + s, err := newStore(tt.hosts, client) + require.NoError(t, err) + + got, err := s.List(context.Background()) + if tt.errExpected { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tt.expected, got) + } + }) + } +} + +func TestWrite(t *testing.T) { + man := `{ "mediaType": "application/vnd.oci.image.manifest.v1+json", "schemaVersion": 2, "config": { "mediaType": "application/vnd.oci.image.config.v1+json", "digest": "sha256:d715ba0d85ee7d37da627d0679652680ed2cb23dde6120f25143a0b8079ee47e", "size": 2842 }, "layers": [ { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:a7ca0d9ba68fdce7e15bc0952d3e898e970548ca24d57698725836c039086639", "size": 103732 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", "size": 21202 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", "size": 716491 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", "size": 317 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", "size": 198 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", "size": 113 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", "size": 385 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", "size": 355 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", "size": 130562 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:76f3a495ffdc00c612747ba0c59fc56d0a2610d2785e80e9edddbf214c2709ef", "size": 36529876 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", "size": 32 } ] }` + + cs := &mocks.MockContentStore{ + Data: map[string]string{ + "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355": man, + }, + } + is := &mocks.MockImageStore{ + Data: map[string]images.Image{ + "ghcr.io/distribution/distribution:v0.0.8": { + Target: ocispec.Descriptor{MediaType: "application/vnd.oci.image.index.v1+json", Digest: digest.Digest("sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355")}, + }, + }, + } + + client, err := containerd.New("", containerd.WithServices(containerd.WithImageStore(is), containerd.WithContentStore(cs))) + require.NoError(t, err) + + for _, tt := range []struct { + d digest.Digest + expected string + errExpected bool + }{ + { + "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355", + man, + false, + }, + { + "ghcr.io/distribution/distribution:latest", + "", + true, + }, + } { + t.Run(fmt.Sprintf("%v-%v", tt.d, tt.errExpected), func(t *testing.T) { + s := store{ + client: client, + platform: platforms.Only(platforms.MustParse("linux/amd64")), + } + + var buf bytes.Buffer + + err := s.Write(context.Background(), &buf, tt.d) + if tt.errExpected { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tt.expected, buf.String()) + } + }) + } +} + +func TestSubscribe(t *testing.T) { + man := `{ "mediaType": "application/vnd.oci.image.manifest.v1+json", "schemaVersion": 2, "config": { "mediaType": "application/vnd.oci.image.config.v1+json", "digest": "sha256:d715ba0d85ee7d37da627d0679652680ed2cb23dde6120f25143a0b8079ee47e", "size": 2842 }, "layers": [ { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:a7ca0d9ba68fdce7e15bc0952d3e898e970548ca24d57698725836c039086639", "size": 103732 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fe5ca62666f04366c8e7f605aa82997d71320183e99962fa76b3209fdfbb8b58", "size": 21202 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:b02a7525f878e61fc1ef8a7405a2cc17f866e8de222c1c98fd6681aff6e509db", "size": 716491 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fcb6f6d2c9986d9cd6a2ea3cc2936e5fc613e09f1af9042329011e43057f3265", "size": 317 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:e8c73c638ae9ec5ad70c49df7e484040d889cca6b4a9af056579c3d058ea93f0", "size": 198 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:1e3d9b7d145208fa8fa3ee1c9612d0adaac7255f1bbc9ddea7e461e0b317805c", "size": 113 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4aa0ea1413d37a58615488592a0b827ea4b2e48fa5a77cf707d0e35f025e613f", "size": 385 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:7c881f9ab25e0d86562a123b5fb56aebf8aa0ddd7d48ef602faf8d1e7cf43d8c", "size": 355 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:5627a970d25e752d971a501ec7e35d0d6fdcd4a3ce9e958715a686853024794a", "size": 130562 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:76f3a495ffdc00c612747ba0c59fc56d0a2610d2785e80e9edddbf214c2709ef", "size": 36529876 }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", "size": 32 } ] }` + + cs := &mocks.MockContentStore{ + Data: map[string]string{ + "sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355": man, + }, + } + is := &mocks.MockImageStore{ + Data: map[string]images.Image{ + "ghcr.io/distribution/distribution:v0.0.8": { + Target: ocispec.Descriptor{MediaType: "application/vnd.oci.image.index.v1+json", Digest: digest.Digest("sha256:44cb2cf712c060f69df7310e99339c1eb51a085446f1bb6d44469acff35b4355")}, + Name: "ghcr.io/distribution/distribution:v0.0.8", + }, + }, + } + + es := &mocks.MockEventService{ + EnvelopeChan: make(chan *events.Envelope), + ErrorsChan: make(chan error), + } + + client, err := containerd.New("", containerd.WithServices(containerd.WithImageStore(is), containerd.WithContentStore(cs), containerd.WithEventService(es))) + require.NoError(t, err) + + s, err := newStore([]string{"ghcr.io"}, client) + require.NoError(t, err) + + gotEnvCh, gotErrCh := s.Subscribe(context.Background()) + require.NotNil(t, gotEnvCh) + require.NotNil(t, gotErrCh) + + testDone := make(chan struct{}) + defer func() { + testDone <- struct{}{} + close(testDone) + }() + errorCount := 0 + eventsCount := 0 + totalCount := 0 + + go func() { + for { + select { + case <-gotEnvCh: + totalCount++ + eventsCount++ + + if eventsCount > 1 { + t.Errorf("got %d events, want 1", eventsCount) + } + + case e := <-gotErrCh: + totalCount++ + errorCount++ + + // Expect one error for the unexpected event. + if errorCount > 1 { + t.Errorf("got %d errors, want 1: %v", errorCount, e) + } + + case <-testDone: + return + } + } + }() + + // Send an unexpected event. + delEvent := eventtypes.ImageDelete{Name: "ghcr.io/distribution/distribution:v0.0.8"} + delAny, err := typeurl.MarshalAny(&delEvent) + require.NoError(t, err) + go func() { + es.EnvelopeChan <- &events.Envelope{ + Timestamp: time.Time{}, + Namespace: DefaultNamespace, + Topic: "unexpected", + Event: delAny, + } + }() + + for { + if totalCount >= 1 { + break + } + time.Sleep(10 * time.Millisecond) + } + + require.Equal(t, 1, errorCount) + require.Equal(t, 0, eventsCount) + require.Equal(t, 1, totalCount) + + // Send an image create event. + createEvent := eventtypes.ImageCreate{Name: "ghcr.io/distribution/distribution:v0.0.8"} + createAny, err := typeurl.MarshalAny(&createEvent) + require.NoError(t, err) + go func() { + es.EnvelopeChan <- &events.Envelope{ + Timestamp: time.Time{}, + Namespace: DefaultNamespace, + Topic: "image-create", + Event: createAny, + } + }() + + for { + if totalCount >= 2 { + break + } + time.Sleep(10 * time.Millisecond) + } + + require.Equal(t, 1, errorCount) // no new error + require.Equal(t, 1, eventsCount) + require.Equal(t, 2, totalCount) +} diff --git a/pkg/k8s/election/election.go b/pkg/k8s/election/election.go new file mode 100644 index 0000000..8a1edad --- /dev/null +++ b/pkg/k8s/election/election.go @@ -0,0 +1,124 @@ +package election + +import ( + "context" + "sync" + "time" + + "github.com/azure/peerd/pkg/k8s" + "github.com/multiformats/go-multiaddr" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" +) + +// LeaderElection provides an interface to elect a leader in a kubernetes cluster. +type LeaderElection interface { + // RunOrDie runs the leader election. + RunOrDie(ctx context.Context, id string) error + + // Leader gets the address of the elected leader. + Leader() (multiaddr.Multiaddr, error) +} + +// leaderElection provides an implementation of LeaderElection. +type leaderElection struct { + // ns is the namespace in which to run the leader election. + ns string + + // name is the name of the leader election resource in the cluster. + name string + + // id is the id of the elected leader. + id string + + cs kubernetes.Interface + initChan chan interface{} + mx sync.RWMutex +} + +var _ LeaderElection = &leaderElection{} + +// Leader gets the address of the elected leader. +func (le *leaderElection) Leader() (multiaddr.Multiaddr, error) { + <-le.initChan + le.mx.RLock() + defer le.mx.RUnlock() + + addr, err := multiaddr.NewMultiaddr(le.id) + if err != nil { + return nil, err + } + return addr, nil +} + +// RunOrDie runs the leader election. +func (le *leaderElection) RunOrDie(ctx context.Context, id string) error { + lockCfg := resourcelock.ResourceLockConfig{ + Identity: id, + } + + rl, err := resourcelock.New(resourcelock.LeasesResourceLock, le.ns, le.name, le.cs.CoreV1(), le.cs.CoordinationV1(), lockCfg) + if err != nil { + return err + } + + go leaderelection.RunOrDie(ctx, le.leaderElectionConfig(rl)) + return nil +} + +// leaderElectionConfig creates a new configuration for the leader election. +func (le *leaderElection) leaderElectionConfig(rl resourcelock.Interface) leaderelection.LeaderElectionConfig { + return leaderelection.LeaderElectionConfig{ + Lock: rl, + ReleaseOnCancel: true, + LeaseDuration: 10 * time.Second, + RenewDeadline: 5 * time.Second, + RetryPeriod: 2 * time.Second, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(_ context.Context) {}, + OnStoppedLeading: func() {}, + OnNewLeader: le.onNewLeader, + }, + } +} + +// onNewLeader is called when a new leader is elected. +// It updates the leaderElection instance with the new leader's identity. +func (le *leaderElection) onNewLeader(identity string) { + if identity == resourcelock.UnknownLeader { + return + } + + select { + case <-le.initChan: + break + default: + // A leader has been elected. + close(le.initChan) + } + + le.mx.Lock() + defer le.mx.Unlock() + le.id = identity +} + +// New build a new LeaderElection instance in the given namespace, with the given name. +// The kubeConfigPath is used to create the kubernetes interface. It may be empty if the runtime environment is a pod. +func New(namespace, name, kubeConfigPath string) LeaderElection { + cs, err := k8s.NewKubernetesInterface(kubeConfigPath) + if err != nil { + panic(err) + } + + return newLeaderElection(namespace, name, cs) +} + +func newLeaderElection(namespace, name string, cs kubernetes.Interface) *leaderElection { + return &leaderElection{ + ns: namespace, + name: name, + cs: cs, + initChan: make(chan interface{}), + } +} diff --git a/pkg/k8s/election/election_test.go b/pkg/k8s/election/election_test.go new file mode 100644 index 0000000..52d9129 --- /dev/null +++ b/pkg/k8s/election/election_test.go @@ -0,0 +1,118 @@ +package election + +import ( + "context" + "testing" + "time" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/leaderelection/resourcelock" +) + +func TestLeader(t *testing.T) { + cs := &kubernetes.Clientset{} + le := newLeaderElection("test", "test", cs) + if le == nil { + t.Fatal("expected leader election") + } + + // Ensure no leader yet. + if le.id != "" { + t.Fatal("expected no leader") + } + + // Ensure no leader is returned as long as we haven't started the leader election. + timeout, cancelFunc := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancelFunc() + leaderElected := make(chan bool) + + go func() { + defer close(leaderElected) + _, err := le.Leader() + if err != nil { + t.Errorf("expected no error, got %s", err) + } + leaderElected <- true + }() + + select { + case <-leaderElected: + t.Fatal("expected no leader") + case <-timeout.Done(): + // No leader found as expected. + } + + leaderAddr := "/ip4/127.0.0.1/tcp/5000" + + // Simulate leader election. + le.onNewLeader(leaderAddr) + + got, err := le.Leader() + if err != nil { + t.Fatal(err) + } + + if got.String() != leaderAddr { + t.Fatalf("expected leader %s, got %s", leaderAddr, got.String()) + } + + // Simulate leader unknown. + le.onNewLeader(resourcelock.UnknownLeader) + + got, err = le.Leader() + if err != nil { + t.Fatal(err) + } + + // We still use the previously available leader. + if got.String() != leaderAddr { + t.Fatalf("expected leader %s, got %s", leaderAddr, got.String()) + } + + leader2Addr := "/ip4/127.0.0.1/tcp/5001" + + // Simulate new leader elected. + le.onNewLeader(leader2Addr) + + got, err = le.Leader() + if err != nil { + t.Fatal(err) + } + + if got.String() != leader2Addr { + t.Fatalf("expected leader %s, got %s", leader2Addr, got.String()) + } +} + +func TestLeaderElectionConfig(t *testing.T) { + cs := &kubernetes.Clientset{} + le := newLeaderElection("test", "test", cs) + if le == nil { + t.Fatal("expected leader election") + } + + c := le.leaderElectionConfig(nil) + if c.Callbacks.OnNewLeader == nil { + t.Fatal("expected onNewLeader callback") + } + + if !c.ReleaseOnCancel { + t.Fatal("expected release on cancel to be true") + } +} + +func TestUnexpectedLeaderId(t *testing.T) { + cs := &kubernetes.Clientset{} + le := newLeaderElection("test", "test", cs) + if le == nil { + t.Fatal("expected leader election") + } + + le.id = "not-a-multi-addr" + close(le.initChan) + + _, err := le.Leader() + if err == nil { + t.Fatal("expected error") + } +} diff --git a/pkg/k8s/k8s.go b/pkg/k8s/k8s.go new file mode 100644 index 0000000..c1c04ed --- /dev/null +++ b/pkg/k8s/k8s.go @@ -0,0 +1,27 @@ +package k8s + +import ( + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// NewKubernetesInterface creates a new interface for k8s API server. +// The current runtime environment is first assumed to be a pod and its identity is used to create the interface. +// If a pod is not detected, the given kubeConfigPath is used to create the interface. +func NewKubernetesInterface(kubeConfigPath string) (kubernetes.Interface, error) { + config, err := rest.InClusterConfig() // Assume run in a Pod or an environment with appropriate env variables set. + if err != nil { + config, err = clientcmd.BuildConfigFromFlags("", kubeConfigPath) + if err != nil { + return nil, err + } + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + + return clientset, nil +} diff --git a/pkg/k8s/k8s_test.go b/pkg/k8s/k8s_test.go new file mode 100644 index 0000000..023d612 --- /dev/null +++ b/pkg/k8s/k8s_test.go @@ -0,0 +1,12 @@ +package k8s + +import ( + "testing" +) + +func TestEmptyConfigOutsidePod(t *testing.T) { + _, err := NewKubernetesInterface("") + if err == nil { + t.Error("Expected non-nil error, got nil") + } +} diff --git a/pkg/math/compare.go b/pkg/math/compare.go new file mode 100644 index 0000000..78889d3 --- /dev/null +++ b/pkg/math/compare.go @@ -0,0 +1,25 @@ +package math + +// Max64 returns the larger of x or y. +func Max64(x, y int64) int64 { + if x > y { + return x + } + return y +} + +// Min64 returns the smaller of x or y. +func Min64(x, y int64) int64 { + if x < y { + return x + } + return y +} + +// Min returns the smaller of x or y. +func Min(x, y int) int { + if x < y { + return x + } + return y +} diff --git a/pkg/math/compare_test.go b/pkg/math/compare_test.go new file mode 100644 index 0000000..b9e22c8 --- /dev/null +++ b/pkg/math/compare_test.go @@ -0,0 +1,57 @@ +package math + +import "testing" + +func TestMax64(t *testing.T) { + for _, tc := range []struct { + x, y int64 + want int64 + }{ + {1, 2, 2}, + {2, 1, 2}, + {1, 1, 1}, + {-1, 1, 1}, + {1000, 0, 1000}, + } { + got := Max64(tc.x, tc.y) + if got != tc.want { + t.Errorf("expected: %v, got: %v", tc.want, got) + } + } +} + +func TestMin64(t *testing.T) { + for _, tc := range []struct { + x, y int64 + want int64 + }{ + {1, 2, 1}, + {2, 1, 1}, + {1, 1, 1}, + {-1, 1, -1}, + {1000, 0, 0}, + } { + got := Min64(tc.x, tc.y) + if got != tc.want { + t.Errorf("expected: %v, got: %v", tc.want, got) + } + } +} + +func TestMin(t *testing.T) { + for _, tc := range []struct { + x, y int + want int + }{ + {1, 2, 1}, + {2, 1, 1}, + {1, 1, 1}, + {-1, 1, -1}, + {1000, 0, 0}, + } { + got := Min(tc.x, tc.y) + if got != tc.want { + t.Errorf("expected: %v, got: %v", tc.want, got) + } + } +} diff --git a/pkg/math/segments.go b/pkg/math/segments.go new file mode 100644 index 0000000..9f04735 --- /dev/null +++ b/pkg/math/segments.go @@ -0,0 +1,49 @@ +package math + +import "fmt" + +// Segments represents a range of segments. +type Segments struct { + offset int64 + step int + size int64 +} + +// Segment represents a single segment. +type Segment struct { + Index int64 + Offset int64 + Count int +} + +// NewSegments creates a new Segments object. +func NewSegments(offset int64, step int, count int64, size int64) (Segments, error) { + if (step & (step - 1)) > 0 { + return Segments{}, fmt.Errorf("step must be power of 2, got %d", step) + } + return Segments{offset, step, Min64(offset+count, size)}, nil +} + +// AlignDown will align down the x by align. For example: +// AlignDown(1, 2) = 0 +// AlignDown(29, 14) = 28 +func AlignDown(x int64, align int64) int64 { + return x / align * align +} + +// All provides a channel of all segments. +func (r Segments) All() chan Segment { + ch := make(chan Segment) + go func() { + for i := AlignDown(r.offset, int64(r.step)); i < r.size; i += int64(r.step) { + absOffset := Max64(i, r.offset) + seg := Segment{Index: i, Offset: absOffset - i} + seg.Count = int(Min64(i+int64(r.step), r.size) - absOffset) + if seg.Count > 0 { + ch <- seg + } + } + close(ch) + }() + return ch +} diff --git a/pkg/math/segments_test.go b/pkg/math/segments_test.go new file mode 100644 index 0000000..f526617 --- /dev/null +++ b/pkg/math/segments_test.go @@ -0,0 +1,143 @@ +package math + +import "testing" + +func TestNewSegments(t *testing.T) { + _, err := NewSegments(0, 3, 100, 100) + if err == nil { + t.Fatal("expected error") + } +} + +func TestAlignDown(t *testing.T) { + for _, testcase := range []struct { + x int64 + align int64 + expected int64 + }{ + { + x: 1, + align: 2, + expected: 0, + }, + { + x: 29, + align: 14, + expected: 28, + }, + { + x: 0, + align: 2, + expected: 0, + }, + { + x: 2, + align: 2, + expected: 2, + }, + { + x: 2147483647, + align: 2, + expected: 2147483646, + }, + { + x: 2147483647, + align: 4, + expected: 2147483644, + }, + { + x: 2147483647, + align: 8, + expected: 2147483640, + }, + { + x: 2147483647, + align: 16, + expected: 2147483632, + }, + { + x: 2147483647, + align: 32, + expected: 2147483616, + }, + } { + got := AlignDown(testcase.x, testcase.align) + + if got != testcase.expected { + t.Errorf("expected: %v, got: %v", testcase.expected, got) + } + } +} + +func TestAll(t *testing.T) { + for _, testcase := range []struct { + offset int64 + step int + count int64 + size int64 + expected []Segment + }{ + { + offset: 0, + step: 4, + count: 10, + size: 10, + expected: []Segment{ + {Index: 0, Offset: 0, Count: 4}, + {Index: 4, Offset: 0, Count: 4}, + {Index: 8, Offset: 0, Count: 2}, + }, + }, + { + offset: 3, + step: 2, + count: 9, + size: 15, + expected: []Segment{ + {Index: 2, Offset: 1, Count: 1}, + {Index: 4, Offset: 0, Count: 2}, + {Index: 6, Offset: 0, Count: 2}, + {Index: 8, Offset: 0, Count: 2}, + {Index: 10, Offset: 0, Count: 2}, + }, + }, + { + offset: 0, + step: 4, + count: 10, + size: 2147483647, + expected: []Segment{ + {Index: 0, Offset: 0, Count: 4}, + {Index: 4, Offset: 0, Count: 4}, + {Index: 8, Offset: 0, Count: 2}, + }, + }, + { + offset: 3, + step: 2, + count: 9, + size: 2147483647, + expected: []Segment{ + {Index: 2, Offset: 1, Count: 1}, + {Index: 4, Offset: 0, Count: 2}, + {Index: 6, Offset: 0, Count: 2}, + {Index: 8, Offset: 0, Count: 2}, + {Index: 10, Offset: 0, Count: 2}, + }, + }, + } { + segs, err := NewSegments(testcase.offset, testcase.step, testcase.count, testcase.size) + if err != nil { + t.Error(err) + } + + i := 0 + for seg := range segs.All() { + expected := testcase.expected[i] + if expected != seg { + t.Errorf("expected: %v, got: %v", expected, seg) + } + i++ + } + } +} diff --git a/pkg/mocks/contentstore.go b/pkg/mocks/contentstore.go new file mode 100644 index 0000000..75f9d89 --- /dev/null +++ b/pkg/mocks/contentstore.go @@ -0,0 +1,74 @@ +package mocks + +import ( + "bytes" + "context" + "fmt" + + "github.com/containerd/containerd/content" + "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +// MockContentStore is a mock implementation of containerd's content store. +type MockContentStore struct { + Data map[string]string +} + +var _ content.Store = &MockContentStore{} + +// Info returns the content.Info for the given digest, if it exists in the mocked data keyed by digest. +func (m *MockContentStore) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) { + if d, ok := m.Data[dgst.String()]; ok { + return content.Info{ + Digest: dgst, + Size: int64(len(d)), + }, nil + } + return content.Info{}, fmt.Errorf("digest not found: %s", dgst.String()) +} + +func (*MockContentStore) Walk(ctx context.Context, fn content.WalkFunc, filters ...string) error { + panic("not implemented") +} + +func (*MockContentStore) Delete(ctx context.Context, dgst digest.Digest) error { + panic("not implemented") +} + +// ReaderAt returns a content.ReaderAt for the given descriptor, if it exists in the mocked data keyed by digest. +func (m *MockContentStore) ReaderAt(ctx context.Context, desc v1.Descriptor) (content.ReaderAt, error) { + s, ok := m.Data[desc.Digest.String()] + if !ok { + return nil, fmt.Errorf("digest not found: %s", desc.Digest.String()) + } + return &readerAt{*bytes.NewReader([]byte(s))}, nil +} + +func (*MockContentStore) Status(ctx context.Context, ref string) (content.Status, error) { + panic("not implemented") +} + +func (*MockContentStore) Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) { + panic("not implemented") +} + +func (*MockContentStore) ListStatuses(ctx context.Context, filters ...string) ([]content.Status, error) { + panic("not implemented") +} + +func (*MockContentStore) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) { + panic("not implemented") +} + +func (*MockContentStore) Abort(ctx context.Context, ref string) error { + panic("not implemented") +} + +type readerAt struct { + bytes.Reader +} + +func (r *readerAt) Close() error { + return nil +} diff --git a/pkg/mocks/eventservice.go b/pkg/mocks/eventservice.go new file mode 100644 index 0000000..4b50364 --- /dev/null +++ b/pkg/mocks/eventservice.go @@ -0,0 +1,30 @@ +package mocks + +import ( + "context" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/events" +) + +type MockEventService struct { + EnvelopeChan chan *events.Envelope + ErrorsChan chan error +} + +// Forward implements containerd.EventService. +func (*MockEventService) Forward(ctx context.Context, envelope *events.Envelope) error { + panic("unimplemented") +} + +// Publish implements containerd.EventService. +func (*MockEventService) Publish(ctx context.Context, topic string, event events.Event) error { + panic("unimplemented") +} + +// Subscribe implements containerd.EventService. +func (m *MockEventService) Subscribe(ctx context.Context, filters ...string) (ch <-chan *events.Envelope, errs <-chan error) { + return m.EnvelopeChan, m.ErrorsChan +} + +var _ containerd.EventService = &MockEventService{} diff --git a/pkg/mocks/host.go b/pkg/mocks/host.go new file mode 100644 index 0000000..8cdd8d9 --- /dev/null +++ b/pkg/mocks/host.go @@ -0,0 +1,75 @@ +package mocks + +import ( + "context" + + "github.com/libp2p/go-libp2p/core/connmgr" + "github.com/libp2p/go-libp2p/core/event" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/peerstore" + "github.com/libp2p/go-libp2p/core/protocol" + multiaddr "github.com/multiformats/go-multiaddr" +) + +// MockHost provides a mock implementation of host.Host for unit testing. +type MockHost struct { + PeerStore peerstore.Peerstore +} + +var _ host.Host = &MockHost{} + +func (*MockHost) Addrs() []multiaddr.Multiaddr { + panic("unimplemented") +} + +func (*MockHost) Close() error { + panic("unimplemented") +} + +func (*MockHost) ConnManager() connmgr.ConnManager { + panic("unimplemented") +} + +func (*MockHost) Connect(ctx context.Context, pi peer.AddrInfo) error { + panic("unimplemented") +} + +func (*MockHost) EventBus() event.Bus { + panic("unimplemented") +} + +// ID returns the peer ID of this host. +func (*MockHost) ID() peer.ID { + return "localhost-peer-for-unit-testing" +} + +func (*MockHost) Mux() protocol.Switch { + panic("unimplemented") +} + +func (*MockHost) Network() network.Network { + panic("unimplemented") +} + +func (*MockHost) NewStream(ctx context.Context, p peer.ID, pids ...protocol.ID) (network.Stream, error) { + panic("unimplemented") +} + +// Peerstore returns the mocked peerstore of this host. +func (m *MockHost) Peerstore() peerstore.Peerstore { + return m.PeerStore +} + +func (*MockHost) RemoveStreamHandler(pid protocol.ID) { + panic("unimplemented") +} + +func (*MockHost) SetStreamHandler(pid protocol.ID, handler network.StreamHandler) { + panic("unimplemented") +} + +func (*MockHost) SetStreamHandlerMatch(protocol.ID, func(protocol.ID) bool, network.StreamHandler) { + panic("unimplemented") +} diff --git a/pkg/mocks/imagestore.go b/pkg/mocks/imagestore.go new file mode 100644 index 0000000..0aa0e67 --- /dev/null +++ b/pkg/mocks/imagestore.go @@ -0,0 +1,59 @@ +package mocks + +import ( + "context" + "fmt" + "strings" + + "github.com/containerd/containerd/images" +) + +// MockImageStore is a mock implementation of containerd's image store. +type MockImageStore struct { + Data map[string]images.Image +} + +var _ images.Store = &MockImageStore{} + +// Get gets an image by name if it exists in the mocked data keyed by name. +func (m *MockImageStore) Get(ctx context.Context, name string) (images.Image, error) { + img, ok := m.Data[name] + if !ok { + return images.Image{}, fmt.Errorf("image with name %s does not exist", name) + } + return img, nil +} + +// List lists the images in the image store filtered by the given filters. +// Note that only some filters are recognized by this mock implementation. +func (m *MockImageStore) List(ctx context.Context, filters ...string) ([]images.Image, error) { + result := []images.Image{} + for _, filter := range filters { + if strings.HasPrefix(filter, "name~=") { + n := strings.TrimLeft(filter, "name~=") + names := strings.Split(n, "|") + for _, name := range names { + name = strings.TrimLeft(name, "\"") + name = strings.TrimRight(name, "\"") + for k, v := range m.Data { + if strings.HasPrefix(k, name) { + result = append(result, v) + } + } + } + } + } + return result, nil +} + +func (*MockImageStore) Create(ctx context.Context, image images.Image) (images.Image, error) { + return images.Image{}, nil +} + +func (*MockImageStore) Update(ctx context.Context, image images.Image, fieldpaths ...string) (images.Image, error) { + return images.Image{}, nil +} + +func (*MockImageStore) Delete(ctx context.Context, name string, opts ...images.DeleteOpt) error { + return nil +} diff --git a/pkg/mocks/peerstore.go b/pkg/mocks/peerstore.go new file mode 100644 index 0000000..7c6ca88 --- /dev/null +++ b/pkg/mocks/peerstore.go @@ -0,0 +1,135 @@ +package mocks + +import ( + "context" + "time" + + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/peerstore" + "github.com/libp2p/go-libp2p/core/protocol" + multiaddr "github.com/multiformats/go-multiaddr" +) + +// MockPeerstore provides a mock implementation of peerstore.Peerstore for unit testing. +type MockPeerstore struct{} + +var _ peerstore.Peerstore = &MockPeerstore{} + +func (*MockPeerstore) AddAddr(p peer.ID, addr multiaddr.Multiaddr, ttl time.Duration) { + panic("unimplemented") +} + +func (*MockPeerstore) AddAddrs(p peer.ID, addrs []multiaddr.Multiaddr, ttl time.Duration) { + panic("unimplemented") +} + +func (*MockPeerstore) AddPrivKey(peer.ID, crypto.PrivKey) error { + panic("unimplemented") +} + +func (*MockPeerstore) AddProtocols(peer.ID, ...protocol.ID) error { + panic("unimplemented") +} + +func (*MockPeerstore) AddPubKey(peer.ID, crypto.PubKey) error { + panic("unimplemented") +} + +func (*MockPeerstore) AddrStream(context.Context, peer.ID) <-chan multiaddr.Multiaddr { + panic("unimplemented") +} + +func (*MockPeerstore) Addrs(p peer.ID) []multiaddr.Multiaddr { + panic("unimplemented") +} + +func (*MockPeerstore) ClearAddrs(p peer.ID) { + panic("unimplemented") +} + +func (*MockPeerstore) Close() error { + panic("unimplemented") +} + +func (*MockPeerstore) FirstSupportedProtocol(peer.ID, ...protocol.ID) (protocol.ID, error) { + panic("unimplemented") +} + +func (*MockPeerstore) Get(p peer.ID, key string) (interface{}, error) { + panic("unimplemented") +} + +func (*MockPeerstore) GetProtocols(peer.ID) ([]protocol.ID, error) { + panic("unimplemented") +} + +func (*MockPeerstore) LatencyEWMA(peer.ID) time.Duration { + panic("unimplemented") +} + +func (*MockPeerstore) PeerInfo(peer.ID) peer.AddrInfo { + panic("unimplemented") +} + +func (*MockPeerstore) Peers() peer.IDSlice { + panic("unimplemented") +} + +func (*MockPeerstore) PeersWithAddrs() peer.IDSlice { + panic("unimplemented") +} + +func (*MockPeerstore) PeersWithKeys() peer.IDSlice { + panic("unimplemented") +} + +// PrivKey generates a new private key for the given peer. +func (*MockPeerstore) PrivKey(peer.ID) crypto.PrivKey { + // Generate a new key for each peer. + priv, _, err := crypto.GenerateKeyPair(crypto.RSA, 2048) + if err != nil { + panic(err) + } + return priv +} + +func (*MockPeerstore) PubKey(peer.ID) crypto.PubKey { + panic("unimplemented") +} + +func (*MockPeerstore) Put(p peer.ID, key string, val interface{}) error { + panic("unimplemented") +} + +func (*MockPeerstore) RecordLatency(peer.ID, time.Duration) { + panic("unimplemented") +} + +func (*MockPeerstore) RemovePeer(peer.ID) { + panic("unimplemented") +} + +func (*MockPeerstore) RemoveProtocols(peer.ID, ...protocol.ID) error { + panic("unimplemented") +} + +func (*MockPeerstore) SetAddr(p peer.ID, addr multiaddr.Multiaddr, ttl time.Duration) { + panic("unimplemented") +} + +func (*MockPeerstore) SetAddrs(p peer.ID, addrs []multiaddr.Multiaddr, ttl time.Duration) { + panic("unimplemented") +} + +func (*MockPeerstore) SetProtocols(peer.ID, ...protocol.ID) error { + panic("unimplemented") +} + +func (*MockPeerstore) SupportsProtocols(peer.ID, ...protocol.ID) ([]protocol.ID, error) { + panic("unimplemented") +} + +func (*MockPeerstore) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL time.Duration) { + panic("unimplemented") +} diff --git a/pkg/peernet/network.go b/pkg/peernet/network.go new file mode 100644 index 0000000..d2ec0bd --- /dev/null +++ b/pkg/peernet/network.go @@ -0,0 +1,125 @@ +package peernet + +import ( + "crypto/tls" + "net/http" + "time" + + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls" +) + +const ( + // defaultTimeout is the total HTTP timeout that should work with most peers to download 1 Mb of data. + defaultTimeout = 90 * time.Second +) + +var ( + // defaultHttpClient is the default HTTP client that does not authenticate peers. + defaultHttpClient = &http.Client{ + Timeout: defaultTimeout, + } +) + +// Network provides the transport and HTTP clients for communicating with peers. +type Network interface { + // DefaultTLSConfig creates a default TLS config. + // This config should not require client certificate verification. + DefaultTLSConfig() *tls.Config + + // RoundTripperFor returns an HTTP round tripper which authenticates the given peer. + // If pid is empty, the round tripper should work for any peer. + RoundTripperFor(pid peer.ID) http.RoundTripper + + // HTTPClientFor returns an HTTP client which authenticates the given peer. + // If pid is empty, the client should work for any peer. + HTTPClientFor(pid peer.ID) *http.Client +} + +type network struct { + id *libp2ptls.Identity + defaultTLSConfig *tls.Config + defaultTransport *http.Transport +} + +var _ Network = &network{} + +// DefaultTLSConfig creates a TLS config to use for this server. +// This config does not require client certificate verification and is resuable. +func (n *network) DefaultTLSConfig() *tls.Config { + return n.defaultTLSConfig +} + +// HTTPClientFor returns a single use HTTP client for the given peer. +// The client does not verify the peer's certificate. +func (n *network) HTTPClientFor(pid peer.ID) *http.Client { + if pid == "" { + return defaultHttpClient + } + + return &http.Client{ + Transport: n.transportFor(pid), + Timeout: defaultTimeout, + } +} + +// RoundTripperFor returns a single use round tripper for the given peer. +// The peer is expected to provide a valid certificate. +// If pid is empty, the round tripper will work for any peer. +func (n *network) RoundTripperFor(pid peer.ID) http.RoundTripper { + if pid == "" { + return n.defaultTransport + } + + return n.transportFor(pid) +} + +// transportFor returns a single use transport for outbound connection to the given peer. +// The peer is expected to provide a valid certificate. +// If pid is empty, the transport will work for any peer. +func (n *network) transportFor(pid peer.ID) *http.Transport { + if pid == "" { + return n.defaultTransport + } + + p2pTlsConfigForPeer, _ := n.id.ConfigForPeer(pid) + + transport := &http.Transport{ + TLSClientConfig: &tls.Config{ + Certificates: p2pTlsConfigForPeer.Certificates, + ClientAuth: tls.RequireAndVerifyClientCert, + VerifyPeerCertificate: p2pTlsConfigForPeer.VerifyPeerCertificate, + InsecureSkipVerify: true, + }, + MaxConnsPerHost: 100, + } + + return transport +} + +// New creates a new network interface for communicating with peers. +func New(h host.Host) (Network, error) { + privKey := h.Peerstore().PrivKey(h.ID()) + + id, err := libp2ptls.NewIdentity(privKey) + if err != nil { + return nil, err + } + + tlsConfig, _ := id.ConfigForPeer(peer.ID("")) + defaultTLSConfig := &tls.Config{ + Certificates: tlsConfig.Certificates, + } + + defaultTransport := &http.Transport{ + TLSClientConfig: defaultTLSConfig, + MaxConnsPerHost: 100, + } + + return &network{ + id: id, + defaultTLSConfig: defaultTLSConfig, + defaultTransport: defaultTransport, + }, nil +} diff --git a/pkg/peernet/network_test.go b/pkg/peernet/network_test.go new file mode 100644 index 0000000..7ff6fac --- /dev/null +++ b/pkg/peernet/network_test.go @@ -0,0 +1,130 @@ +package peernet + +import ( + "crypto/tls" + "net/http" + "testing" + + "github.com/azure/peerd/pkg/mocks" +) + +func TestNew(t *testing.T) { + h := &mocks.MockHost{PeerStore: &mocks.MockPeerstore{}} + + _, err := New(h) + if err != nil { + t.Fatal(err) + } +} + +func TestDefaultTLSConfig(t *testing.T) { + h := &mocks.MockHost{PeerStore: &mocks.MockPeerstore{}} + + n, err := New(h) + if err != nil { + t.Fatal(err) + } + + config := n.DefaultTLSConfig() + if config == nil { + t.Fatal("expected non-nil TLS config") + } + + if config.Certificates == nil { + t.Fatal("expected non-nil certificates") + } +} + +func TestTransportFor(t *testing.T) { + h := &mocks.MockHost{PeerStore: &mocks.MockPeerstore{}} + + n, err := New(h) + if err != nil { + t.Fatal(err) + } + + testPeerTransport := n.(*network).transportFor("test-peer") + if testPeerTransport == nil { + t.Fatal("expected non-nil transport") + } + + if testPeerTransport.TLSClientConfig == nil { + t.Fatal("expected non-nil TLS config") + } + + if testPeerTransport.TLSClientConfig.VerifyPeerCertificate == nil { + t.Fatal("expected non-nil VerifyPeerCertificate") + } + + if testPeerTransport.TLSClientConfig.ClientAuth != tls.RequireAndVerifyClientCert { + t.Fatal("expected RequireAndVerifyClientCert") + } + + if testPeerTransport.TLSClientConfig.InsecureSkipVerify != true { + t.Fatal("expected InsecureSkipVerify") + } + + testPeerTransport2 := n.(*network).transportFor("test-peer-2") + if testPeerTransport2 == nil { + t.Fatal("expected non-nil transport") + } else if testPeerTransport2 == testPeerTransport { + t.Fatal("expected different transport") + } + + defaultTransport := n.(*network).transportFor("") + if defaultTransport == nil { + t.Fatal("expected non-nil transport") + } + + if defaultTransport.TLSClientConfig == nil { + t.Fatal("expected non-nil TLS config") + } + + if defaultTransport.TLSClientConfig.ClientAuth != tls.NoClientCert { + t.Fatal("expected NoClientCert") + } +} + +func TestHTTPClientFor(t *testing.T) { + h := &mocks.MockHost{PeerStore: &mocks.MockPeerstore{}} + + n, err := New(h) + if err != nil { + t.Fatal(err) + } + + c := n.HTTPClientFor("test-peer") + if c == nil { + t.Fatal("expected non-nil client") + } + + if c.Transport == nil { + t.Fatal("expected non-nil transport") + } + + if c.Timeout == 0 { + t.Fatal("expected non-zero timeout") + } +} + +func TestRoundTripperFor(t *testing.T) { + h := &mocks.MockHost{PeerStore: &mocks.MockPeerstore{}} + + n, err := New(h) + if err != nil { + t.Fatal(err) + } + + c := n.RoundTripperFor("test-peer") + if c == nil { + t.Fatal("expected non-nil client") + } + + if c.(*http.Transport).TLSClientConfig == nil { + t.Fatal("expected non-nil TLS config") + } + + if c.(*http.Transport).TLSClientConfig.ClientAuth != tls.RequireAndVerifyClientCert { + t.Fatal("expected RequireAndVerifyClientCert") + } +} diff --git a/pkg/urlparser/azure.go b/pkg/urlparser/azure.go new file mode 100644 index 0000000..5aa1c2b --- /dev/null +++ b/pkg/urlparser/azure.go @@ -0,0 +1,37 @@ +package urlparser + +import ( + "fmt" + "regexp" + + "github.com/opencontainers/go-digest" +) + +var ( + regexes = []*regexp.Regexp{ + // Azure Container Registry public cloud data endpoints. + regexp.MustCompile(`https:\/\/[a-zA-Z0-9\.]+\.azurecr\.[a-z\.]+\?[a-zA-Z0-9\.\&\=\-]+\&d=sha256:([a-zA-Z0-9]{64})[.]*`), + + // Microsoft Artifact Registry public cloud data endpoints. + regexp.MustCompile(`https:\/\/[a-zA-Z0-9]+\.data.mcr.microsoft.com\/[a-zA-Z0-9\-]+\/\/docker\/registry\/v2\/blobs\/sha256\/[a-z0-9]{2}\/([a-zA-Z0-9]{64})\/data.*`), + + // Azure Blob Storage public cloud blob endpoints. + regexp.MustCompile(`https:\/\/[a-zA-Z0-9]+\.blob\.[a-z\.]+\/[a-zA-Z0-9\-]+\/\/docker\/registry\/v2\/blobs\/sha256\/[a-z0-9]{2}\/([a-zA-Z0-9]{64})\/data.*`), + } +) + +// parseDigestFromAzureUrl parses the digest from the given blob url or returns an error. +func parseDigestFromAzureUrl(url string) (digest.Digest, error) { + if url == "" { + return "", fmt.Errorf("empty url") + } + + for _, r := range regexes { + matches := r.FindStringSubmatch(url) + if len(matches) == 2 { + return digest.Digest("sha256:" + matches[1]), nil + } + } + + return "", fmt.Errorf("unknown url") +} diff --git a/pkg/urlparser/azure_test.go b/pkg/urlparser/azure_test.go new file mode 100644 index 0000000..58e281c --- /dev/null +++ b/pkg/urlparser/azure_test.go @@ -0,0 +1,63 @@ +package urlparser + +import ( + "testing" + + "github.com/opencontainers/go-digest" +) + +var ( + azureTestCases = []struct { + url string + digest string + valid bool + }{ + { + "https://aviral26.eastus.data.azurecr.io?t=A2DC7F4DA8D829EB400019A84F4EF18C88652134&h=aviral26.azurecr.io&c=3d109b5a-6a3e-48fe-aa7f-0c02abc773de&r=spegel&d=sha256:dd5ad9c9c29f04b41a0155c720cf5ccab28ef6d353f1fe17a06c579c70054f0a&p=myOe89N4rv2atPh7zOrPqZAm-hE-ySHDGacBDMzEMkYZP2IAPWnjkZdWyXOi6bpCmplEOw7UwesbL46W7u1UXujVN8rmsw-qp3N0l0U79_Uy4T2ajdvZCE2tvP00zXDgBlEW3J9A-78-P2wOqfwaocBBJJFYjTIgtN82pOBX-mqdqeNqv4a9cykSMLGdicfHINC6H9lXCbowKJNB4sv-PXnl0aL02OhgrB4Ki2F7szOMGYaOG1DEXwWYpn1mYWhytMU8kBPnqvVS39Yo-umiq6A7zJnhkYlVGIzDeOWd-OCV-qfrD4AWjJK8WFv986KlWam5kjs9n-dQetKN9eclNNNEbwvqEV_7pRTvTXUsMNR-BqqTTumUAjB8nII8h18gabzAuN80s1oG4ZF9VeZuFKIeCGlhZj1LwvMays7TV_ILCAcNyexshWI3tWSfrdotK8-LYINqW_pD63iMJBShb1-EZWzSd_mOcYrBHViQaFf_-3qI14aqNrL7ASGb3rzmizH1dFqphsYm7ltQ60CY18zbugsFCob-6yWFggpv6NlJ7ko5B-sT8VY0ljH1zHEFtOvf32pDVKR2hsOJwilpF0yzFSy0_di1OkjmIYnChxovvaCSpiMACRh_N5OPT29D&s=dyqCD1Q1d278Z_4nuZn7k7WcBnbq-D5p0kllhH7LrB8uvcxCrEBydZFZ5fe6UOp30kjmKodMvW88eVoWmNNrvKnRwkuKL9ZkHULPzUHqrVnH8rPZb2GrsUpFVDszPXTt8Z9eptNOCWj9jq15fMW6aWIlYEHC81fHrx_XBU1y6Sg0Bl2scQp0TFxvkl_SR64yzcRrUPMOAfgTLe9ILXZIaagkoEzpgyWk-AwIedBjP9X3Y_yZmMvDb6IPL6trC3rh8qyfG09VmSkWLyAx3OwFtKyk4IK4BNgAB-kg2SHaoQ67jJLf-lR3CDRu6HvJNIa3_7gPBUZMQmpsE7rYvMCNZQ&v=1", + "sha256:dd5ad9c9c29f04b41a0155c720cf5ccab28ef6d353f1fe17a06c579c70054f0a", + true, + }, + { + "https://westus2.data.mcr.microsoft.com/01031d61e1024861afee5d512651eb9f-h36fskt2ei//docker/registry/v2/blobs/sha256/1b/1b930d010525941c1d56ec53b97bd057a67ae1865eebf042686d2a2d18271ced/data?se=2023-09-20T01%3A14%3A49Z&sig=m4Cr%2BYTZHZQlN5LznY7nrTQ4LCIx2OqnDDM3Dpedbhs%3D&sp=r&spr=https&sr=b&sv=2018-03-28®id=01031d61e1024861afee5d512651eb9f", + "sha256:1b930d010525941c1d56ec53b97bd057a67ae1865eebf042686d2a2d18271ced", + true, + }, + { + "https://eusreplstore28.blob.core.windows.net/dd5ad9c9c29f04b4-46d325e77acf422cbc239cd963f8d78d-4643a09878//docker/registry/v2/blobs/sha256/dd/dd5ad9c9c29f04b41a0155c720cf5ccab28ef6d353f1fe17a06c579c70054f0a/data?se=2023-09-20T01%3A15%3A41Z&sig=6V%2FV9T7i373TPyxD4dzXlN16KzEW3GchbULPHg2EKjE%3D&sp=r&spr=https&sr=b&sv=2018-03-28®id=46d325e77acf422cbc239cd963f8d78d", + "sha256:dd5ad9c9c29f04b41a0155c720cf5ccab28ef6d353f1fe17a06c579c70054f0a", + true, + }, + { + "https://aviral26.eastus.data.azurecr.io?t=A2DC7F4DA8D829EB400019A84F4EF18C88652134&h=aviral26.azurecr.io&c=3d109b5a-6a3e-48fe-aa7f-0c02abc773de&r=spegel&d=sha512:dd5ad9c9c29f04b41a0155c720cf5ccab28ef6d353f1fe17a06c579c70054f0a&p=myOe89N4rv2atPh7zOrPqZAm-hE-ySHDGacBDMzEMkYZP2IAPWnjkZdWyXOi6bpCmplEOw7UwesbL46W7u1UXujVN8rmsw-qp3N0l0U79_Uy4T2ajdvZCE2tvP00zXDgBlEW3J9A-78-P2wOqfwaocBBJJFYjTIgtN82pOBX-mqdqeNqv4a9cykSMLGdicfHINC6H9lXCbowKJNB4sv-PXnl0aL02OhgrB4Ki2F7szOMGYaOG1DEXwWYpn1mYWhytMU8kBPnqvVS39Yo-umiq6A7zJnhkYlVGIzDeOWd-OCV-qfrD4AWjJK8WFv986KlWam5kjs9n-dQetKN9eclNNNEbwvqEV_7pRTvTXUsMNR-BqqTTumUAjB8nII8h18gabzAuN80s1oG4ZF9VeZuFKIeCGlhZj1LwvMays7TV_ILCAcNyexshWI3tWSfrdotK8-LYINqW_pD63iMJBShb1-EZWzSd_mOcYrBHViQaFf_-3qI14aqNrL7ASGb3rzmizH1dFqphsYm7ltQ60CY18zbugsFCob-6yWFggpv6NlJ7ko5B-sT8VY0ljH1zHEFtOvf32pDVKR2hsOJwilpF0yzFSy0_di1OkjmIYnChxovvaCSpiMACRh_N5OPT29D&s=dyqCD1Q1d278Z_4nuZn7k7WcBnbq-D5p0kllhH7LrB8uvcxCrEBydZFZ5fe6UOp30kjmKodMvW88eVoWmNNrvKnRwkuKL9ZkHULPzUHqrVnH8rPZb2GrsUpFVDszPXTt8Z9eptNOCWj9jq15fMW6aWIlYEHC81fHrx_XBU1y6Sg0Bl2scQp0TFxvkl_SR64yzcRrUPMOAfgTLe9ILXZIaagkoEzpgyWk-AwIedBjP9X3Y_yZmMvDb6IPL6trC3rh8qyfG09VmSkWLyAx3OwFtKyk4IK4BNgAB-kg2SHaoQ67jJLf-lR3CDRu6HvJNIa3_7gPBUZMQmpsE7rYvMCNZQ&v=1", + "", + false, + }, + { + "https://westus2.data.mcr.microsoft.com/01031d61e1024861afee5d512651eb9f-h36fskt2ei//docker//v2/blobs/sha256/1b/1b930d010525941c1d56ec53b97bd057a67ae1865eebf042686d2a2d18271ced/data?se=2023-09-20T01%3A14%3A49Z&sig=m4Cr%2BYTZHZQlN5LznY7nrTQ4LCIx2OqnDDM3Dpedbhs%3D&sp=r&spr=https&sr=b&sv=2018-03-28®id=01031d61e1024861afee5d512651eb9f", + "", + false, + }, + { + "https://eusreplstore28.blob.core.windows.net/dd5ad9c9c29f04b4-46d325e77acf422cbc239cd963f8d78d-4643a09878//docker/registry/v2/blobs/sha256/dd/data?se=2023-09-20T01%3A15%3A41Z&sig=6V%2FV9T7i373TPyxD4dzXlN16KzEW3GchbULPHg2EKjE%3D&sp=r&spr=https&sr=b&sv=2018-03-28®id=46d325e77acf422cbc239cd963f8d78d", + "", + false, + }, + } +) + +func TestUrls(t *testing.T) { + for _, test := range azureTestCases { + got, err := parseDigestFromAzureUrl(test.url) + if test.valid { + if err != nil { + t.Errorf("expected no error parsing digest from url %s", test.url) + } else if got != digest.Digest(test.digest) { + t.Errorf("expected digest %s, got %s", test.digest, got) + } + } else { + if err == nil { + t.Errorf("expected error parsing digest from url %s", test.url) + } + } + } +} diff --git a/pkg/urlparser/parser.go b/pkg/urlparser/parser.go new file mode 100644 index 0000000..089d878 --- /dev/null +++ b/pkg/urlparser/parser.go @@ -0,0 +1,28 @@ +// Package urlparser provides interfaces and implementations for parsing information from a URL. +package urlparser + +import ( + "github.com/opencontainers/go-digest" +) + +// Parser describes an interface for parsing information from a URL. +type Parser interface { + // ParseDigest parses the digest from the given URL. + // If none found, implementations should return an error. + ParseDigest(url string) (digest.Digest, error) +} + +type parser struct{} + +var _ Parser = &parser{} + +// ParseDigest parses the digest from the given URL. +// If none found, returns an error. +func (p *parser) ParseDigest(url string) (digest.Digest, error) { + return parseDigestFromAzureUrl(url) +} + +// New returns a new Parser. +func New() Parser { + return &parser{} +} diff --git a/pkg/urlparser/parser_test.go b/pkg/urlparser/parser_test.go new file mode 100644 index 0000000..9e8023f --- /dev/null +++ b/pkg/urlparser/parser_test.go @@ -0,0 +1,30 @@ +package urlparser + +import ( + "testing" + + "github.com/opencontainers/go-digest" +) + +func TestParser(t *testing.T) { + p := New() + if p == nil { + t.Errorf("expected non-nil parser") + } + + // Test Azure URLs + for _, test := range azureTestCases { + got, err := p.ParseDigest(test.url) + if test.valid { + if err != nil { + t.Errorf("expected no error parsing digest from url %s", test.url) + } else if got != digest.Digest(test.digest) { + t.Errorf("expected digest %s, got %s", test.digest, got) + } + } else { + if err == nil { + t.Errorf("expected error parsing digest from url %s", test.url) + } + } + } +} diff --git a/scripts/coverage.sh b/scripts/coverage.sh new file mode 100644 index 0000000..8e3e38c --- /dev/null +++ b/scripts/coverage.sh @@ -0,0 +1,94 @@ +#!/bin/bash +set -e + +## Parameters +source_dir=${1?Source directory is required. Please pass as a parameter the absolute path to the directory of the source code to be tested.} +test_pkgs=${2?Test packages are required. Please pass as a parameter the packages to test.} +clear_COVERAGE_DIR=${3:-false} +test_params=${4:-"-timeout 240s"} + +## Variables +script_dir="$( dirname "${BASH_SOURCE[0]}" )" +initial_dir=$( pwd ) + +## Variables depending on sources +results_dir="$dest_dir/$COVERAGE_DIR" + +## Functions +show_help() { + usageStr=" +This script creates coverage for golang projects. +As a part of this, it does the following tasks. + - Installs the following support modules if they are not already installed: + - gotestsum + - gcov2lcov + - gocov + - gocov-xml + - Runs the tests + - This creates the following files: + - coverage.txt + - jsongotest.log + - coverage.xml + - Generates the following test results (if the coverage.txt file was generated): + - coverage/coverage.html + - coverage/coverage.txt + - coverage/lcov.info + - coverage/coverage.cobertura.xml + +Parameters: + Source Directory (required) The absolute path to the directory of the source code to be tested + Test Packages (required) The list of packages to test + Clear Test Results Directory (default: false) Controls whether or not any existing Test Results Directory is cleared + Test Parameters (default: timeout 240s) Additional test parameters to pass to the test wrapper + +EXAMPLES: + coverage.sh /path/to/go/project 'package1 package2 package3 package4' true + - Executes tests related to packages 1-4 in the folder '/path/to/go/project' and clears the test results directory if it exists + coverage.sh /path/to/go/project 'packageA packageB' false '-timeout 5s' + - Executes tests related to packages A and B in the folder '/path/to/go/project', does not clear any existing test results directory, and passes the timeout parameter to the testing wrapper, with a timeout of 5 seconds. +" + echo "$usageStr" +} + +## Main +echo -e "\n------ Generating test results ------\n" + +echo "Current working directory: $initial_dir" +echo "Script directory: $script_dir" +echo -e "Source directory: $source_dir\n" + +# If any of the required modules are not installed, notify the user to install them +if [ -z $(command -v "gotestsum") ] || [ -z $(command -v "gotestsum") ] || [ -z $(command -v "gotestsum") ] || [ -z $(command -v "gotestsum") ]; then + + echo -e "\nPlease install the required modules and run this script again." + echo -e "The script to install the missing modules can be found at $script_dir/install-go-modules.sh" + exit 1 + +fi; + +cd $source_dir + +if [[ $clear_COVERAGE_DIR = true ]] && [ -d "$COVERAGE_DIR" ]; then + echo -e "\nClearing test results directory\n" + rm -rf $COVERAGE_DIR +fi; + +if [ ! -d "$COVERAGE_DIR" ]; then + echo -e "\nCreating test results directory\n" + mkdir -p $COVERAGE_DIR +fi; + +echo -e "\n------ Running tests ------\n" +## coverage.txt format - https://github.com/golang/go/blob/0104a31b8fbcbe52728a08867b26415d282c35d2/src/cmd/cover/profile.go#L56 +gotestsum --format standard-verbose --junitfile $COVERAGE_DIR/coverage.xml --jsonfile $COVERAGE_DIR/jsongotest.log -- -cover -coverprofile=$COVERAGE_DIR/coverage.txt -covermode=atomic $test_params $test_pkgs | tee $COVERAGE_DIR/testoutput.txt + +echo -e "\n------ Generating coverage - lcov ------\n" +GOROOT=$(go env GOROOT) gcov2lcov -infile=$COVERAGE_DIR/coverage.txt -outfile=$COVERAGE_DIR/lcov.info + +echo -e "------ Generating coverage - cobertura ------\n" +GOROOT=$(go env GOROOT) gocov convert $COVERAGE_DIR/coverage.txt | gocov-xml > $COVERAGE_DIR/coverage.cobertura.xml + +echo -e "------ Generating coverage - html ------\n" +go tool cover -html=$COVERAGE_DIR/coverage.txt -o $COVERAGE_DIR/coverage.html + +cd $initial_dir \ No newline at end of file diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 0000000..0a63dfe --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,20 @@ +.PHONY: tests-build +tests-build: ## Builds the tests binary + @echo "+ $@" + @( cd $(ROOT_DIR); $(GOBUILD) -o $(TESTS_BIN_DIR)/tests ./tests/cmd ) + +.PHONY: tests-scanner-image +tests-scanner-image: ## Builds the 'scanner' tests image + @echo "+ $@" +ifndef CONTAINER_REGISTRY + $(eval CONTAINER_REGISTRY := localhost) +endif + $(call build-image-internal,$(ROOT_DIR)/tests/dockerfiles/scanner.Dockerfile,scanner,$(ROOT_DIR)) + +.PHONY: tests-random-image +tests-random-image: ## Builds the 'random' tests image + @echo "+ $@" +ifndef CONTAINER_REGISTRY + $(eval CONTAINER_REGISTRY := localhost) +endif + $(call build-image-internal,$(ROOT_DIR)/tests/dockerfiles/random.Dockerfile,random,$(ROOT_DIR)) \ No newline at end of file diff --git a/tests/cmd/cmd.go b/tests/cmd/cmd.go new file mode 100644 index 0000000..dae5ff1 --- /dev/null +++ b/tests/cmd/cmd.go @@ -0,0 +1,17 @@ +package main + +type RandomCmd struct { + Secrets string `arg:"--secrets" help:"space separated SAS URLs"` + ProxyHost string `arg:"--proxy" help:"The proxy host, such as http://p2p:5000"` + NodeCount int `arg:"--node-count" help:"number of nodes in the p2p network"` +} + +type ScannerCmd struct{} + +type Arguments struct { + Random *RandomCmd `arg:"subcommand:random"` + Scanner *ScannerCmd `arg:"subcommand:scanner"` + Version bool `arg:"-v" help:"show version and exit"` +} + +var version string diff --git a/tests/cmd/main.go b/tests/cmd/main.go new file mode 100644 index 0000000..846aae3 --- /dev/null +++ b/tests/cmd/main.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/alexflint/go-arg" + p2pcontext "github.com/azure/peerd/internal/context" + "github.com/azure/peerd/tests/random" + "github.com/azure/peerd/tests/scanner" + "github.com/rs/zerolog" +) + +func main() { + args := &Arguments{} + arg.MustParse(args) + + zerolog.SetGlobalLevel(zerolog.InfoLevel) + l := zerolog.New(os.Stdout).With().Timestamp().Str("node", p2pcontext.NodeName).Str("version", version).Logger() + ctx := l.WithContext(context.Background()) + + err := run(ctx, args) + if err != nil { + l.Error().Err(err).Msg("error") + os.Exit(1) + } + + l.Info().Msg("shutdown") +} + +func run(ctx context.Context, args *Arguments) error { + ctx, cancel := signal.NotifyContext(ctx, syscall.SIGTERM) + defer cancel() + + switch { + case args.Version: + zerolog.Ctx(ctx).Info().Msg("version") // version field is already added to the logger + return nil + + case args.Random != nil: + return random.Random(ctx, args.Random.Secrets, args.Random.NodeCount, args.Random.ProxyHost) + + case args.Scanner != nil: + return scanner.Scanner(ctx) + + default: + return fmt.Errorf("unknown subcommand") + } +} diff --git a/tests/dockerfiles/random.Dockerfile b/tests/dockerfiles/random.Dockerfile new file mode 100644 index 0000000..23538d7 --- /dev/null +++ b/tests/dockerfiles/random.Dockerfile @@ -0,0 +1,25 @@ +FROM mcr.microsoft.com/oss/go/microsoft/golang:1.22-fips-cbl-mariner2.0 as builder + +COPY ./ /src/ + +RUN tdnf install make -y && \ + tdnf install git -y + +WORKDIR /src + +RUN make tests-build + +FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 as runtime + +ARG USER_ID=6192 + +RUN tdnf update -y && \ + tdnf install ca-certificates-microsoft -y && \ + tdnf install shadow-utils -y + +RUN groupadd -g $USER_ID random && \ + useradd -g random -u $USER_ID random + +COPY --from=builder --chown=scanner:root /src/bin/tests/tests /src/bin/tests/tests + +ENTRYPOINT ["/src/bin/tests/tests", "random"] diff --git a/tests/dockerfiles/scanner.Dockerfile b/tests/dockerfiles/scanner.Dockerfile new file mode 100644 index 0000000..ef78079 --- /dev/null +++ b/tests/dockerfiles/scanner.Dockerfile @@ -0,0 +1,32 @@ +FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 as scannerbase + +ARG FILE_PATH=/usr/local/bin/scannerbase + +RUN dd if=/dev/urandom of=$FILE_PATH bs=1 count=$((600 * 1024 * 1024)) + +FROM mcr.microsoft.com/oss/go/microsoft/golang:1.21-fips-cbl-mariner2.0 as builder + +COPY ./ /src/ + +RUN tdnf install make -y && \ + tdnf install git -y + +WORKDIR /src + +RUN make tests-build + +FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 as scanner + +ARG USER_ID=6190 + +RUN tdnf update -y && \ + tdnf install ca-certificates-microsoft -y && \ + tdnf install shadow-utils -y + +RUN groupadd -g $USER_ID scanner && \ + useradd -g scanner -u $USER_ID scanner + +COPY --from=scannerbase --chown=scanner:root /usr/local/bin/scannerbase /usr/local/bin/scannerbase +COPY --from=builder --chown=scanner:root /src/bin/tests/tests /src/bin/tests/tests + +ENTRYPOINT ["/src/bin/tests/tests", "scanner"] diff --git a/tests/random/random.go b/tests/random/random.go new file mode 100644 index 0000000..20691c0 --- /dev/null +++ b/tests/random/random.go @@ -0,0 +1,256 @@ +package random + +import ( + "context" + "crypto/rand" + "crypto/tls" + "encoding/binary" + "errors" + "fmt" + "io" + "math/big" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/azure/peerd/internal/math" + "github.com/rs/zerolog" + "golang.org/x/sync/errgroup" +) + +var client = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, // local kind clusters use self-signed certs + }, + }, +} + +func Random(ctx context.Context, secrets string, n int, proxyHost string) error { + l := zerolog.Ctx(ctx) + + if secrets == "" { + return errors.New("secrets required") + } + + if n <= 0 { + return errors.New("node count must be positive") + } + + if proxyHost == "" { + return errors.New("proxy host required") + } + + secretValue := strings.TrimSpace(secrets) + + // Parse the SAS URLs from the secret value + upstreamSasUrls := strings.Fields(secretValue) + p2pSasUrls := getP2pSasUrls(strings.Fields(secretValue), proxyHost) + + var g errgroup.Group + var upstreamPercentiles, p2pPercentiles []float64 + var upstreamErrorRate, p2pErrorRate float64 + + g.Go(func() error { + upstreamPercentiles, upstreamErrorRate = benchmark(l, "upstream", upstreamSasUrls, n) + return nil + }) + + g.Go(func() error { + p2pPercentiles, p2pErrorRate = benchmark(l, "p2p", p2pSasUrls, n) + return nil + }) + + _ = g.Wait() + + // Print the results + if len(upstreamPercentiles) > 0 { + l.Info(). + Float64("upstream.p50", upstreamPercentiles[0]). + Float64("upstream.p75", upstreamPercentiles[1]). + Float64("upstream.p90", upstreamPercentiles[2]). + Float64("upstream.p95", upstreamPercentiles[3]). + Float64("upstream.p100", upstreamPercentiles[4]). + Msg("speeds (MB/s)") + } + + if len(p2pPercentiles) > 0 { + l.Info(). + Float64("p2p.p50", p2pPercentiles[0]). + Float64("p2p.p75", p2pPercentiles[1]). + Float64("p2p.p90", p2pPercentiles[2]). + Float64("p2p.p95", p2pPercentiles[3]). + Float64("p2p.p100", p2pPercentiles[4]). + Msg("speeds (MB/s)") + } + + l.Info(). + Float64("p2p.error_rate", p2pErrorRate).Float64("upstream.error_rate", upstreamErrorRate). + Msg("error rates") + + return nil +} + +// benchmark runs the benchmark and returns the measured download speeds. +func benchmark(l *zerolog.Logger, name string, urls []string, n int) ([]float64, float64) { + log := l.With().Str("mode", name).Logger() + + // Group the SAS URLs randomly into groups of size n + groups := math.RandomizedGroups(urls, n) + + // Download the SAS URLs in each group and measure the download speed + var speeds []float64 + failures := 0 + + var wg sync.WaitGroup + for _, group := range groups { + wg.Add(1) + go func(group []string) { + defer wg.Done() + s, f := downloadSASURLs(&log, group) + speeds = append(speeds, s...) + failures += f + }(group) + } + + wg.Wait() + + // Calculate the percentiles + return math.PercentilesFloat64Reverse(speeds, 0.50, 0.75, 0.9, 0.99, 1), float64(failures) / float64(len(urls)) +} + +// getP2pSasUrls returns the SAS URLs for the p2p network +func getP2pSasUrls(upstreamSasUrls []string, proxyHost string) []string { + p2pSasUrls := make([]string, len(upstreamSasUrls)) + for _, s := range upstreamSasUrls { + p2pSasUrls = append(p2pSasUrls, proxyHost+"/blobs/"+s) + } + return p2pSasUrls +} + +// downloadSASURLs downloads the SAS URLs in a group and measures the download speed +func downloadSASURLs(l *zerolog.Logger, group []string) ([]float64, int) { + readsPerBlob := 5 + var wg sync.WaitGroup + l.Info().Int("groupSize", len(group)).Int("readsPerBlob", readsPerBlob).Strs("urls", group).Msg("downloading blobs") + speeds := []float64{} + failures := 0 + + speedsChan := make(chan []float64, readsPerBlob) + failuresChan := make(chan int, readsPerBlob) + + for _, sasURL := range group { + wg.Add(1) + go func(sasURL string) { + defer wg.Done() + + if sasURL == "" { + l.Warn().Str("url", sasURL).Msg("skipping SAS URL") + return + } + + blobSpeeds, f, err := downloadSASURL(l, sasURL, readsPerBlob) + if err != nil { + l.Error().Err(err).Str("url", sasURL).Msg("download error") + failuresChan <- f + } + + speedsChan <- blobSpeeds + }(sasURL) + } + + doneChan := make(chan bool, 1) + go func() { + wg.Wait() + doneChan <- true + }() + + for { + select { + case blobSpeeds := <-speedsChan: + speeds = append(speeds, blobSpeeds...) + + case failure := <-failuresChan: + failures += failure + + case <-doneChan: + return speeds, failures + } + } +} + +// downloadSASURL downloads a SAS URL and returns the number of bytes downloaded. +func downloadSASURL(l *zerolog.Logger, sasURL string, readsPerBlob int) ([]float64, int, error) { + failures := -1 + req, err := http.NewRequest("HEAD", sasURL, nil) + if err != nil { + return nil, failures, err + } + + resp, err := client.Do(req) + if err != nil { + return nil, failures, err + } else if resp.StatusCode != http.StatusOK { + return nil, failures, fmt.Errorf("unexpected status code %d", resp.StatusCode) + } + blobSize, _ := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) + + l.Info().Int64("size", blobSize).Int("readsPerBlob", readsPerBlob).Msg("downloading blob") + + errs := []error{} + speeds := []float64{} + failures = 0 + + for i := 0; i < readsPerBlob; i++ { + req, err = http.NewRequest("GET", sasURL, nil) + if err != nil { + return speeds, failures, err + } + + // Set a random Range header + s, _ := rand.Int(rand.Reader, big.NewInt(int64(blobSize))) + start := s.Int64() + e, _ := rand.Int(rand.Reader, big.NewInt(int64(blobSize-start+1))) + end := start + e.Int64() + contentRange := fmt.Sprintf("bytes=%d-%d", start, end) + req.Header.Set("Range", contentRange) + + l2 := l.With().Int("size", int(blobSize)).Str("url", sasURL).Str("http.request.range", contentRange).Logger() + + st := time.Now() + resp, err = client.Do(req) + if err != nil { + continue + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + l2.Error().Err(err).Int("status", resp.StatusCode).Msg("unexpected status code") + errs = append(errs, fmt.Errorf("unexpected status code %d", resp.StatusCode)) + failures++ + continue + } + + defer resp.Body.Close() + n, err := io.Copy(io.Discard, resp.Body) + if err != nil { + l2.Error().Err(err).Msg("error reading response body") + errs = append(errs, err) + failures++ + } else { + since := time.Since(st) + speeds = append(speeds, float64(n)/since.Seconds()) + } + + sleep() + } + + return speeds, failures, errors.Join(errs...) +} + +func sleep() { + var n int64 + _ = binary.Read(rand.Reader, binary.LittleEndian, &n) + n = (n%(25-3+1) + 3) + time.Sleep(time.Duration(n) * time.Millisecond) +} diff --git a/tests/scanner/scanner.go b/tests/scanner/scanner.go new file mode 100644 index 0000000..9c74f75 --- /dev/null +++ b/tests/scanner/scanner.go @@ -0,0 +1,65 @@ +package scanner + +import ( + "context" + "crypto/rand" + "encoding/binary" + "io" + "os" + "time" + + "github.com/rs/zerolog" + "github.com/schollz/progressbar/v3" +) + +const ( + path = "/usr/local/bin/scannerbase" // This should match the path in the Dockerfile. +) + +func Scanner(ctx context.Context) error { + l := zerolog.Ctx(ctx) + + sleep(l) + + l.Info().Str("path", path).Msg("starting scanner") + + f, _ := os.OpenFile(path, os.O_RDONLY, 0644) + defer f.Close() + + info, err := f.Stat() + if err != nil { + l.Fatal().Err(err).Msg("failed to stat file") + return err + } + + size := info.Size() + + bar := progressbar.DefaultBytes(size, "reading") + + w, err := io.Copy(io.MultiWriter(io.Discard, bar), f) + if err != nil { + l.Fatal().Err(err).Msg("failed to read file") + return err + } else { + l.Info().Int64("size", size).Int64("read", w).Msg("complete") + } + + return nil +} + +func sleep(l *zerolog.Logger) { + var n uint64 + err := binary.Read(rand.Reader, binary.LittleEndian, &n) + if err != nil { + l.Error().Err(err).Msg("SLEEP FAILED") + return + } + + n = n % 100 + n = n + 1 + + l.Info().Uint64("seconds", n).Msg("sleeping") + + // Sleep for n seconds + time.Sleep(time.Duration(n) * time.Second) +}