diff --git a/.bazelrc b/.bazelrc index 8d9e55f5..b93a9e43 100644 --- a/.bazelrc +++ b/.bazelrc @@ -5,4 +5,4 @@ build --java_toolchain=@bazel_tools//tools/jdk:toolchain_java11 # It will create issues when JVM unlinks the file at shutdown and may trigger a sandbox fault http://b/205838938. # -XX:+PerfDisableSharedMem to force JVM uses regular memory instead. # -XX:-UsePerfData to disable /tmp/hsperfdata references. We don't use the perf data here so we disable it -build --jvmopt="-XX:-UsePerfData" \ No newline at end of file +build --jvmopt="-XX:-UsePerfData" diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml new file mode 120000 index 00000000..8b5fdf87 --- /dev/null +++ b/.markdownlint-cli2.yaml @@ -0,0 +1 @@ +builders/etc/.markdownlint-cli2.yaml \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..a43977a2 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,183 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +exclude: (?x)^( + bazel-(bin|out|testlogs|workspace)/.*| + .bazel_output/.*| + javatests/testData/.*| + version.txt + )$ + +fail_fast: true +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: end-of-file-fixer + - id: fix-byte-order-marker + - id: mixed-line-ending + - id: trailing-whitespace + - id: check-case-conflict + - id: check-merge-conflict + - id: check-yaml + exclude: (?x)^( + tools/load_tests/template.yaml + )$ + - id: check-json + - id: check-symlinks + files: ^$ + - id: check-added-large-files + - id: check-vcs-permalinks + - id: check-executables-have-shebangs + - id: detect-private-key + +- repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 3.0.0 + hooks: + - id: git-check + files: ^$ + - id: script-must-not-have-extension + files: ^$ + exclude: '^google_internal/.*/kokoro_(presubmit|continuous).sh$' + - id: script-must-have-extension + - id: require-ascii + - id: shellcheck + files: ^$ + exclude: '^(production|tools|google_internal|builders/images)/.*$' + +- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks + rev: v2.4.0 + hooks: + - id: pretty-format-java + name: Google Java Formatter + args: [--autofix] + files: ^$ + +- repo: local + hooks: + - id: addlicense + name: addlicense + language: golang + additional_dependencies: + - github.com/google/addlicense@v1.1.0 + always_run: false + pass_filenames: true + types_or: + - text + entry: addlicense -v + exclude: &addlicense-ignores (?x)^( + adtech/.*| + coordinator/.*| + deploy-testenv/.*| + java/com/google/aggregate/adtech/worker/gcp/.*| + java/com/google/aggregate/simulation/.*| + java/com/google/experimental/.*| + javatests/com/google/aggregate/adtech/worker/e2e/.*| + javatests/com/google/aggregate/simulation/.*| + kokoro/.*| + .*gcp.*| + release/.*| + release/aws/.*| + .*/.terraform.lock.hcl| + tools/opensource/.*| + version.adtech| + version.coordinator| + worker/aws/testing/.*| + worker/testing/data/1k/BUILD| + WORKSPACE + )$ + + - id: addlicense-check + name: addlicense check + language: golang + additional_dependencies: + - github.com/google/addlicense@v1.1.0 + always_run: false + pass_filenames: true + types_or: + - text + entry: addlicense -check + exclude: *addlicense-ignores + + - id: inclusive-language + name: inclusive language check + language: script + pass_filenames: true + entry: tools/opensource/inclusive_language/inclusive-language-check + types_or: + - text + exclude: (?x)^( + WORKSPACE| + tools/opensource/inclusive_language/.*| + licenses/.* + )$ + + - id: terraform-fmt + name: terraform fmt + description: Run terraform via docker to format Terraform files + language: script + pass_filenames: false + entry: builders/tools/terraform fmt -write=true -recursive + types_or: + - terraform + files: ^$ + + - id: hadolint + name: Lint Dockerfiles + description: Run hadolint via docker to lint Dockerfiles + language: script + types_or: + - dockerfile + entry: builders/tools/hadolint + files: ^$ + +- repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.7.1 + hooks: + - id: prettier + name: prettier markdown + types_or: + - markdown + +- repo: https://github.com/DavidAnson/markdownlint-cli2 + rev: v0.5.1 + hooks: + - id: markdownlint-cli2 + name: lint markdown + +- repo: local + hooks: + - id: buildifier + name: buildifier + description: Format bazel WORKSPACE, BUILD and .bzl files with a standard convention. + language: golang + additional_dependencies: + - github.com/bazelbuild/buildtools/buildifier@5.1.0 + always_run: true + pass_filenames: true + types_or: + - bazel + entry: buildifier + args: + - -lint=fix + - -mode=fix + - -warnings=all + +- repo: https://github.com/psf/black + rev: 22.8.0 + hooks: + - id: black + name: black python formatter diff --git a/.prettierrc.yaml b/.prettierrc.yaml new file mode 120000 index 00000000..2a9cf23e --- /dev/null +++ b/.prettierrc.yaml @@ -0,0 +1 @@ +builders/etc/.prettierrc.yaml \ No newline at end of file diff --git a/BUILD b/BUILD index 7a9c7bac..e2f3ad5e 100644 --- a/BUILD +++ b/BUILD @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -package(default_visibility = ["//visibility:public"]) - load("@com_github_bazelbuild_buildtools//buildifier:def.bzl", "buildifier") +package(default_visibility = ["//visibility:public"]) + buildifier( name = "buildifier_check", mode = "check", diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..6dd5de20 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,31 @@ +# Changelog + +## [0.5.0](https://github.com/privacysandbox/aggregation-service/compare/v0.4.0...v0.5.0) (2022-12-02) + +### Changes + +* Upgraded control plane shared library dependency to [v0.39.0](https://github.com/privacysandbox/control-plane-shared-libraries/tree/v0.39.0), which requires 6 Elastic IP addresses for the subnets of the created VPC. With the default [AWS VPC EIPs quota](https://docs.aws.amazon.com/vpc/latest/userguide/amazon-vpc-limits.html) set to 5 per region, users may need to request for more quota when deploying the Aggregate Service or decrease the number of subnets created for the aggregation service VPC. To stay within the default quota you can decrease the number of subnets, by setting `vpc_availability_zones = ["a","b","c","d","e"]` in your `.auto.tfvars` + +* Addressed both [security issues](https://github.com/privacysandbox/aggregation-service/blob/v0.4.0/README.md#general-security-notes) by uprading the control plane shared library dependency. + +## [0.4.0](https://github.com/privacysandbox/aggregation-service/compare/v0.3.0...v0.4.0) (2022-10-24) + +### Changes + +* Multi-party coordinator support added and made the default. +* Aggregation Service enforces the [no-duplicate](https://github.com/WICG/attribution-reporting-api/blob/main/AGGREGATION_SERVICE_TEE.md#no-duplicates-rule) rule. We recommend users design their systems keeping the no-duplicate rule in consideration. We suggest reading the [debugging](./DEBUGGING.md) document for debug aggregation runs. + + +## [0.3.0](https://github.com/privacysandbox/aggregation-service/compare/v0.2.0...v0.3.0) (2022-06-30) + +### Changes + +* Support for updated Attribution Reporting API [Aggregatable Reports](https://github.com/WICG/conversion-measurement-api/blob/main/AGGREGATION_SERVICE_TEE.md#aggregatable-reports) format. + +## [0.2.0](https://github.com/privacysandbox/aggregation-service/compare/v0.1.2...v0.2.0) (2022-06-09) + +### Changes + +* The [`no-duplicate`](https://github.com/WICG/attribution-reporting-api/blob/main/AGGREGATION_SERVICE_TEE.md#no-duplicates-rule) rule for privacy budget is not enforced anymore but will be enforced in the future. We recommend you design your systems with the `no-duplicate` rule in consideration. +* Added support for relative paths in LocalTestingTool binary +* Fixed issue with unexpected error messages in LocalTestingTool binary diff --git a/DEBUGGING.md b/DEBUGGING.md deleted file mode 100644 index d5a58b60..00000000 --- a/DEBUGGING.md +++ /dev/null @@ -1,120 +0,0 @@ -# Debug aggregation runs with encrypted payloads - -This document describes the debugging support for the Aggregation Service running in a TEE using encrypted payloads of aggregatable reports. This allows you to debug your production setup and understand how the encrypted payloads of aggregatable reports are processed. Reports with debug_cleartext_payload can be used with the [local testing tool](./README.md#using-the-local-testing-tool) and are helpful for understanding the content of reports and validating that registrations on the browser client or device are configured properly. - -To test the Aggregation Service, you can enable debug aggregation runs which use encrypted payloads of aggregatable reports to generate debug summary reports. When executing a debug run, no noise is added to the debug summary report, and annotations are added to indicate whether keys are present in domain input and/or reports. This allows developers to: - -* Analyze the reports -* Determine if the aggregation was completed correctly, per the adtech’s specifications -* Understand the impact of noise in summary reports -* Determine how to set the proper domain keys - -Additionally, debug runs do not enforce the [No-Duplicates rule](https://github.com/WICG/attribution-reporting-api/blob/main/AGGREGATION_SERVICE_TEE.md#no-duplicates-rule) across batches. The No-Duplicates rule is still enforced within a batch. This allows adtech to try different batches without worrying about making them [disjoint](https://github.com/WICG/attribution-reporting-api/blob/main/AGGREGATION_SERVICE_TEE.md#disjoint-batches) during testing or debugging. - -Once third-party cookies are deprecated, the client (browser or operating system) will no longer generate aggregatable reports that are enabled for debugging. At that time, debug runs with encrypted payloads will no longer be supported for reports from real user devices or browsers. - -In this document, you’ll find code snippets and instructions for how to debug the Aggregation Service and create debug summary reports. - -## Create a debug job - -To create an aggregation debug job, add the `debug_run` parameter to the `job_parameters` object of the `createJob` API request. - -`POST https://.execute-api.us-east-1.amazonaws.com/stage/v1alpha/createJob` - -```json -{ - "input_data_blob_prefix":"input/reports.avro", - "input_data_bucket_name":"", - "output_data_blob_prefix":"output/summary_report.avro", - "output_data_bucket_name":"", - "job_parameters":{ - "attribution_report_to":"", - "output_domain_blob_prefix":"domain/domain.avro", - "output_domain_bucket_name":"", - "debug_run":"true" -}, - "Job_request_id":"test01" -} -``` - -If `debug_run` is not present in` job_parameters` or it's set to `false`, a normal noised aggregation run is created. More details about `createJob` API can be found in [detailed API spec](./API.md#createjob-endpoint). - -## Debuggable aggregatable reports - -A debug run only considers reports that have the flag `"debug_mode": "enabled"` in the report shared_info ([aggregatable report sample](./COLLECTING.md#aggregatable-report-sample)). Reports with the `debug_mode` flag missing or the `debug_mode` value isn’t set to `enabled` aren’t included in the results generated by a debug run. - -The count of reports that were not processed during a debug run is returned in the job response, which can be previewed in the [detailed API spec](./API.md#createjob-endpoint). In the `error_counts` field, the category `NUM_REPORTS_DEBUG_NOT_ENABLED` shows the numbers of reports not processed during a debug run. - - -## Results - -Two summary reports are generated from a debug run: a regular summary report and a debug summary report. The regular summary report format generated from the debug run is consistent with that of a regular aggregation run. The debug summary report has a [different format](#debug-summary-report). The path of the summary report is set in the [createJob](./API.md#createjob-endpoint) API. The debug summary report will be stored in the "debug" folder under the summary report's path with the same object name. - -Considering the following createJob parameters for `output_data_bucket_name` and `output_data_blob_prefix`: - -```json -"output_data_blob_prefix": "output/summary_report.avro", -"output_data_bucket_name": "", -``` - -the following objects are created by a debug run: - -`s3:///output/summary_report.avro` and - -`s3:///output/debug/summary_report.avro`. - -Note that the regular summary report generated during a debug run will only include reports which have the flag `"debug_mode": "enabled"` in the reports `shared_info`. - -### Debug summary report - -The debug summary report includes the following data: - -* `bucket`: The aggregation key -* `unnoised_metric`: The aggregation value without noise -* `noise`: The approximate noise applied to the aggregated results in the regular summary report -* `annotations`: The annotations associated with the bucket - -The keys in the debug summary report will include all the keys from the output domain. - -If the key is only present in the output domain (not in any of the processed aggregatable reports), the key will be included in the debug report with `unnoised_metric=0` and `annotations=["in_domain"]`. - -The keys that are only present in aggregatable reports (not in output domain) will also be included in the debug report with `unnoised_metric=` and `annotations=["in_reports"]`. - -Keys that are present in domain and aggregatable reports will have annotations for both `[“in_domain”, “in_reports”]`. - -The schema of debug summary reports is in the following [Avro](https://avro.apache.org/) format: - -```avro -{ -"type": "record", -"name": "DebugAggregatedFact", -"fields": [ - { - "name": "bucket", - "type": "bytes", - "doc": "Histogram bucket used in aggregation. 128-bit integer encoded as a 16-byte big-endian bytestring. Leading 0-bits will be left out." - }, - { - "name": "unnoised_metric", - "type": "long", - "doc": "Unnoised metric associated with the bucket." - }, - { - "name": "noise", - "type": "long", - "doc": "The noise applied to the metric in the regular result." - } - { - "name":"annotations", - "type": - { - "type": "array", - "items": { - "type":"enum", - "name":"bucket_tags", - "symbols":["in_domain","in_reports"] - } - } - ] -} -``` diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index ae44b762..80bc10ff 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -1,14 +1,17 @@ # Dependencies and Licenses -The deployment of the Amazon Web Services [Nitro Enclaves](https://aws.amazon.com/ec2/nitro/nitro-enclaves/) based Aggregation Service depends on several packaged -artifacts listed below. -These artifacts can be downloaded with the [download_prebuilt_dependencies.sh](./terraform/aws/download_prebuilt_dependencies.sh) -script. See [README](/README.md#download-terraform-scripts-and-prebuilt-dependencies). +The deployment of the Amazon Web Services +[Nitro Enclaves](https://aws.amazon.com/ec2/nitro/nitro-enclaves/) based Aggregation Service depends +on several packaged artifacts listed below. These artifacts can be downloaded with the +[download_prebuilt_dependencies.sh](/terraform/aws/download_prebuilt_dependencies.sh) script. More +information can be found in the +[README](/README.md#download-terraform-scripts-and-prebuilt-dependencies). ## Packaged AWS Lambda Jars -### AwsChangeHandlerLambda_{version}.jar +### AwsChangeHandlerLambda\_{version}.jar + | groupId | artifactId | Version | License | URL | |--|--|--|--|--| | aopalliance | aopalliance | 1.0 |Public Domain | N/A | @@ -82,9 +85,11 @@ script. See [README](/README.md#download-terraform-scripts-and-prebuilt-dependen | software.amazon.awssdk|url-connection-client|2.16.104|Apache License, Version 2.0 | | | software.amazon.awssdk|utils|2.16.104|Apache License, Version 2.0 | | | software.amazon.eventstream|eventstream|1.0.1|Apache License, Version 2.0 | | + -### aws_apigateway_frontend_{version}.jar +### aws_apigateway_frontend\_{version}.jar + | groupId | artifactId | Version | License | URL | |--|--|--|--|--| | aopalliance | aopalliance | 1.0 |Public Domain | N/A | @@ -158,9 +163,11 @@ script. See [README](/README.md#download-terraform-scripts-and-prebuilt-dependen | software.amazon.awssdk|url-connection-client|2.16.104|Apache License, Version 2.0 | | | software.amazon.awssdk|utils|2.16.104|Apache License, Version 2.0 | | | software.amazon.eventstream|eventstream|1.0.1|Apache License, Version 2.0 | | + -### AwsFrontendCleanupLambda_{version}.jar +### AwsFrontendCleanupLambda\_{version}.jar + | groupId | artifactId | Version | License | URL | |--|--|--|--|--| | aopalliance | aopalliance | 1.0 |Public Domain | N/A | @@ -234,9 +241,11 @@ script. See [README](/README.md#download-terraform-scripts-and-prebuilt-dependen | software.amazon.awssdk|url-connection-client|2.16.104|Apache License, Version 2.0 | | | software.amazon.awssdk|utils|2.16.104|Apache License, Version 2.0 | | | software.amazon.eventstream|eventstream|1.0.1|Apache License, Version 2.0 | | + -### AsgCapacityHandlerLambda_{version}.jar +### AsgCapacityHandlerLambda\_{version}.jar + | groupId | artifactId | Version | License | URL | |--|--|--|--|--| | aopalliance | aopalliance | 1 | Public Domain | @@ -298,9 +307,11 @@ script. See [README](/README.md#download-terraform-scripts-and-prebuilt-dependen | software.amazon.awssdk | url-connection-client | 2.16.104 | Apache License, Version 2.0 | | | software.amazon.awssdk | utils | 2.16.104 | Apache License, Version 2.0 | | | software.amazon.eventstream | eventstream | 1.0.1 | Apache License, Version 2.0 | | + -### TerminatedInstanceHandlerLambda_{version}.jar +### TerminatedInstanceHandlerLambda\_{version}.jar + | groupId | artifactId | Version | License | URL | |--|--|--|--|--| | aopalliance | aopalliance | 1 | Public Domain | @@ -362,6 +373,7 @@ script. See [README](/README.md#download-terraform-scripts-and-prebuilt-dependen | software.amazon.awssdk | url-connection-client | 2.16.104 | Apache License, Version 2.0 | | | software.amazon.awssdk | utils | 2.16.104 | Apache License, Version 2.0 | | | software.amazon.eventstream | eventstream | 1.0.1 | Apache License, Version 2.0 | | + ## License of artifacts in this repository diff --git a/README.md b/README.md index 9226bd24..6d4a0c0c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,25 @@ # Set up Aggregation Service for Aggregatable Reports -**[NOTE] The latest aggregatable reports generated with Chrome version 104+ are only supported with version `0.3.0` and later. Please follow the [update instructions](#updating-the-system) for your environment.** - -This repository contains instructions and scripts to set up and test -the Aggregation Service for [Aggregatable Reports](https://github.com/WICG/conversion-measurement-api/blob/main/AGGREGATION_SERVICE_TEE.md#aggregatable-reports) -locally and on Amazon Web Services [Nitro Enclaves](https://aws.amazon.com/ec2/nitro/nitro-enclaves/). -If you want to learn more about the [Privacy Sandbox](https://privacysandbox.com/) -Aggregation Service for the Attribution Reporting API, aggregatable, and summary -reports click, read the [Aggregation Service proposal](https://github.com/WICG/conversion-measurement-api/blob/main/AGGREGATION_SERVICE_TEE.md#aggregatable-reports). +**[NOTE] The latest aggregatable reports generated with Chrome version 104+ are only supported with +version `0.3.0` and later. Please follow the [update instructions](#updating-the-system) for your +environment.** + +This repository contains instructions and scripts to set up and test the Aggregation Service for +[Aggregatable Reports](https://github.com/WICG/conversion-measurement-api/blob/main/AGGREGATION_SERVICE_TEE.md#aggregatable-reports) +locally and on Amazon Web Services +[Nitro Enclaves](https://aws.amazon.com/ec2/nitro/nitro-enclaves/). If you want to learn more about +the [Privacy Sandbox](https://privacysandbox.com/) Aggregation Service for the Attribution Reporting +API, aggregatable, and summary reports click, read the +[Aggregation Service proposal](https://github.com/WICG/conversion-measurement-api/blob/main/AGGREGATION_SERVICE_TEE.md#aggregatable-reports). ## Set up local testing -You can process [aggregatable debug reports](https://github.com/WICG/conversion-measurement-api/blob/main/AGGREGATE.md#aggregatable-reports) -locally with the `LocalTestingTool_{VERSION}.jar` into summary reports. -Learn [how to setup debug reports](https://docs.google.com/document/d/1BXchEk-UMgcr2fpjfXrQ3D8VhTR-COGYS1cwK_nyLfg/edit#heading=h.fvp017tkgw79). +You can process +[aggregatable debug reports](https://github.com/WICG/conversion-measurement-api/blob/main/AGGREGATE.md#aggregatable-reports) +locally with the `LocalTestingTool_{VERSION}.jar` into summary reports. Learn +[how to setup debug reports](https://docs.google.com/document/d/1BXchEk-UMgcr2fpjfXrQ3D8VhTR-COGYS1cwK_nyLfg/edit#heading=h.fvp017tkgw79). -*Disclaimer: encrypted reports can **not** be processed with the local testing tool!* +_Disclaimer: encrypted reports can **not** be processed with the local testing tool!_ ### Clone the repository @@ -36,14 +40,17 @@ VERSION=$(cat VERSION); curl -f -o LocalTestingTool_$VERSION.jar https://aggrega You'll need [Java JRE](https://adoptium.net/) installed to use the tool. -*The `SHA256` of the `LocalTestingTool_{version}.jar` can be found on the [releases page](https://github.com/privacysandbox/aggregation-service/releases).* +_The `SHA256` of the `LocalTestingTool_{version}.jar` can be found on the +[releases page](https://github.com/privacysandbox/aggregation-service/releases).\_ -Follow the instructions on how to [collect and batch aggregatable reports](#collect-and-batch-aggregatable-reports). -Create an output domain file: `output_domain.avro`. For testing you can use our [sample debug batch](./sampledata/output_debug_reports.avro) -with the corresponding [output domain avro](./sampledata/output_domain.avro). +Follow the instructions on how to +[collect and batch aggregatable reports](#collect-and-batch-aggregatable-reports). Create an output +domain file: `output_domain.avro`. For testing you can use our +[sample debug batch](./sampledata/output_debug_reports.avro) with the corresponding +[output domain avro](./sampledata/output_domain.avro). -To aggregate the resulting avro batch `output_debug_reports.avro` file into a summary report -in the same directory where you run the tool, run the following command: +To aggregate the resulting avro batch `output_debug_reports.avro` file into a summary report in the +same directory where you run the tool, run the following command: ```sh java -jar LocalTestingTool_{version}.jar \ @@ -53,9 +60,9 @@ java -jar LocalTestingTool_{version}.jar \ ``` To see all supported flags for the local testing tool run -`java -jar LocalTestingTool_{version}.jar --help`, e.g. you can adjust the noising -epsilon with the `--epsilon` flag or disable noising all together with the -`--no_noising` flag. [See all flags and descriptions](./API.md#local-testing-tool). +`java -jar LocalTestingTool_{version}.jar --help`, e.g. you can adjust the noising epsilon with the +`--epsilon` flag or disable noising all together with the `--no_noising` flag. +[See all flags and descriptions](/docs/API.md#local-testing-tool). ## Test on AWS with support for encrypted reports @@ -63,83 +70,100 @@ epsilon with the `--epsilon` flag or disable noising all together with the #### Privacy Budget Enforcement -Aggregation Service enforces the [no-duplicate](https://github.com/WICG/attribution-reporting-api/blob/main/AGGREGATION_SERVICE_TEE.md#no-duplicates-rule) rule. We recommend users design their systems keeping the no-duplicate rule in consideration. We suggest reading the [debugging](./DEBUGGING.md) document for debug aggregation runs. +Aggregation Service enforces the +[no-duplicate](https://github.com/WICG/attribution-reporting-api/blob/main/AGGREGATION_SERVICE_TEE.md#no-duplicates-rule) +rule. We recommend users design their systems keeping the no-duplicate rule in consideration. We +suggest reading the [debugging](/docs/DEBUGGING.md) document for debug aggregation runs. ### Prerequisites To test the aggregation service with support for encrypted reports, you need the following: -* Have an [AWS account](https://portal.aws.amazon.com/gp/aws/developer/registration/index.html) available to you. -* [Register](https://developer.chrome.com/origintrials/#/view_trial/771241436187197441) -for the Privacy Sandbox Relevance and Measurement origin trial (OT) -* Complete the aggregation service [onboarding form](https://forms.gle/EHoecersGKhpcLPNA) +- Have an [AWS account](https://portal.aws.amazon.com/gp/aws/developer/registration/index.html) + available to you. +- [Register](https://developer.chrome.com/origintrials/#/view_trial/771241436187197441) for the + Privacy Sandbox Relevance and Measurement origin trial (OT) +- Complete the aggregation service [onboarding form](https://forms.gle/EHoecersGKhpcLPNA) -Once you’ve submitted the onboarding form, we will contact you to verify your information. Then, we’ll send you the remaining instructions and information needed for this setup.
-*You won't be able to successfully setup your AWS system without registering for the origin trial and completing the onboarding process!* +Once you've submitted the onboarding form, we will contact you to verify your information. Then, +we'll send you the remaining instructions and information needed for this setup.
_You won't be +able to successfully setup your AWS system without registering for the origin trial and completing +the onboarding process!_ To set up aggregation service in AWS you'll use [Terraform](https://www.terraform.io/). ### Set up AWS client -Make sure you [install](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) -and [set up](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) -the latest AWS client. +Make sure you +[install](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) and +[set up](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) the latest +AWS client. ### Set up Terraform -Change into the `/terraform/aws` folder. See [clone the repository](#clone-the-repository) if you have not cloned the repository so far. +Change into the `/terraform/aws` folder. See +[clone the repository](#clone-the-repository) if you have not cloned the repository so far. -The setup scripts require terraform version `1.2.3`. -You can download Terraform version 1.2.3 from [https://releases.hashicorp.com/terraform/1.2.3/](https://releases.hashicorp.com/terraform/1.2.3/) or -*at your own risk*, you can install and use +The setup scripts require terraform version `1.2.3`. You can download Terraform version 1.2.3 from +[https://releases.hashicorp.com/terraform/1.2.3/](https://releases.hashicorp.com/terraform/1.2.3/) +or _at your own risk_, you can install and use [Terraform version manager](https://github.com/tfutils/tfenv) instead. -If you have the Terraform version manager `tfenv` installed, run the following -in your `` to set Terraform to version `1.2.3`. +If you have the Terraform version manager `tfenv` installed, run the following in your +`` to set Terraform to version `1.2.3`. ```sh tfenv install 1.2.3; tfenv use 1.2.3 ``` -We recommend you store the [Terraform state](https://www.terraform.io/language/state) -in a cloud bucket. -Create a S3 bucket via the console/cli, which we'll reference as -`tf_state_bucket_name`. Consider enabling `versioning` to preserve, retrieve, and restore previous versions and set appropriate policies for this bucket to prevent accidental changes and deletion. +We recommend you store the [Terraform state](https://www.terraform.io/language/state) in a cloud +bucket. Create a S3 bucket via the console/cli, which we'll reference as `tf_state_bucket_name`. +Consider enabling `versioning` to preserve, retrieve, and restore previous versions and set +appropriate policies for this bucket to prevent accidental changes and deletion. ### Download Terraform scripts and prebuilt dependencies -*Note: The prebuilt Amazon Machine Image (AMI) for the aggregation service is only available in the `us-east-1` region. If you like to deploy the aggregation service in a different region you need to copy the released AMI to your account or build it using our provided scripts.* +_Note: The prebuilt Amazon Machine Image (AMI) for the aggregation service is only available in the +`us-east-1` region. If you like to deploy the aggregation service in a different region you need to +copy the released AMI to your account or build it using our provided scripts._ -If you like to build the Amazon Machine Image including the enclave container, as well as the Lambda jars in your account, please -follow the instructions in [build-scripts/aws](build-scripts/aws/README.md). This will skip running `bash download_prebuilt_dependencies.sh` and run `bash fetch_terraform.sh` instead. Continue with the [next deployment step](#set-up-your-deployment-environment) after -building and downloading your self-build jars. +If you like to build the Amazon Machine Image including the enclave container, as well as the Lambda +jars in your account, please follow the instructions in +[build-scripts/aws](/build-scripts/aws/README.md). This will skip running +`bash download_prebuilt_dependencies.sh` and run `bash fetch_terraform.sh` instead. Continue with +the [next deployment step](#set-up-your-deployment-environment) after building and downloading your +self-build jars. -The Terraform scripts to deploy the aggregation service depend on 5 packaged jars for Lambda functions deployment. -These jars are hosted on Amazon S3 (https://aggregation-service-published-artifacts.s3.amazonaws.com/aggregation-service/{version}/{jar_file}) +The Terraform scripts to deploy the aggregation service depend on 5 packaged jars for Lambda +functions deployment. These jars are hosted on Amazon S3 +(}) and can be downloaded with the `/terraform/aws/download_prebuilt_dependencies.sh` -script. The script downloads the terrafrom scripts and jars which will be stored in `/terraform/aws`. -License information of downloaded dependencies can be found in the [DEPENDENCIES.md](./DEPENDENCIES.md) +script. The script downloads the terrafrom scripts and jars which will be stored in +`/terraform/aws`. License information of downloaded dependencies can be found in +the [DEPENDENCIES.md](/DEPENDENCIES.md) -Run the following script in the `/terraform/aws` folder to download the prebuilt dependencies. +Run the following script in the `/terraform/aws` folder to download the prebuilt +dependencies. ```bash bash download_prebuilt_dependencies.sh ``` -* Note: The above script needs to be run with `bash` and does not support `sh`* +- Note: The above script needs to be run with `bash` and does not support `sh`\* -For manual download into the `/terraform/aws/jars` folder you -can download them from the links on our [releases page](https://github.com/privacysandbox/aggregation-service/releases). +For manual download into the `/terraform/aws/jars` folder you can download them +from the links on our +[releases page](https://github.com/privacysandbox/aggregation-service/releases). ### Set up your deployment environment -We use the following folder structure `/terraform/aws/environments/` to separate -deployment environments. +We use the following folder structure +`/terraform/aws/environments/` to separate deployment +environments. -To set up your first environment (e.g `dev`), copy the `demo` environment. Run -the following commands from the `/terraform/aws/environments` -folder: +To set up your first environment (e.g `dev`), copy the `demo` environment. Run the following +commands from the `/terraform/aws/environments` folder: ```sh mkdir dev @@ -147,11 +171,10 @@ cp -R demo/* dev cd dev ``` -Make the following adjustments in the `/terraform/aws/environments/dev` -folder: +Make the following adjustments in the `/terraform/aws/environments/dev` folder: -1. Add the `tf_state_bucket_name` to your `main.tf` by uncommenting and replacing -the values using `<...>`: +1. Add the `tf_state_bucket_name` to your `main.tf` by uncommenting and replacing the values using + `<...>`: ```sh # backend "s3" { @@ -161,10 +184,10 @@ the values using `<...>`: # } ``` -1. Rename `example.auto.tfvars` to `.auto.tfvars` and -add the `...assume_role...` values using the information you received in the -onboarding email. Delete the line that reads `assume_role_parameter = "arn:aws:iam::example:role/example"` -Leave all other values as-is for the initial deployment. +1. Rename `example.auto.tfvars` to `.auto.tfvars` and add the `...assume_role...` + values using the information you received in the onboarding email. Delete the line that reads + `assume_role_parameter = "arn:aws:iam::example:role/example"` Leave all other values as-is for + the initial deployment. ```sh environment = "" @@ -177,19 +200,23 @@ Leave all other values as-is for the initial deployment. alarm_notification_email = "" ``` - * environment: name of your environment - * coordinator_a_assume_role_parameter: IAM role for Coordinator A given by us in the onboarding or upgrade email - * coordinator_b_assume_role_parameter: IAM role for Coordinator B given by us in the onboarding or upgrade email - * alarm_notification_email: Email to receive alarm notifications. Requires - confirmation subscription through sign up email sent to this address. + - environment: name of your environment + - coordinator_a_assume_role_parameter: IAM role for Coordinator A given by us in the + onboarding or upgrade email + - coordinator_b_assume_role_parameter: IAM role for Coordinator B given by us in the + onboarding or upgrade email + - alarm_notification_email: Email to receive alarm notifications. Requires confirmation + subscription through sign up email sent to this address. + +1. **Skip this step if you use our prebuilt AMI and Lambda jars** -1. **Skip this step if you use our prebuilt AMI and Lambda jars** - - If you [self-build your AMI and jars](build-scripts/aws/README.md), you need to copy the contents of the `release_params.auto.tfvars` file into a new file `self_build_params.auto.tfvars` remove the `release_params.auto.tfvars` file afterwards. + If you [self-build your AMI and jars](/build-scripts/aws/README.md), you need to copy the + contents of the `release_params.auto.tfvars` file into a new file + `self_build_params.auto.tfvars` remove the `release_params.auto.tfvars` file afterwards. To copy without symlink, run the following in the -`/terraform/aws/environments/dev` folder - + `/terraform/aws/environments/dev` folder + ```sh cp -L release_params.auto.tfvars self_build_params.auto.tfvars ``` @@ -200,10 +227,11 @@ Leave all other values as-is for the initial deployment. rm release_params.auto.tfvars ``` - And change the line `ami_owners = ["971056657085"]` to `ami_owners = ["self"]` in your `self_build_params.auto.tfvars`. + And change the line `ami_owners = ["971056657085"]` to `ami_owners = ["self"]` in your + `self_build_params.auto.tfvars`. -1. Once you’ve adjusted the configuration, run the following in the -`/terraform/aws/environments/dev` folder +1. Once you've adjusted the configuration, run the following in the + `/terraform/aws/environments/dev` folder Install all Terraform modules: @@ -224,8 +252,7 @@ Leave all other values as-is for the initial deployment. Plan: 141 to add, 0 to change, 0 to destroy. ``` - you can continue to apply the changes (needs confirmation after the - planning step) + you can continue to apply the changes (needs confirmation after the planning step) ```sh terraform apply @@ -243,37 +270,37 @@ Leave all other values as-is for the initial deployment. ``` The terraform scripts create `createJob` and `getJob` API endpoints: - * Create Job Endpoint: `https://.execute-api..amazonaws.com/stage/v1alpha/createJob` - * Get Job Endpoint: `https://.execute-api..amazonaws.com/stage/v1alpha/getJob` - These are authenticated endpoints, refer to the - [Testing the System](#testing-the-system) section to learn how - to use them. + - Create Job Endpoint: + `https://.execute-api..amazonaws.com/stage/v1alpha/createJob` + - Get Job Endpoint: + `https://.execute-api..amazonaws.com/stage/v1alpha/getJob` - *If you run into any issues during deployment of your system, please - consult the [Troubleshooting](#troubleshooting) and [Support](#support) - sections.* + These are authenticated endpoints, refer to the [Testing the System](#testing-the-system) + section to learn how to use them. + + _If you run into any issues during deployment of your system, please consult the + [Troubleshooting](#troubleshooting) and [Support](#support) sections._ ### Testing the system -To test the system, you'll need encrypted aggregatable reports in avro batch -format (follow the [collecting and batching instructions](#collect-and-batch-aggregatable-reports)) -accessible by the aggregation service. +To test the system, you'll need encrypted aggregatable reports in avro batch format (follow the +[collecting and batching instructions](#collect-and-batch-aggregatable-reports)) accessible by the +aggregation service. -If your inputs are larger than a few hundred MB, we suggest sharding the input reports and domain file into smaller shards. +If your inputs are larger than a few hundred MB, we suggest sharding the input reports and domain +file into smaller shards. -1. Create an S3 bucket for your input and output data, we will refer to it as -`data_bucket`. This bucket must be created in the same AWS account where -you set up the aggregation service. - * Consider enabling `versioning` to preserve, retrieve, and restore previous versions and set appropriate policies for this bucket to prevent accidental changes and deletion. +1. Create an S3 bucket for your input and output data, we will refer to it as `data_bucket`. This + bucket must be created in the same AWS account where you set up the aggregation service. \* + Consider enabling `versioning` to preserve, retrieve, and restore previous versions and set + appropriate policies for this bucket to prevent accidental changes and deletion. -1. Copy your reports.avro with batched encrypted aggregatable reports to -`/input`. +1. Copy your reports.avro with batched encrypted aggregatable reports to `/input`. 1. Create an aggregation job with the `createJob` API. - `POST` - `https://.execute-api.us-east-1.amazonaws.com/stage/v1alpha/createJob` + `POST` `https://.execute-api.us-east-1.amazonaws.com/stage/v1alpha/createJob` ```json { @@ -290,20 +317,28 @@ you set up the aggregation service. } ``` - Note: This API requires authentication. Follow the [AWS instructions](https://aws.amazon.com/premiumsupport/knowledge-center/iam-authentication-api-gateway/) + Note: This API requires authentication. Follow the + [AWS instructions](https://aws.amazon.com/premiumsupport/knowledge-center/iam-authentication-api-gateway/) for sending an authenticated request. 1. Check the status of your job with the `getJob` API, replace values in `<...>` - `GET` `https://.execute-api..amazonaws.com/stage/v1alpha/getJob?job_request_id=test01` - Note: This API requires authentication. Follow the [AWS instructions](https://aws.amazon.com/premiumsupport/knowledge-center/iam-authentication-api-gateway/) - for sending an authenticated request. [Detailed API spec](API.md#getjob-endpoint) + `GET` + `https://.execute-api..amazonaws.com/stage/v1alpha/getJob?job_request_id=test01` + Note: This API requires authentication. Follow the + [AWS instructions](https://aws.amazon.com/premiumsupport/knowledge-center/iam-authentication-api-gateway/) + for sending an authenticated request. [Detailed API spec](/docs/API.md#getjob-endpoint) ### Updating the system -If you have deployed the system before, we recommend to run `terraform destroy` in your environment folder (e.g. `/terraform/aws/environments/dev`) when upgrading from `0.3.z` to `0.4.z+` and follow the [setup steps](#set-up-your-deployment-environment) again. +If you have deployed the system before, we recommend to run `terraform destroy` in your environment +folder (e.g. `/terraform/aws/environments/dev`) when upgrading from `0.3.z` to +`0.4.z+` and follow the [setup steps](#set-up-your-deployment-environment) again. -After your upgrade to `0.4.z+` and if you have followed the above setup, next time you can update your system to the latest version by checking out the latest tagged version and running `terraform apply` in your environment folder (e.g. `/terraform/aws/environments/dev`). +After your upgrade to `0.4.z+` and if you have followed the above setup, next time you can update +your system to the latest version by checking out the latest tagged version and running +`terraform apply` in your environment folder (e.g. +`/terraform/aws/environments/dev`). Run the following in the ``. @@ -315,9 +350,8 @@ terraform apply ## Collect and batch aggregatable reports -Both the local testing tool and the aggregation service running on AWS Nitro -Enclave expect aggregatable reports batched in the following -[Avro](https://avro.apache.org/) format. +Both the local testing tool and the aggregation service running on AWS Nitro Enclave expect +aggregatable reports batched in the following [Avro](https://avro.apache.org/) format. ```avro { @@ -340,9 +374,8 @@ Enclave expect aggregatable reports batched in the following } ``` -Additionally an output domain file is needed to declare all expected aggregation -keys for aggregating the aggregatable reports (keys not listed in the domain -file won't be aggregated) +Additionally an output domain file is needed to declare all expected aggregation keys for +aggregating the aggregatable reports (keys not listed in the domain file won't be aggregated) ```avro { @@ -361,43 +394,49 @@ file won't be aggregated) } ``` -[Review code snippets](./COLLECTING.md) which demonstrate how to collect and -batch aggregatable reports. +[Review code snippets](/docs/COLLECTING.md) which demonstrate how to collect and batch aggregatable +reports. ## Generate debug summary reports -Please refer to [Debug aggregation runs](./DEBUGGING.md) for more details about -debugging support in aggregation service. +Please refer to [Debug aggregation runs](/docs/DEBUGGING.md) for more details about debugging +support in aggregation service. ## Troubleshooting -The following error message points to a potential lack of instance availability. -If you encounter this situation, run `terraform destroy` to remove your -deployment and run `terraform apply` again. +- The following error message points to a potential lack of instance availability. If you + encounter this situation, run `terraform destroy` to remove your deployment and run + `terraform apply` again. -```txt -Error: Error creating Auto Scaling Group: ValidationError: You must use a valid -fully-formed launch template. Your requested instance type (m5.2xlarge) is not -supported in your requested Availability Zone (us-east-1e). -Please retry your request by not specifying an Availability Zone or choosing -us-east-1a, us-east-1b, us-east-1c, us-east-1d, us-east-1f. -``` + ```txt + Error: Error creating Auto Scaling Group: ValidationError: You must use a valid + fully-formed launch template. Your requested instance type (m5.2xlarge) is not + supported in your requested Availability Zone (us-east-1e). + Please retry your request by not specifying an Availability Zone or choosing + us-east-1a, us-east-1b, us-east-1c, us-east-1d, us-east-1f. + ``` -## Support +- The following error message points to a potential lack of sufficient elastic VPC IPs quota in + your deployment region. Request a + [quota increase](https://docs.aws.amazon.com/vpc/latest/userguide/amazon-vpc-limits.html) or + decrease the number of subnets created for the aggregation service VPC to resolve the issue. To + stay within the default quota you can decrease the number of subnets, by setting + `vpc_availability_zones = ["a","b","c","d","e"]` in your `.auto.tfvars`. -You can reach out to us for support through creating issues on this repository -or sending us an email at aggregation-service-support\google.com. -This address is monitored and only visible to selected support staff. + ```txt + Error: Error creating EIP: AddressLimitExceeded: The maximum number of addresses has been reached. + status code: 400, request id: 2c7a924c-c807-4714-8d77-8558a463c68b -## General security notes + with module.operator_service.module.vpc[0].aws_eip.elastic_ip["us-east-1a"], + on ../../modules/vpc/main.tf line 277, in resource "aws_eip" "elastic_ip": + 277: resource "aws_eip" "elastic_ip" { + ``` + +## Support -* The [VPC subnet property](./terraform/aws/services/worker/network.tf#L51) -`map_public_ip_on_launch` is currently set to `true` which assigns a public -IP address to all instances in the subnet. This allows for easier console -access, yet is considered a risk and will be addressed in a future release. -* The worker [VPC security group](./terraform/aws/services/worker/network.tf#L99) -currently allows for inbound connections on port 22 from any source IP. -This is considered a risk and will be addressed in a future release. +You can reach out to us for support through creating issues on this repository or sending us an +email at aggregation-service-support\google.com. This address is monitored and only visible to +selected support staff. ## License @@ -407,4 +446,8 @@ Apache 2.0 - See [LICENSE](LICENSE) for more information. ### Where should I post feedback/questions, this repo or the Attribution API repo? -This repo hosts an implementation of the [Attribution Reporting API](https://github.com/WICG/attribution-reporting-api). For feedback/questions encountered during using this particular aggregation service implementation, please use the support channels provided by this repo. For feedback/requests related to the APIs in general, please initiate discussions in the Attribution Reporting API repo. +This repo hosts an implementation of the +[Attribution Reporting API](https://github.com/WICG/attribution-reporting-api). For +feedback/questions encountered during using this particular aggregation service implementation, +please use the support channels provided by this repo. For feedback/requests related to the APIs in +general, please initiate discussions in the Attribution Reporting API repo. diff --git a/SECURITY.md b/SECURITY.md index 4648e5e3..082f7076 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,4 +1,3 @@ -To report a security issue, please use [https://g.co/vulnz](https://g.co/vulnz). -We use g.co/vulnz for our intake, and do coordination and disclosure here on -GitHub (including using GitHub Security Advisory). The Google Security Team will -respond within 5 working days of your report on g.co/vulnz. +To report a security issue, please use [https://g.co/vulnz](https://g.co/vulnz). We use g.co/vulnz +for our intake, and do coordination and disclosure here on GitHub (including using GitHub Security +Advisory). The Google Security Team will respond within 5 working days of your report on g.co/vulnz. diff --git a/VERSION b/VERSION index 1d0ba9ea..8f0916f7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.0 +0.5.0 diff --git a/WORKSPACE b/WORKSPACE index 99cc86f7..a04caa06 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -16,7 +16,10 @@ http_archive( url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, ) -COORDINATOR_VERSION = "v0.30.0" # latest as of 2022-10-04 +# Declare explicit protobuf version, to override any implicit dependencies. +PROTOBUF_CORE_VERSION = "3.19.4" + +COORDINATOR_VERSION = "v0.39.0" # latest as of 2022-11-18 JACKSON_VERSION = "2.12.2" @@ -40,6 +43,15 @@ rules_jvm_external_setup() load("@rules_jvm_external//:defs.bzl", "maven_install") +http_archive( + name = "com_google_protobuf", + sha256 = "3bd7828aa5af4b13b99c191e8b1e884ebfa9ad371b0ce264605d347f135d2568", + strip_prefix = "protobuf-%s" % PROTOBUF_CORE_VERSION, + urls = [ + "https://github.com/protocolbuffers/protobuf/archive/v%s.tar.gz" % PROTOBUF_CORE_VERSION, + ], +) + # Use following instead of git_repository for local development #local_repository( # name = "com_google_adm_cloud_scp", @@ -48,17 +60,11 @@ load("@rules_jvm_external//:defs.bzl", "maven_install") git_repository( name = "com_google_adm_cloud_scp", - patch_args = [ - # Needed to import Git-based patches. - "-p1", - ], - patches = ["//build_defs/scp:scp-0.30.0.patch"], remote = "https://github.com/privacysandbox/control-plane-shared-libraries", tag = COORDINATOR_VERSION, ) -load("@com_google_adm_cloud_scp//build_defs/tink:tink_defs.bzl", "import_tink_git") -load("@com_google_adm_cloud_scp//build_defs/tink:tink_defs.bzl", "TINK_MAVEN_ARTIFACTS") +load("@com_google_adm_cloud_scp//build_defs/tink:tink_defs.bzl", "TINK_MAVEN_ARTIFACTS", "import_tink_git") import_tink_git(repo_name = "@com_google_adm_cloud_scp") @@ -95,6 +101,10 @@ maven_install( "com.google.api:gax:" + GOOGLE_GAX_VERSION, "com.google.http-client:google-http-client-jackson2:1.40.0", #"com.google.crypto.tink:tink:" + TINK_VERSION, # Using Tink from github master branch until new version releases + "com.google.cloud:google-cloud-monitoring:3.4.1", + "com.google.api.grpc:proto-google-cloud-monitoring-v3:3.4.1", + "com.google.protobuf:protobuf-java:" + PROTOBUF_CORE_VERSION, + "com.google.protobuf:protobuf-java-util:" + PROTOBUF_CORE_VERSION, "com.google.guava:guava:30.1-jre", "com.google.guava:guava-testlib:30.1-jre", "com.google.inject:guice:4.2.3", @@ -183,6 +193,14 @@ load("@com_google_differential_privacy//:differential_privacy_deps.bzl", "differ differential_privacy_deps() +############### +# Proto rules # +############### + +load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") + +protobuf_deps() + ############# # PKG Rules # ############# @@ -226,8 +244,8 @@ http_archive( ], ) -load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") +load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") go_rules_dependencies() diff --git a/build-scripts/aws/README.md b/build-scripts/aws/README.md index 784b0eff..460a85d7 100644 --- a/build-scripts/aws/README.md +++ b/build-scripts/aws/README.md @@ -4,45 +4,45 @@ ### Set up AWS client -Make sure you [install](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) -and [set up](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) -the latest AWS client. +Make sure you +[install](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) and +[set up](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) the latest +AWS client. ### Set up Terraform Change into the `/build-scripts/aws/terraform` folder. -The setup scripts require terraform version `1.2.3`. -You can download Terraform version `1.2.3` from [https://releases.hashicorp.com/terraform/1.2.3/](https://releases.hashicorp.com/terraform/1.2.3/) or -*at your own risk*, you can install and use +The setup scripts require terraform version `1.2.3`. You can download Terraform version `1.2.3` from +[https://releases.hashicorp.com/terraform/1.2.3/](https://releases.hashicorp.com/terraform/1.2.3/) +or _at your own risk_, you can install and use [Terraform version manager](https://github.com/tfutils/tfenv) instead. -If you have the Terraform version manager `tfenv` installed, run the following -in your `/build-scripts/aws/terraform` to set Terraform to version `1.2.3`. +If you have the Terraform version manager `tfenv` installed, run the following in your +`/build-scripts/aws/terraform` to set Terraform to version `1.2.3`. ```sh tfenv install 1.2.3; tfenv use 1.2.3 ``` -We recommend you store the [Terraform state](https://www.terraform.io/language/state) -in a cloud bucket. -Create a S3 bucket via the console/cli, which we'll reference as -`tf_state_bucket_name`. Please consider enabling versioning for this bucket. +We recommend you store the [Terraform state](https://www.terraform.io/language/state) in a cloud +bucket. Create a S3 bucket via the console/cli, which we'll reference as `tf_state_bucket_name`. +Please consider enabling versioning for this bucket. ## Configure CodeBuild Setup -Copy `main.tf_sample` and `codebuild.auto.tfvars_sample` and adjust values. -Run in `/build-scripts/aws/terraform` +Copy `main.tf_sample` and `codebuild.auto.tfvars_sample` and adjust values. Run in +`/build-scripts/aws/terraform` ```sh cp main.tf_sample main.tf cp codebuild.auto.tfvars_sample codebuild.auto.tfvars ``` -Open `main.tf` to configure your terraform state backend storage. -Open `codebuild.auto.tfvars` to set build region, artifact_output location and github access credentials. -All available variables to configure can be found in codebuild_variables.tf +Open `main.tf` to configure your terraform state backend storage. Open `codebuild.auto.tfvars` to +set build region, artifact_output location and github access credentials. All available variables to +configure can be found in codebuild_variables.tf Run `terrform init` to setup terraform. @@ -63,12 +63,14 @@ To trigger the build run: aws codebuild start-build --project-name bazel-build-container --region ``` -The build can take several minutes. You can check the status at `https://.console.aws.amazon.com/codesuite/codebuild/projects`. +The build can take several minutes. You can check the status at +`https://.console.aws.amazon.com/codesuite/codebuild/projects`. ## Building artifacts -To build the aggregation service artifacts the above build container is required. Make sure the build for -the build container ran successful before starting the build for the aggregation service artifacts. +To build the aggregation service artifacts the above build container is required. Make sure the +build for the build container ran successful before starting the build for the aggregation service +artifacts. To trigger the build run: @@ -76,12 +78,14 @@ To trigger the build run: aws codebuild start-build --project-name aggregation-service-artifacts-build --region ``` -The build can take several minutes. You can check the status at `https://.console.aws.amazon.com/codesuite/codebuild/projects`. +The build can take several minutes. You can check the status at +`https://.console.aws.amazon.com/codesuite/codebuild/projects`. ## Download artifacts -To download the artifacts you can use `aws s3` commands. Download the artifacts to `/terraform/aws/jars`. -Run the following in `/terraform/aws` +To download the artifacts you can use `aws s3` commands. Download the artifacts to +`/terraform/aws/jars`. Run the following in `/terraform/aws` + ```sh mkdir -p jars aws s3 cp s3:///aggregation-service/$(cat ../../VERSION)/ jars/ --recursive @@ -93,4 +97,5 @@ Switch to `/terraform/aws`. Run `bash fetch_terraform.sh`. -After downloading the artifacts and running above script continue with [Set up your deployment environment](/README.md#set-up-your-deployment-environment) \ No newline at end of file +After downloading the artifacts and running above script continue with +[Set up your deployment environment](/README.md#set-up-your-deployment-environment) diff --git a/build-scripts/aws/buildspec.yml b/build-scripts/aws/buildspec.yml index e8b623b7..7cedb417 100644 --- a/build-scripts/aws/buildspec.yml +++ b/build-scripts/aws/buildspec.yml @@ -38,4 +38,4 @@ phases: cache: paths: - - '/root/.cache/bazel/**/*' \ No newline at end of file + - '/root/.cache/bazel/**/*' diff --git a/build-scripts/aws/publish.sh b/build-scripts/aws/publish.sh index ce38f62e..c41d18f6 100644 --- a/build-scripts/aws/publish.sh +++ b/build-scripts/aws/publish.sh @@ -17,7 +17,7 @@ VERSION=$(cat ../../VERSION) # build jar artifacts with release version suffix and publish to S3 -# using ENV variables JARS_PUBLISH_BUCKET and JARS_PUBLISH_BUCKET_PATH for publish destination +# using ENV variables JARS_PUBLISH_BUCKET and JARS_PUBLISH_BUCKET_PATH for publish destination bazel run //terraform/aws:aws_change_handler_lambda_release \ --//terraform/aws:bucket_flag=$JARS_PUBLISH_BUCKET --//terraform/aws:bucket_path_flag=$JARS_PUBLISH_BUCKET_PATH \ -- --version=$VERSION @@ -35,4 +35,4 @@ bazel run //terraform/aws:aws_frontend_cleanup_handler_lambda_release \ -- --version=$VERSION bazel run //terraform/aws:local_testing_tool_release \ --//terraform/aws:bucket_flag=$JARS_PUBLISH_BUCKET --//terraform/aws:bucket_path_flag=$JARS_PUBLISH_BUCKET_PATH \ --- --version=$VERSION \ No newline at end of file +-- --version=$VERSION diff --git a/build-scripts/aws/set_ami_to_public.sh b/build-scripts/aws/set_ami_to_public.sh index 9806f6dc..1469bbd3 100644 --- a/build-scripts/aws/set_ami_to_public.sh +++ b/build-scripts/aws/set_ami_to_public.sh @@ -33,4 +33,4 @@ set_ami_to_public_by_prefix() { --launch-permission "Add=[{Group=all}]" } -"$@" \ No newline at end of file +"$@" diff --git a/build-scripts/aws/terraform/codebuild.auto.tfvars_sample b/build-scripts/aws/terraform/codebuild.auto.tfvars_sample index 35a8452c..a4c02ecb 100644 --- a/build-scripts/aws/terraform/codebuild.auto.tfvars_sample +++ b/build-scripts/aws/terraform/codebuild.auto.tfvars_sample @@ -16,4 +16,4 @@ region = "" build_artifacts_output_bucket = "" -github_personal_access_token = "" \ No newline at end of file +github_personal_access_token = "" diff --git a/build-scripts/aws/terraform/codebuild.tf b/build-scripts/aws/terraform/codebuild.tf index 73de5359..e727b098 100644 --- a/build-scripts/aws/terraform/codebuild.tf +++ b/build-scripts/aws/terraform/codebuild.tf @@ -233,7 +233,7 @@ resource "aws_codebuild_project" "bazel_build_container" { buildspec = "build-scripts/aws/build-container/buildspec.yml" } - source_version = var.aggregation_service_github_repo_version == "" ? "v${local.release_version}" : var.aggregation_service_github_repo_version + source_version = var.aggregation_service_github_repo_branch == "" ? "v${local.release_version}" : var.aggregation_service_github_repo_branch } diff --git a/build-scripts/aws/terraform/codebuild_variables.tf b/build-scripts/aws/terraform/codebuild_variables.tf index 9ec8bd9e..fbbc32ab 100644 --- a/build-scripts/aws/terraform/codebuild_variables.tf +++ b/build-scripts/aws/terraform/codebuild_variables.tf @@ -67,4 +67,4 @@ variable "aggregation_service_github_repo_branch" { type = string description = "Aggregation Service Github repository branch" default = "" -} \ No newline at end of file +} diff --git a/build_defs/release.bzl b/build_defs/release.bzl index 6cfa4ec0..72cabb9b 100644 --- a/build_defs/release.bzl +++ b/build_defs/release.bzl @@ -75,6 +75,9 @@ fi s3_jar_release_rule = rule( implementation = _s3_jar_release_impl, attrs = { + "artifact_base_name": attr.string( + mandatory = True, + ), "jar_target": attr.label( allow_single_file = True, mandatory = True, @@ -88,9 +91,6 @@ s3_jar_release_rule = rule( mandatory = True, providers = [BuildSettingInfo], ), - "artifact_base_name": attr.string( - mandatory = True, - ), }, executable = True, ) diff --git a/build_defs/scp/BUILD b/build_defs/scp/BUILD deleted file mode 100644 index 7a829587..00000000 --- a/build_defs/scp/BUILD +++ /dev/null @@ -1,3 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -exports_files(glob(["*.patch"])) diff --git a/build_defs/scp/scp-0.30.0.patch b/build_defs/scp/scp-0.30.0.patch deleted file mode 100644 index 8a93c56c..00000000 --- a/build_defs/scp/scp-0.30.0.patch +++ /dev/null @@ -1,33 +0,0 @@ -diff --git a/build_defs/aws/enclave/container.bzl b/build_defs/aws/enclave/container.bzl -index 3a407dd0..05ae4dc2 100644 ---- a/build_defs/aws/enclave/container.bzl -+++ b/build_defs/aws/enclave/container.bzl -@@ -14,7 +14,7 @@ - - load("@io_bazel_rules_docker//container:container.bzl", "container_image") - --_PROXY_BINARY_FILES = [Label("//cc/aws/proxy/src:proxy_preload"), Label("//cc/aws/proxy/src:proxify")] -+_PROXY_BINARY_FILES = [Label("//cc/aws/proxy:libproxy_preload.so"), Label("//cc/aws/proxy:proxify")] - - def java_enclave_image( - *, -diff --git a/cc/aws/proxy/BUILD b/cc/aws/proxy/BUILD -index c102f74f..a6554501 100644 ---- a/cc/aws/proxy/BUILD -+++ b/cc/aws/proxy/BUILD -@@ -76,10 +76,13 @@ genrule( - bazel/cc/aws/proxy:build_container \ - bash -c ' - set -eux -- yum install gcc gcc-c++ tar git -y > /dev/null -+ yum install glibc-devel-2.26-60.amzn2 glibc-headers-2.26-60.amzn2 gcc-7.3.1-15.amzn2 gcc-c++-7.3.1-15.amzn2 tar git -y > /dev/null - tar xf /source_code.tar -C / - cd /scp -- bazel build //cc/aws/proxy/src:proxify //cc/aws/proxy/src:proxy //cc/aws/proxy/src:proxy_preload -+ bazel build \ -+ //cc/aws/proxy/src:proxify \ -+ //cc/aws/proxy/src:proxy \ -+ //cc/aws/proxy/src:proxy_preload - cp $$(bazel info bazel-bin)/cc/aws/proxy/src/{libproxy_preload.so,proxify,proxy} /output - ' - cp "$${OUTPUT_DIR}/libproxy_preload.so" $(location libproxy_preload.so) diff --git a/API.md b/docs/API.md similarity index 80% rename from API.md rename to docs/API.md index 796b6950..f8a70345 100644 --- a/API.md +++ b/docs/API.md @@ -59,7 +59,7 @@ Duplicate job (job_request_id already taken): 409 Conflict #### Success Response Payload -`{​} // Empty object in response body for success` +`{} // Empty object in response body for success` #### Error Response body @@ -67,21 +67,23 @@ These match the [Google Cloud Error Model](https://cloud.google.com/apis/design/ ```jsonc { - "error": { - "code": 3, - // Corresponds to this - "message": "detailed error message string", - "status": "INVALID_ARGUMENT", - "details": [{ - "reason": "API_KEY_INVALID", - "domain": "foo.com", - // might not be present - "metadata": { - // Map, might not be present - "service": "translate.googleapis.com" - } - }] - } + "error": { + "code": 3, + // Corresponds to this + "message": "detailed error message string", + "status": "INVALID_ARGUMENT", + "details": [ + { + "reason": "API_KEY_INVALID", + "domain": "foo.com", + // might not be present + "metadata": { + // Map, might not be present + "service": "translate.googleapis.com" + } + } + ] + } } ``` @@ -138,7 +140,7 @@ Not found: 404 Not Found "category": , "count": }, - … + ... ] } } @@ -165,21 +167,23 @@ These match the [Google Cloud Error Model](https://cloud.google.com/apis/design/ ```jsonc { - "error": { - "code": 3, - // Corresponds to this - "message": "detailed error message string", - "status": "INVALID_ARGUMENT", - "details": [{ - "reason": "API_KEY_INVALID", - "domain": "foo.com", - // might not be present - "metadata": { - // Map, might not be present - "service": "translate.googleapis.com" - } - }] - } + "error": { + "code": 3, + // Corresponds to this + "message": "detailed error message string", + "status": "INVALID_ARGUMENT", + "details": [ + { + "reason": "API_KEY_INVALID", + "domain": "foo.com", + // might not be present + "metadata": { + // Map, might not be present + "service": "translate.googleapis.com" + } + } + ] + } } ``` diff --git a/COLLECTING.md b/docs/COLLECTING.md similarity index 58% rename from COLLECTING.md rename to docs/COLLECTING.md index 0fdee47b..3b017ce9 100644 --- a/COLLECTING.md +++ b/docs/COLLECTING.md @@ -1,82 +1,85 @@ # Collecting and Batching Aggregatable Reports -This document provides instructions and code snippets -on how to collect, transform and batch [Aggregatable Reports](https://github.com/WICG/conversion-measurement-api/blob/main/AGGREGATE.md#aggregatable-reports) -produced by the [Attribution Reporting API](https://github.com/WICG/conversion-measurement-api/blob/main/AGGREGATE.md) +This document provides instructions and code snippets on how to collect, transform and batch +[Aggregatable Reports](https://github.com/WICG/conversion-measurement-api/blob/main/AGGREGATE.md#aggregatable-reports) +produced by the +[Attribution Reporting API](https://github.com/WICG/conversion-measurement-api/blob/main/AGGREGATE.md) The Attribution Reporting API can generate 4 possible types of reports during the [Privacy Sandbox Relevance and Measurement origin trials (OT)](https://developer.chrome.com/origintrials/#/view_trial/771241436187197441). -These reports are sent to predefined endpoints to the domain registered during -source registration (such as ). -See this [demo](https://goo.gle/attribution-reporting-demo) for examples. +These reports are sent to predefined endpoints to the domain registered during source registration +(such as ). See this [demo](https://goo.gle/attribution-reporting-demo) +for examples. 1. Event-level report - - Reporting URL: `http://adtech.localhost/.well-known/attribution-reporting/report-event-attribution` + - Reporting URL: + `http://adtech.localhost/.well-known/attribution-reporting/report-event-attribution` 1. Event-level debug report - - Reporting URL: `http://adtech.localhost/.well-known/attribution-reporting/debug/report-event-attribution` + - Reporting URL: + `http://adtech.localhost/.well-known/attribution-reporting/debug/report-event-attribution` 1. Aggregatable report - - Reporting URL: `http://adtech.localhost/.well-known/attribution-reporting/report-aggregate-attribution` + - Reporting URL: + `http://adtech.localhost/.well-known/attribution-reporting/report-aggregate-attribution` 1. Aggregatable debug report - - Reporting URL: `http://adtech.localhost/.well-known/attribution-reporting/debug/report-aggregate-attribution` + - Reporting URL: + `http://adtech.localhost/.well-known/attribution-reporting/debug/report-aggregate-attribution` -*The `.well-known/…` paths are predefined paths which can not be customized. -To collect reports, you need to run an endpoint that can respond to POST requests -on the above paths.* +_The `.well-known/...` paths are predefined paths which can not be customized. To collect reports, +you need to run an endpoint that can respond to POST requests on the above paths._ ## Aggregatable report sample This is a sample aggregatable report produced with the -[Attribution Reporting API Demo](https://goo.gle/attribution-reporting-demo) -with debugging enabled. +[Attribution Reporting API Demo](https://goo.gle/attribution-reporting-demo) with debugging enabled. ```json { - "aggregation_service_payloads": [ - { - "debug_cleartext_payload": "omRkYXRhgaJldmFsdWVEAACAAGZidWNrZXRQAAAAAAAAAAAAAAAAAAAFWWlvcGVyYXRpb25paGlzdG9ncmFt", - "key_id": "e101cca5-3dec-4d4f-9823-9c7984b0bafe", - "payload": "26/oZSjHABFqsIxR4Gyh/DpmJLNA/fcp43Wdc1/sblss3eAkAPsqJLphnKjAC2eLFR2bQolMTOneOU5sMWuCfag2tmFlQKLjTkNv85Wq6HAmLg+Zq+YU0gxF573yzK38Cj2pWtb65lhnq9dl4Yiz" - } - ], - "attribution_destination": "http://shoes.localhost", - "shared_info": "{\"debug_mode\":\"enabled\",\"privacy_budget_key\":\"OtLi6K1k0yNpebFbh92gUh/Cf8HgVBVXLo/BU50SRag=\",\"report_id\":\"00cf2236-a4fa-40e5-a7aa-d2ceb33a4d9d\",\"reporting_origin\":\"http://adtech.localhost:3000\",\"scheduled_report_time\":\"1649652363\",\"version\":\"\"}", - "source_debug_key": "531933890459023", - "source_registration_time": "1649635200", - "source_site": "http://news.localhost", - "trigger_debug_key": "531933890459023" + "aggregation_service_payloads": [ + { + "debug_cleartext_payload": "omRkYXRhgaJldmFsdWVEAACAAGZidWNrZXRQAAAAAAAAAAAAAAAAAAAFWWlvcGVyYXRpb25paGlzdG9ncmFt", + "key_id": "e101cca5-3dec-4d4f-9823-9c7984b0bafe", + "payload": "26/oZSjHABFqsIxR4Gyh/DpmJLNA/fcp43Wdc1/sblss3eAkAPsqJLphnKjAC2eLFR2bQolMTOneOU5sMWuCfag2tmFlQKLjTkNv85Wq6HAmLg+Zq+YU0gxF573yzK38Cj2pWtb65lhnq9dl4Yiz" + } + ], + "attribution_destination": "http://shoes.localhost", + "shared_info": "{\"debug_mode\":\"enabled\",\"privacy_budget_key\":\"OtLi6K1k0yNpebFbh92gUh/Cf8HgVBVXLo/BU50SRag=\",\"report_id\":\"00cf2236-a4fa-40e5-a7aa-d2ceb33a4d9d\",\"reporting_origin\":\"http://adtech.localhost:3000\",\"scheduled_report_time\":\"1649652363\",\"version\":\"\"}", + "source_debug_key": "531933890459023", + "source_registration_time": "1649635200", + "source_site": "http://news.localhost", + "trigger_debug_key": "531933890459023" } ``` -The `debug_cleartext_payload` field contains the base64 encoded [CBOR](https://cbor.io/) -payload. The above CBOR payload decodes into the following data in JSON format -(Decoded with [CBOR Playground](https://cbor.me)). The bucket value is encoded -as a sequence of 'characters' representing the underlying bytes. While some -bytes may be represented as ASCII characters, others are unicode escaped. +The `debug_cleartext_payload` field contains the base64 encoded [CBOR](https://cbor.io/) payload. +The above CBOR payload decodes into the following data in JSON format (Decoded with +[CBOR Playground](https://cbor.me)). The bucket value is encoded as a sequence of 'characters' +representing the underlying bytes. While some bytes may be represented as ASCII characters, others +are unicode escaped. ```json { - "data": [ - { - "value": "\u0000\u0000\x80\u0000", - "bucket": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0005Y" - } - ], - "operation": "histogram" + "data": [ + { + "value": "\u0000\u0000\x80\u0000", + "bucket": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0005Y" + } + ], + "operation": "histogram" } ``` ## Convert the aggregatable report into Avro binary representation -The [sample report](#aggregatable-report-sample) lists a `debug_cleartext_payload` -field that is *not* encrypted and can be processed with the local testing tool. +The [sample report](#aggregatable-report-sample) lists a `debug_cleartext_payload` field that is +_not_ encrypted and can be processed with the local testing tool. -Follow the instructions in the [README.md#using-the-local-testing-tool](./README.md#using-the-local-testing-tool) -to download the local testing tool. +Follow the instructions in the +[README.md#using-the-local-testing-tool](/README.md#using-the-local-testing-tool) to download the +local testing tool. When testing the aggregation service locally and on Amazon Web Services -[Nitro Enclaves](https://aws.amazon.com/ec2/nitro/nitro-enclaves/), -an [Avro](https://avro.apache.org/) batch with the following record schema is -expected. +[Nitro Enclaves](https://aws.amazon.com/ec2/nitro/nitro-enclaves/), an +[Avro](https://avro.apache.org/) batch with the following record schema is expected. ### `reports.avsc` @@ -101,39 +104,34 @@ expected. } ``` -For local testing, the avro `payload` field expects a byte array of the -`debug_cleartext_payload` field (`base64` encoded). The `debug_cleartext_payload` -field is present in each aggregation service payload object in the -`aggregation_service_payloads` list of an aggregatable report with debugging +For local testing, the avro `payload` field expects a byte array of the `debug_cleartext_payload` +field (`base64` encoded). The `debug_cleartext_payload` field is present in each aggregation service +payload object in the `aggregation_service_payloads` list of an aggregatable report with debugging enabled. For testing with encrypted reports on the Amazon Web Services -[Nitro Enclaves](https://aws.amazon.com/ec2/nitro/nitro-enclaves/), the avro -`payload` field expects a byte array of the aggregatable report's -`aggregation_service_payloads` object's `payload` field. +[Nitro Enclaves](https://aws.amazon.com/ec2/nitro/nitro-enclaves/), the avro `payload` field expects +a byte array of the aggregatable report's `aggregation_service_payloads` object's `payload` field. ## Collect, transform and batch reports -The following code snippets are in Golang, but can be adapted to other -programming languages. +The following code snippets are in Golang, but can be adapted to other programming languages. ### Listen on predefined endpoints -When debugging is enabled for the Attribution Reporting API, additional fields -are present in the reports, and a duplicate debug report is sent immediately. -The following 2 predefined endpoints are used: +When debugging is enabled for the Attribution Reporting API, additional fields are present in the +reports, and a duplicate debug report is sent immediately. The following 2 predefined endpoints are +used: -1. `.well-known/attribution-reporting/report-aggregate-attribution` for regular, -scheduled (delayed) reports with encrypted payloads. If debugging is enabled, -these will contain additional fields: for example, a cleartext payload if both -debug keys are also set. -2. `.well-known/attribution-reporting/debug/report-aggregate-attribution` for -debug reports that are duplicates of the regular reports, but sent immediately -at generation time. +1. `.well-known/attribution-reporting/report-aggregate-attribution` for regular, scheduled (delayed) + reports with encrypted payloads. If debugging is enabled, these will contain additional fields: + for example, a cleartext payload if both debug keys are also set. +2. `.well-known/attribution-reporting/debug/report-aggregate-attribution` for debug reports that are + duplicates of the regular reports, but sent immediately at generation time. First, lets define all types we will work with: -- Aggregatable report generated from the Attribution Reporting API +- Aggregatable report generated from the Attribution Reporting API ```go // AggregatableReport contains the information generated by the Attribution @@ -162,8 +160,8 @@ First, lets define all types we will work with: } ``` -- Aggregatable report in Avro format, as expected by the aggregation service (you'll -need to import [gopkg.in/avro.v0](https://pkg.go.dev/gopkg.in/avro.v0)) +- Aggregatable report in Avro format, as expected by the aggregation service (you'll need to + import [gopkg.in/avro.v0](https://pkg.go.dev/gopkg.in/avro.v0)) ```go // AvroAggregatableReport format expected by aggregation service and local testing tool @@ -176,15 +174,15 @@ need to import [gopkg.in/avro.v0](https://pkg.go.dev/gopkg.in/avro.v0)) Now let's register request handlers and start an http server: - ```go - func main() { - http.HandleFunc("/.well-known/attribution-reporting/report-aggregate-attribution", collectEndpoint) - http.HandleFunc("/.well-known/attribution-reporting/debug/report-aggregate-attribution", collectEndpoint) - var address = ":3001" - log.Printf("Starting Collector on address %v", address) - log.Fatal(http.ListenAndServe(address, nil)) - } - ``` +```go +func main() { + http.HandleFunc("/.well-known/attribution-reporting/report-aggregate-attribution", collectEndpoint) + http.HandleFunc("/.well-known/attribution-reporting/debug/report-aggregate-attribution", collectEndpoint) + var address = ":3001" + log.Printf("Starting Collector on address %v", address) + log.Fatal(http.ListenAndServe(address, nil)) +} +``` And here is how we handle incoming reports in our `HandlerFunc` implementation: @@ -276,15 +274,15 @@ func collectEndpoint(w http.ResponseWriter, r *http.Request) { Once an aggregatable report has been collected, it'll be stored in the `output_regular_reports_.avro` and `output_regular_clear_text_reports_.avro` -for report received on the `.well-known/attribution-reporting/report-aggregate-attribution` -endpoint and `output_debug_reports_.avro` and `output_debug_clear_text_reports_.avro` +for report received on the `.well-known/attribution-reporting/report-aggregate-attribution` endpoint +and `output_debug_reports_.avro` and `output_debug_clear_text_reports_.avro` for report received on the `.well-known/attribution-reporting/debug/report-aggregate-attribution` endpoint respectively. ## Process Avro batch files -To process the above Avro files, you must specify the expected bucket keys -in a domain file `output_domain.avro` with the following Avro schema. +To process the above Avro files, you must specify the expected bucket keys in a domain file +`output_domain.avro` with the following Avro schema. ### `output_domain.avsc` @@ -304,19 +302,19 @@ in a domain file `output_domain.avro` with the following Avro schema. ### Generate a output domain Avro file -You can use the [Avro Tools](https://www.apache.org/dyn/closer.cgi/avro/) to -generate a `output_domain.avro` from a JSON input file. +You can use the [Avro Tools](https://www.apache.org/dyn/closer.cgi/avro/) to generate a +`output_domain.avro` from a JSON input file. -You can download the Avro Tools jar 1.11.0 [here](http://archive.apache.org/dist/avro/avro-1.11.0/java/avro-tools-1.11.0.jar) +You can download the Avro Tools jar 1.11.0 +[here](http://archive.apache.org/dist/avro/avro-1.11.0/java/avro-tools-1.11.0.jar) -We use the following `output_domain.json` input file to generate our -`output_domain.avro` file. This uses the bucket from the above -[sample aggregatable report](#aggregatable-report-sample). The below sample uses -unicode escaped "characters" to encode the byte array bucket value. +We use the following `output_domain.json` input file to generate our `output_domain.avro` file. This +uses the bucket from the above [sample aggregatable report](#aggregatable-report-sample). The below +sample uses unicode escaped "characters" to encode the byte array bucket value. ```json { - "bucket": "\u0005Y" + "bucket": "\u0005Y" } ``` @@ -329,13 +327,15 @@ java -jar avro-tools-1.11.0.jar fromjson \ ### Produce a summary report locally -Using the local testing tool, you now can generate a summary report. [See all flags and descriptions](./API.md#local-testing-tool) +Using the local testing tool, you now can generate a summary report. +[See all flags and descriptions](./docs/API.md#local-testing-tool) -Follow the instructions in the [README.md#using-the-local-testing-tool](./README.md#using-the-local-testing-tool) -to download the local testing tool. +Follow the instructions in the +[README.md#using-the-local-testing-tool](/README.md#using-the-local-testing-tool) to download the +local testing tool. -We will run the tool, without adding noise to the summary report, to receive the -expected value of `32768` from the [sample aggregatable report](#aggregatable-report-sample). +We will run the tool, without adding noise to the summary report, to receive the expected value of +`32768` from the [sample aggregatable report](#aggregatable-report-sample). ```sh java -jar LocalRunner_deploy.jar \ @@ -345,14 +345,13 @@ java -jar LocalRunner_deploy.jar \ --no_noising ``` -The output of above tool execution will be in `output.json` with the following -content +The output of above tool execution will be in `output.json` with the following content ```json [ - { - "bucket" : "\u0005Y", - "value" : 32768 - } + { + "bucket": "\u0005Y", + "value": 32768 + } ] ``` diff --git a/docs/DEBUGGING.md b/docs/DEBUGGING.md new file mode 100644 index 00000000..dc1adbf8 --- /dev/null +++ b/docs/DEBUGGING.md @@ -0,0 +1,157 @@ +# Debug aggregation runs with encrypted payloads + +This document describes the debugging support for the Aggregation Service running in a TEE using +encrypted payloads of aggregatable reports. This allows you to debug your production setup and +understand how the encrypted payloads of aggregatable reports are processed. Reports with +debug_cleartext_payload can be used with the +[local testing tool](/README.md#using-the-local-testing-tool) and are helpful for understanding the +content of reports and validating that registrations on the browser client or device are configured +properly. + +To test the Aggregation Service, you can enable debug aggregation runs which use encrypted payloads +of aggregatable reports to generate debug summary reports. When executing a debug run, no noise is +added to the debug summary report, and annotations are added to indicate whether keys are present in +domain input and/or reports. This allows developers to: + +- Analyze the reports +- Determine if the aggregation was completed correctly, per the adtech's specifications +- Understand the impact of noise in summary reports +- Determine how to set the proper domain keys + +Additionally, debug runs do not enforce the +[No-Duplicates rule](https://github.com/WICG/attribution-reporting-api/blob/main/AGGREGATION_SERVICE_TEE.md#no-duplicates-rule) +across batches. The No-Duplicates rule is still enforced within a batch. This allows adtech to try +different batches without worrying about making them +[disjoint](https://github.com/WICG/attribution-reporting-api/blob/main/AGGREGATION_SERVICE_TEE.md#disjoint-batches) +during testing or debugging. + +Once third-party cookies are deprecated, the client (browser or operating system) will no longer +generate aggregatable reports that are enabled for debugging. At that time, debug runs with +encrypted payloads will no longer be supported for reports from real user devices or browsers. + +In this document, you'll find code snippets and instructions for how to debug the Aggregation +Service and create debug summary reports. + +## Create a debug job + +To create an aggregation debug job, add the `debug_run` parameter to the `job_parameters` object of +the `createJob` API request. + +`POST https://.execute-api.us-east-1.amazonaws.com/stage/v1alpha/createJob` + +```json +{ + "input_data_blob_prefix": "input/reports.avro", + "input_data_bucket_name": "", + "output_data_blob_prefix": "output/summary_report.avro", + "output_data_bucket_name": "", + "job_parameters": { + "attribution_report_to": "", + "output_domain_blob_prefix": "domain/domain.avro", + "output_domain_bucket_name": "", + "debug_run": "true" + }, + "Job_request_id": "test01" +} +``` + +If `debug_run` is not present in`job_parameters` or it's set to `false`, a normal noised aggregation +run is created. More details about `createJob` API can be found in +[detailed API spec](/docs/API.md#createjob-endpoint). + +## Debuggable aggregatable reports + +A debug run only considers reports that have the flag `"debug_mode": "enabled"` in the report +shared_info ([aggregatable report sample](/docs/COLLECTING.md#aggregatable-report-sample)). Reports +with the `debug_mode` flag missing or the `debug_mode` value isn't set to `enabled` aren't included +in the results generated by a debug run. + +The count of reports that were not processed during a debug run is returned in the job response, +which can be previewed in the [detailed API spec](/docs/API.md#createjob-endpoint). In the +`error_counts` field, the category `NUM_REPORTS_DEBUG_NOT_ENABLED` shows the numbers of reports not +processed during a debug run. + +## Results + +Two summary reports are generated from a debug run: a regular summary report and a debug summary +report. The regular summary report format generated from the debug run is consistent with that of a +regular aggregation run. The debug summary report has a [different format](#debug-summary-report). +The path of the summary report is set in the [createJob](/docs/API.md#createjob-endpoint) API. The +debug summary report will be stored in the "debug" folder under the summary report's path with the +same object name. + +Considering the following createJob parameters for `output_data_bucket_name` and +`output_data_blob_prefix`: + +```json +"output_data_blob_prefix": "output/summary_report.avro", +"output_data_bucket_name": "", +``` + +the following objects are created by a debug run: + +`s3:///output/summary_report.avro` and + +`s3:///output/debug/summary_report.avro`. + +Note that the regular summary report generated during a debug run will only include reports which +have the flag `"debug_mode": "enabled"` in the reports `shared_info`. + +### Debug summary report + +The debug summary report includes the following data: + +- `bucket`: The aggregation key +- `unnoised_metric`: The aggregation value without noise +- `noise`: The approximate noise applied to the aggregated results in the regular summary report +- `annotations`: The annotations associated with the bucket + +The keys in the debug summary report will include all the keys from the output domain. + +If the key is only present in the output domain (not in any of the processed aggregatable reports), +the key will be included in the debug report with `unnoised_metric=0` and +`annotations=["in_domain"]`. + +The keys that are only present in aggregatable reports (not in output domain) will also be included +in the debug report with `unnoised_metric=` and +`annotations=["in_reports"]`. + +Keys that are present in domain and aggregatable reports will have annotations for both +`["in_domain", "in_reports"]`. + +The schema of debug summary reports is in the following [Avro](https://avro.apache.org/) format: + +```avro +{ +"type": "record", +"name": "DebugAggregatedFact", +"fields": [ + { + "name": "bucket", + "type": "bytes", + "doc": "Histogram bucket used in aggregation. 128-bit integer encoded as a 16-byte big-endian bytestring. Leading 0-bits will be left out." + }, + { + "name": "unnoised_metric", + "type": "long", + "doc": "Unnoised metric associated with the bucket." + }, + { + "name": "noise", + "type": "long", + "doc": "The noise applied to the metric in the regular result." + } + { + "name":"annotations", + "type": + { + "type": "array", + "items": { + "type":"enum", + "name":"bucket_tags", + "symbols":["in_domain","in_reports"] + } + } + ] +} +``` diff --git a/java/com/google/aggregate/adtech/worker/AggregationWorkerModule.java b/java/com/google/aggregate/adtech/worker/AggregationWorkerModule.java index 809161e2..77235ed9 100644 --- a/java/com/google/aggregate/adtech/worker/AggregationWorkerModule.java +++ b/java/com/google/aggregate/adtech/worker/AggregationWorkerModule.java @@ -262,7 +262,7 @@ protected void configure() { // result logger install(args.resultLoggerModuleSelector().getResultLoggerModule()); switch (args.resultLoggerModuleSelector()) { - case LOCAL_AVRO_TO_S3: + case LOCAL_TO_CLOUD: bind(Path.class) .annotatedWith(ResultWorkingDirectory.class) .toInstance(Paths.get(args.getResultWorkingDirectoryPathString())); diff --git a/java/com/google/aggregate/adtech/worker/AggregationWorkerReturnCode.java b/java/com/google/aggregate/adtech/worker/AggregationWorkerReturnCode.java index 22b2a97d..f51f8e5c 100644 --- a/java/com/google/aggregate/adtech/worker/AggregationWorkerReturnCode.java +++ b/java/com/google/aggregate/adtech/worker/AggregationWorkerReturnCode.java @@ -35,6 +35,18 @@ public enum AggregationWorkerReturnCode { /** The job had invalid configuration and could not be processed. */ INVALID_JOB, + /** Error encountered while logging result. */ + RESULT_LOGGING_ERROR, + + /** Error encountered while processing domain. */ + DOMAIN_PROCESS_EXCEPTION, + /** A permission issue occurred and the job couldn't be processed. */ - PERMISSION_ERROR + PERMISSION_ERROR, + + /** Report or Domain input data ead filed. */ + INPUT_DATA_READ_FAILED, + + /** Aggregation Job completed successfully. */ + SUCCESS } diff --git a/java/com/google/aggregate/adtech/worker/AggregationWorkerRunner.java b/java/com/google/aggregate/adtech/worker/AggregationWorkerRunner.java index 610208ac..e91d1169 100644 --- a/java/com/google/aggregate/adtech/worker/AggregationWorkerRunner.java +++ b/java/com/google/aggregate/adtech/worker/AggregationWorkerRunner.java @@ -17,10 +17,16 @@ package com.google.aggregate.adtech.worker; import com.beust.jcommander.JCommander; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; final class AggregationWorkerRunner { + private static final Logger logger = LoggerFactory.getLogger(AggregationWorkerRunner.class); + public static void main(String[] args) { + logger.info("Worker Args: \n" + String.join("\n", args)); + AggregationWorkerArgs cliArgs = new AggregationWorkerArgs(); JCommander.newBuilder().allowParameterOverwriting(true).addObject(cliArgs).build().parse(args); diff --git a/java/com/google/aggregate/adtech/worker/Annotations.java b/java/com/google/aggregate/adtech/worker/Annotations.java index 3227eca6..1afdf10e 100644 --- a/java/com/google/aggregate/adtech/worker/Annotations.java +++ b/java/com/google/aggregate/adtech/worker/Annotations.java @@ -34,13 +34,13 @@ public final class Annotations { @BindingAnnotation @Target({FIELD, PARAMETER, METHOD}) @Retention(RUNTIME) - @interface WorkerServiceManager {} + public @interface WorkerServiceManager {} /** Annotation for the {@link com.google.common.util.concurrent.Service} doing the work. */ @BindingAnnotation @Target({FIELD, PARAMETER, METHOD}) @Retention(RUNTIME) - @interface PullWorkService {} + public @interface PullWorkService {} /** Annotation for the thread pool doing the non-blocking work. */ @BindingAnnotation diff --git a/java/com/google/aggregate/adtech/worker/BUILD b/java/com/google/aggregate/adtech/worker/BUILD index 704ef640..a8162e0e 100644 --- a/java/com/google/aggregate/adtech/worker/BUILD +++ b/java/com/google/aggregate/adtech/worker/BUILD @@ -12,8 +12,75 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_binary", "java_library") + package(default_visibility = ["//visibility:public"]) +java_library( + name = "worker_util", + srcs = [ + "AggregationWorker.java", + "AggregationWorkerReturnCode.java", + "Annotations.java", + "DecryptionModuleSelector.java", + "DomainFormatSelector.java", + "JobProcessor.java", + "LocalFileToCloudStorageLogger.java", + "LocalFileToCloudStorageLoggerModule.java", + "NoisingSelector.java", + "PrivacyBudgetingSelector.java", + "RecordReader.java", + "RecordReaderFactory.java", + "ResultLogger.java", + "ResultLoggerModule.java", + "ResultLoggerModuleSelector.java", + "StopwatchExportSelector.java", + "WorkerModule.java", + "WorkerPullWorkService.java", + ], + deps = [ + "//java/com/google/aggregate/adtech/worker/aggregation/domain", + "//java/com/google/aggregate/adtech/worker/aggregation/domain:avro_domain", + "//java/com/google/aggregate/adtech/worker/aggregation/domain:text_domain", + "//java/com/google/aggregate/adtech/worker/aggregation/engine", + "//java/com/google/aggregate/adtech/worker/aggregation/privacy:http_privacy_budgeting_service_bridge", + "//java/com/google/aggregate/adtech/worker/aggregation/privacy:privacy_budgeting_service_bridge", + "//java/com/google/aggregate/adtech/worker/aggregation/privacy:unlimited_privacy_budgeting_service_bridge", + "//java/com/google/aggregate/adtech/worker/configs", + "//java/com/google/aggregate/adtech/worker/decryption", + "//java/com/google/aggregate/adtech/worker/decryption/hybrid", + "//java/com/google/aggregate/adtech/worker/decryption/noop", + "//java/com/google/aggregate/adtech/worker/exceptions", + "//java/com/google/aggregate/adtech/worker/model", + "//java/com/google/aggregate/adtech/worker/testing:in_memory_logger", + "//java/com/google/aggregate/adtech/worker/util", + "//java/com/google/aggregate/adtech/worker/validation", + "//java/com/google/aggregate/adtech/worker/writer", + "//java/com/google/aggregate/adtech/worker/writer/avro", + "//java/com/google/aggregate/perf", + "//java/com/google/aggregate/perf/export:aws_plain_file_exporter", + "//java/com/google/aggregate/perf/export:no_op_exporter", + "//java/com/google/aggregate/perf/export:pain_file_exporter", + "//java/com/google/aggregate/privacy/noise", + "//java/com/google/aggregate/privacy/noise/proto:privacy_parameters_java_proto", + "//java/com/google/aggregate/privacy/noise/testing", + "//java/com/google/aggregate/protocol/avro:avro_report", + "//java/external:clients_blobstorageclient", + "//java/external:clients_blobstorageclient_model", + "//java/external:clients_jobclient", + "//java/external:clients_jobclient_model", + "//java/external:clients_metricclient", + "//java/external:clients_metricclient_model", + "//java/external:guava", + "//java/external:guice", + "//java/external:javax_inject", + "//java/external:operator_protos", + "//java/external:scp_shared_proto", + "//java/external:shared_model", + "//java/external:slf4j", + ], +) + java_library( name = "worker", srcs = [ @@ -22,8 +89,8 @@ java_library( "Annotations.java", "ErrorSummaryAggregator.java", "JobProcessor.java", - "LocalAvroToS3LoggerModule.java", "LocalFileToCloudStorageLogger.java", + "LocalFileToCloudStorageLoggerModule.java", "RecordReader.java", "RecordReaderFactory.java", "ReportDecrypterAndValidator.java", @@ -34,8 +101,10 @@ java_library( ], deps = [ "//java/com/google/aggregate/adtech/worker/aggregation/engine", + "//java/com/google/aggregate/adtech/worker/aggregation/privacy:privacy_budgeting_service_bridge", "//java/com/google/aggregate/adtech/worker/configs", "//java/com/google/aggregate/adtech/worker/decryption", + "//java/com/google/aggregate/adtech/worker/exceptions", "//java/com/google/aggregate/adtech/worker/model", "//java/com/google/aggregate/adtech/worker/util", "//java/com/google/aggregate/adtech/worker/validation", @@ -71,10 +140,8 @@ java_library( "AggregationWorkerModule.java", "AggregationWorkerRunner.java", "DecryptionModuleSelector.java", - "DependencyMetadata.java", "DomainFormatSelector.java", "LibraryAnnotations.java", - "LicenseUtil.java", "LocalAvroResultLoggerModule.java", "LocalJsonResultLoggerModule.java", "LocalResultLogger.java", @@ -100,6 +167,7 @@ java_library( "//java/com/google/aggregate/adtech/worker/decryption", "//java/com/google/aggregate/adtech/worker/decryption/hybrid", "//java/com/google/aggregate/adtech/worker/decryption/noop", + "//java/com/google/aggregate/adtech/worker/exceptions", "//java/com/google/aggregate/adtech/worker/local:localblob_client", "//java/com/google/aggregate/adtech/worker/model", "//java/com/google/aggregate/adtech/worker/model/serdes", @@ -119,9 +187,8 @@ java_library( "//java/com/google/aggregate/privacy/noise", "//java/com/google/aggregate/privacy/noise/proto:privacy_parameters_java_proto", "//java/com/google/aggregate/privacy/noise/testing", + "//java/com/google/aggregate/shared", "//java/com/google/aggregate/shared/mapper", - "//java/external:autovalue", - "//java/external:autovalue_annotations", "//java/external:aws_apache_http", "//java/external:aws_http_client_spi", "//java/external:aws_regions", @@ -147,6 +214,14 @@ java_library( "//java/external:jackson_databind", "//java/external:javax_inject", "//java/external:jcommander", + "//java/external:slf4j", + ], +) + +java_library( + name = "return_code", + srcs = [ + "AggregationWorkerReturnCode.java", ], ) diff --git a/java/com/google/aggregate/adtech/worker/DecryptionModuleSelector.java b/java/com/google/aggregate/adtech/worker/DecryptionModuleSelector.java index 1b8732c1..7426c75a 100644 --- a/java/com/google/aggregate/adtech/worker/DecryptionModuleSelector.java +++ b/java/com/google/aggregate/adtech/worker/DecryptionModuleSelector.java @@ -31,7 +31,7 @@ public enum DecryptionModuleSelector { this.decryptionModule = decryptionModule; } - DecryptionModule getDecryptionModule() { + public DecryptionModule getDecryptionModule() { return decryptionModule; } } diff --git a/java/com/google/aggregate/adtech/worker/DomainFormatSelector.java b/java/com/google/aggregate/adtech/worker/DomainFormatSelector.java index 635aaaf0..ffd330f2 100644 --- a/java/com/google/aggregate/adtech/worker/DomainFormatSelector.java +++ b/java/com/google/aggregate/adtech/worker/DomainFormatSelector.java @@ -30,7 +30,7 @@ public enum DomainFormatSelector { this.domainProcessorClass = domainProcessorClass; } - Class getDomainProcessorClass() { + public Class getDomainProcessorClass() { return domainProcessorClass; } } diff --git a/java/com/google/aggregate/adtech/worker/JobProcessor.java b/java/com/google/aggregate/adtech/worker/JobProcessor.java index dee712f9..73026836 100644 --- a/java/com/google/aggregate/adtech/worker/JobProcessor.java +++ b/java/com/google/aggregate/adtech/worker/JobProcessor.java @@ -16,18 +16,14 @@ package com.google.aggregate.adtech.worker; +import com.google.aggregate.adtech.worker.exceptions.AggregationJobProcessException; import com.google.scp.operator.cpio.jobclient.model.Job; import com.google.scp.operator.cpio.jobclient.model.JobResult; +import java.util.concurrent.ExecutionException; /** Interface for consuming jobs pulled from the pubsub */ public interface JobProcessor { - JobResult process(Job Job) throws AggregationJobProcessException; - - final class AggregationJobProcessException extends Exception { - - public AggregationJobProcessException(Throwable cause) { - super(cause); - } - } + JobResult process(Job Job) + throws ExecutionException, InterruptedException, AggregationJobProcessException; } diff --git a/java/com/google/aggregate/adtech/worker/LocalFileToCloudStorageLogger.java b/java/com/google/aggregate/adtech/worker/LocalFileToCloudStorageLogger.java index 3a07703f..398483ac 100644 --- a/java/com/google/aggregate/adtech/worker/LocalFileToCloudStorageLogger.java +++ b/java/com/google/aggregate/adtech/worker/LocalFileToCloudStorageLogger.java @@ -26,6 +26,7 @@ import com.google.aggregate.adtech.worker.Annotations.DebugWriter; import com.google.aggregate.adtech.worker.Annotations.ResultWriter; +import com.google.aggregate.adtech.worker.exceptions.ResultLogException; import com.google.aggregate.adtech.worker.model.AggregatedFact; import com.google.aggregate.adtech.worker.writer.LocalResultFileWriter; import com.google.aggregate.adtech.worker.writer.LocalResultFileWriter.FileWriteException; @@ -165,5 +166,5 @@ private String getLocalDebugFileName(Job ctx) { @BindingAnnotation @Target({FIELD, PARAMETER, METHOD}) @Retention(RUNTIME) - @interface ResultWorkingDirectory {} + public @interface ResultWorkingDirectory {} } diff --git a/java/com/google/aggregate/adtech/worker/LocalAvroToS3LoggerModule.java b/java/com/google/aggregate/adtech/worker/LocalFileToCloudStorageLoggerModule.java similarity index 95% rename from java/com/google/aggregate/adtech/worker/LocalAvroToS3LoggerModule.java rename to java/com/google/aggregate/adtech/worker/LocalFileToCloudStorageLoggerModule.java index 400e3fe5..88090dac 100644 --- a/java/com/google/aggregate/adtech/worker/LocalAvroToS3LoggerModule.java +++ b/java/com/google/aggregate/adtech/worker/LocalFileToCloudStorageLoggerModule.java @@ -26,7 +26,7 @@ * Module that uses the {@code LocalFileToCloudStorageLogger}, {@code LocalAvroResultFileWriter}, * and {@code SinglePartAwsS3Writer} */ -public final class LocalAvroToS3LoggerModule extends ResultLoggerModule { +public final class LocalFileToCloudStorageLoggerModule extends ResultLoggerModule { @Override public Class getResultLoggerImplementation() { diff --git a/java/com/google/aggregate/adtech/worker/LocalResultLogger.java b/java/com/google/aggregate/adtech/worker/LocalResultLogger.java index 844b8e8d..f735114d 100644 --- a/java/com/google/aggregate/adtech/worker/LocalResultLogger.java +++ b/java/com/google/aggregate/adtech/worker/LocalResultLogger.java @@ -21,6 +21,7 @@ import com.google.aggregate.adtech.worker.Annotations.DebugWriter; import com.google.aggregate.adtech.worker.Annotations.ResultWriter; import com.google.aggregate.adtech.worker.LibraryAnnotations.LocalOutputDirectory; +import com.google.aggregate.adtech.worker.exceptions.ResultLogException; import com.google.aggregate.adtech.worker.model.AggregatedFact; import com.google.aggregate.adtech.worker.writer.LocalResultFileWriter; import com.google.aggregate.adtech.worker.writer.LocalResultFileWriter.FileWriteException; diff --git a/java/com/google/aggregate/adtech/worker/LocalRunner.java b/java/com/google/aggregate/adtech/worker/LocalRunner.java index 20e76878..87a05795 100644 --- a/java/com/google/aggregate/adtech/worker/LocalRunner.java +++ b/java/com/google/aggregate/adtech/worker/LocalRunner.java @@ -17,6 +17,7 @@ package com.google.aggregate.adtech.worker; import com.beust.jcommander.JCommander; +import com.google.aggregate.shared.LicenseUtil; import com.google.common.util.concurrent.ServiceManager; import java.io.IOException; import java.nio.file.Files; diff --git a/java/com/google/aggregate/adtech/worker/NoisingSelector.java b/java/com/google/aggregate/adtech/worker/NoisingSelector.java index 4f3522eb..57828410 100644 --- a/java/com/google/aggregate/adtech/worker/NoisingSelector.java +++ b/java/com/google/aggregate/adtech/worker/NoisingSelector.java @@ -21,7 +21,7 @@ import com.google.inject.Module; /** CLI enum to select the noising implementation. */ -enum NoisingSelector { +public enum NoisingSelector { DP_NOISING(new DpNoisedAggregationModule()), CONSTANT_NOISING(new ConstantNoiseModule()); @@ -31,7 +31,7 @@ enum NoisingSelector { this.noisingModule = noisingModule; } - Module getNoisingModule() { + public Module getNoisingModule() { return noisingModule; } } diff --git a/java/com/google/aggregate/adtech/worker/PrivacyBudgetingSelector.java b/java/com/google/aggregate/adtech/worker/PrivacyBudgetingSelector.java index ade7e67b..aa23edba 100644 --- a/java/com/google/aggregate/adtech/worker/PrivacyBudgetingSelector.java +++ b/java/com/google/aggregate/adtech/worker/PrivacyBudgetingSelector.java @@ -21,7 +21,7 @@ import com.google.aggregate.adtech.worker.aggregation.privacy.UnlimitedPrivacyBudgetingServiceBridge; /** CLI enum to select the privacy budgeting bridge */ -enum PrivacyBudgetingSelector { +public enum PrivacyBudgetingSelector { UNLIMITED(UnlimitedPrivacyBudgetingServiceBridge.class), HTTP(HttpPrivacyBudgetingServiceBridge.class); @@ -31,7 +31,7 @@ enum PrivacyBudgetingSelector { this.bridgeClass = bridgeClass; } - Class getBridge() { + public Class getBridge() { return bridgeClass; } } diff --git a/java/com/google/aggregate/adtech/worker/ResultLogger.java b/java/com/google/aggregate/adtech/worker/ResultLogger.java index 789ab188..d67d61fc 100644 --- a/java/com/google/aggregate/adtech/worker/ResultLogger.java +++ b/java/com/google/aggregate/adtech/worker/ResultLogger.java @@ -16,6 +16,7 @@ package com.google.aggregate.adtech.worker; +import com.google.aggregate.adtech.worker.exceptions.ResultLogException; import com.google.aggregate.adtech.worker.model.AggregatedFact; import com.google.scp.operator.cpio.blobstorageclient.model.DataLocation; import com.google.scp.operator.cpio.jobclient.model.Job; @@ -29,11 +30,4 @@ public interface ResultLogger { /** Takes the aggregation results and logs them to debug results. */ DataLocation logDebugResults(Stream results, Job ctx) throws ResultLogException; - - class ResultLogException extends Exception { - - public ResultLogException(Throwable cause) { - super(cause); - } - } } diff --git a/java/com/google/aggregate/adtech/worker/ResultLoggerModuleSelector.java b/java/com/google/aggregate/adtech/worker/ResultLoggerModuleSelector.java index 7136a1dc..136bede6 100644 --- a/java/com/google/aggregate/adtech/worker/ResultLoggerModuleSelector.java +++ b/java/com/google/aggregate/adtech/worker/ResultLoggerModuleSelector.java @@ -21,7 +21,7 @@ /** CLI enum to select a {@code ResultLoggerModule} to use */ public enum ResultLoggerModuleSelector { IN_MEMORY(new InMemoryResultLoggerModule()), - LOCAL_AVRO_TO_S3(new LocalAvroToS3LoggerModule()); + LOCAL_TO_CLOUD(new LocalFileToCloudStorageLoggerModule()); private final ResultLoggerModule resultLoggerModule; @@ -29,7 +29,7 @@ public enum ResultLoggerModuleSelector { this.resultLoggerModule = resultLoggerModule; } - ResultLoggerModule getResultLoggerModule() { + public ResultLoggerModule getResultLoggerModule() { return resultLoggerModule; } } diff --git a/java/com/google/aggregate/adtech/worker/StopwatchExportSelector.java b/java/com/google/aggregate/adtech/worker/StopwatchExportSelector.java index 873ec174..dbac2c9c 100644 --- a/java/com/google/aggregate/adtech/worker/StopwatchExportSelector.java +++ b/java/com/google/aggregate/adtech/worker/StopwatchExportSelector.java @@ -33,7 +33,7 @@ public enum StopwatchExportSelector { this.exporterClass = exporterClass; } - Class getExporterClass() { + public Class getExporterClass() { return exporterClass; } } diff --git a/java/com/google/aggregate/adtech/worker/WorkerModule.java b/java/com/google/aggregate/adtech/worker/WorkerModule.java index 4e918ba7..cd5c94c8 100644 --- a/java/com/google/aggregate/adtech/worker/WorkerModule.java +++ b/java/com/google/aggregate/adtech/worker/WorkerModule.java @@ -32,7 +32,7 @@ import java.util.function.Supplier; import javax.inject.Singleton; -final class WorkerModule extends AbstractModule { +public final class WorkerModule extends AbstractModule { @Provides @WorkerServiceManager diff --git a/java/com/google/aggregate/adtech/worker/WorkerPullWorkService.java b/java/com/google/aggregate/adtech/worker/WorkerPullWorkService.java index db11506b..cf6af88b 100644 --- a/java/com/google/aggregate/adtech/worker/WorkerPullWorkService.java +++ b/java/com/google/aggregate/adtech/worker/WorkerPullWorkService.java @@ -22,6 +22,7 @@ import com.google.aggregate.adtech.worker.Annotations.BenchmarkMode; import com.google.aggregate.adtech.worker.Annotations.BlockingThreadPool; import com.google.aggregate.adtech.worker.Annotations.NonBlockingThreadPool; +import com.google.aggregate.adtech.worker.exceptions.AggregationJobProcessException; import com.google.aggregate.adtech.worker.validation.JobValidator; import com.google.aggregate.perf.StopwatchExporter; import com.google.aggregate.perf.StopwatchExporter.StopwatchExportException; @@ -37,6 +38,7 @@ import com.google.scp.operator.protos.shared.backend.ErrorSummaryProto.ErrorSummary; import com.google.scp.operator.protos.shared.backend.ResultInfoProto.ResultInfo; import com.google.scp.shared.proto.ProtoUtil; +import java.time.Clock; import java.time.Instant; import java.util.Optional; import javax.inject.Inject; @@ -44,7 +46,7 @@ import org.slf4j.LoggerFactory; /** Guava service for repeatedly pulling from the pubsub and processing the request */ -final class WorkerPullWorkService extends AbstractExecutionThreadService { +public final class WorkerPullWorkService extends AbstractExecutionThreadService { private static final Logger logger = LoggerFactory.getLogger(WorkerPullWorkService.class); @@ -54,6 +56,7 @@ final class WorkerPullWorkService extends AbstractExecutionThreadService { private final StopwatchRegistry stopwatchRegistry; private final StopwatchExporter stopwatchExporter; private final boolean benchmarkMode; + private final Clock clock; private final ListeningExecutorService nonBlockingThreadPool; private final ListeningExecutorService blockingThreadPool; @@ -71,6 +74,7 @@ final class WorkerPullWorkService extends AbstractExecutionThreadService { JobClient jobClient, JobProcessor jobProcessor, MetricClient metricClient, + Clock clock, StopwatchRegistry stopwatchRegistry, StopwatchExporter stopwatchExporter, @NonBlockingThreadPool ListeningExecutorService nonBlockingThreadPool, @@ -80,6 +84,7 @@ final class WorkerPullWorkService extends AbstractExecutionThreadService { this.jobProcessor = jobProcessor; this.metricClient = metricClient; this.moreNewRequests = true; + this.clock = clock; this.stopwatchRegistry = stopwatchRegistry; this.stopwatchExporter = stopwatchExporter; this.nonBlockingThreadPool = nonBlockingThreadPool; @@ -92,10 +97,9 @@ protected void run() { logger.info("Aggregation worker started"); while (moreNewRequests) { - + Optional job = Optional.empty(); try { - Optional job = jobClient.getJob(); - + job = jobClient.getJob(); if (job.isEmpty()) { logger.info("No job pulled."); @@ -146,11 +150,15 @@ protected void run() { jobClient.markJobCompleted(jobResult); recordWorkerJobMetric(JOB_COMPLETION_METRIC_NAME, "Success"); + } catch (AggregationJobProcessException e) { + processException(e, jobClient, job.get()); + // TODO(b/197999001) report job with error in some monitoring counter + recordWorkerJobMetric(JOB_COMPLETION_METRIC_NAME, "Success"); } catch (Exception e) { recordWorkerJobMetric(JOB_ERROR_METRIC_NAME, "JobHandlingError"); - logger.error("Exception occurred in worker", e); + logger.error( + String.format("%s caught in WorkerPullWorkService: ", e.getClass().getSimpleName()), e); } - // Stopwatches only get exported once this loop exits. When run in benchmark mode (for perf // tests), we only expect one worker item. if (benchmarkMode) { @@ -188,4 +196,27 @@ private void recordWorkerJobMetric(String metricName, String type) { logger.warn(String.format("Could not record job metric %s.\n%s", metricName, e)); } } + + private JobResult createErrorJobResult(Job job, String errorCode, String errorMessage) { + return JobResult.builder() + .setJobKey(job.jobKey()) + .setResultInfo( + ResultInfo.newBuilder() + .setReturnMessage(errorMessage) + .setReturnCode(errorCode) + .setErrorSummary(ErrorSummary.getDefaultInstance()) + .setFinishedAt(ProtoUtil.toProtoTimestamp(Instant.now(clock))) + .build()) + .build(); + } + + private void processException(AggregationJobProcessException e, JobClient jobClient, Job job) { + logger.error("Exception while running job :", e); + JobResult jobResult = createErrorJobResult(job, e.getCode().name(), e.getMessage()); + try { + jobClient.markJobCompleted(jobResult); + } catch (JobClient.JobClientException ex) { + logger.error("Marking Job complete failed: ", ex.getMessage()); + } + } } diff --git a/java/com/google/aggregate/adtech/worker/aggregation/concurrent/BUILD b/java/com/google/aggregate/adtech/worker/aggregation/concurrent/BUILD index 425bc318..57c2bbfa 100644 --- a/java/com/google/aggregate/adtech/worker/aggregation/concurrent/BUILD +++ b/java/com/google/aggregate/adtech/worker/aggregation/concurrent/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( @@ -25,6 +27,7 @@ java_library( "//java/com/google/aggregate/adtech/worker/aggregation/engine", "//java/com/google/aggregate/adtech/worker/aggregation/privacy:privacy_budgeting_service_bridge", "//java/com/google/aggregate/adtech/worker/decryption", + "//java/com/google/aggregate/adtech/worker/exceptions", "//java/com/google/aggregate/adtech/worker/model", "//java/com/google/aggregate/adtech/worker/util", "//java/com/google/aggregate/adtech/worker/validation", @@ -43,6 +46,5 @@ java_library( "//java/external:scp_shared_proto", "//java/external:shared_model", "//java/external:slf4j", - "@com_google_adm_cloud_scp//java/com/google/scp/operator/shared/dao/metadatadb/model", ], ) diff --git a/java/com/google/aggregate/adtech/worker/aggregation/concurrent/ConcurrentAggregationProcessor.java b/java/com/google/aggregate/adtech/worker/aggregation/concurrent/ConcurrentAggregationProcessor.java index 298bdbda..19ed7b16 100644 --- a/java/com/google/aggregate/adtech/worker/aggregation/concurrent/ConcurrentAggregationProcessor.java +++ b/java/com/google/aggregate/adtech/worker/aggregation/concurrent/ConcurrentAggregationProcessor.java @@ -16,18 +16,18 @@ package com.google.aggregate.adtech.worker.aggregation.concurrent; +import static com.google.aggregate.adtech.worker.AggregationWorkerReturnCode.DOMAIN_PROCESS_EXCEPTION; +import static com.google.aggregate.adtech.worker.AggregationWorkerReturnCode.INPUT_DATA_READ_FAILED; import static com.google.aggregate.adtech.worker.AggregationWorkerReturnCode.INVALID_JOB; import static com.google.aggregate.adtech.worker.AggregationWorkerReturnCode.PERMISSION_ERROR; import static com.google.aggregate.adtech.worker.AggregationWorkerReturnCode.PRIVACY_BUDGET_ERROR; import static com.google.aggregate.adtech.worker.AggregationWorkerReturnCode.PRIVACY_BUDGET_EXHAUSTED; -import static com.google.common.base.Throwables.getStackTraceAsString; +import static com.google.aggregate.adtech.worker.AggregationWorkerReturnCode.RESULT_LOGGING_ERROR; +import static com.google.aggregate.adtech.worker.AggregationWorkerReturnCode.SUCCESS; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.util.concurrent.Futures.immediateFuture; import static com.google.common.util.concurrent.Futures.whenAllSucceed; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static com.google.scp.operator.protos.shared.backend.ReturnCodeProto.ReturnCode.INPUT_DATA_READ_FAILED; -import static com.google.scp.operator.protos.shared.backend.ReturnCodeProto.ReturnCode.OUTPUT_DATAWRITE_FAILED; -import static com.google.scp.operator.protos.shared.backend.ReturnCodeProto.ReturnCode.SUCCESS; import static com.google.scp.operator.shared.model.BackendModelUtil.toJobKeyString; import com.google.aggregate.adtech.worker.Annotations.BlockingThreadPool; @@ -37,13 +37,16 @@ import com.google.aggregate.adtech.worker.JobProcessor; import com.google.aggregate.adtech.worker.ReportDecrypterAndValidator; import com.google.aggregate.adtech.worker.ResultLogger; -import com.google.aggregate.adtech.worker.ResultLogger.ResultLogException; import com.google.aggregate.adtech.worker.aggregation.domain.OutputDomainProcessor; -import com.google.aggregate.adtech.worker.aggregation.domain.OutputDomainProcessor.DomainReadException; import com.google.aggregate.adtech.worker.aggregation.engine.AggregationEngine; import com.google.aggregate.adtech.worker.aggregation.privacy.PrivacyBudgetingServiceBridge; import com.google.aggregate.adtech.worker.aggregation.privacy.PrivacyBudgetingServiceBridge.PrivacyBudgetUnit; import com.google.aggregate.adtech.worker.aggregation.privacy.PrivacyBudgetingServiceBridge.PrivacyBudgetingServiceBridgeException; +import com.google.aggregate.adtech.worker.exceptions.AggregationJobProcessException; +import com.google.aggregate.adtech.worker.exceptions.ConcurrentShardReadException; +import com.google.aggregate.adtech.worker.exceptions.DomainProcessException; +import com.google.aggregate.adtech.worker.exceptions.DomainReadException; +import com.google.aggregate.adtech.worker.exceptions.ResultLogException; import com.google.aggregate.adtech.worker.model.AggregatedFact; import com.google.aggregate.adtech.worker.model.AvroRecordEncryptedReportConverter; import com.google.aggregate.adtech.worker.model.DebugBucketAnnotation; @@ -104,7 +107,6 @@ public final class ConcurrentAggregationProcessor implements JobProcessor { public static final String JOB_PARAM_ATTRIBUTION_REPORT_TO = "attribution_report_to"; public static final String JOB_PARAM_OUTPUT_DOMAIN_BLOB_PREFIX = "output_domain_blob_prefix"; public static final String JOB_PARAM_OUTPUT_DOMAIN_BUCKET_NAME = "output_domain_bucket_name"; - public static final String JOB_PARAM_DEBUG_PRIVACY_BUDGET_LIMIT = "debug_privacy_budget_limit"; // Key to indicate whether this is a debug job public static final String JOB_PARAM_DEBUG_RUN = "debug_run"; @@ -166,8 +168,19 @@ public final class ConcurrentAggregationProcessor implements JobProcessor { this.domainOptional = domainOptional; } + /** + * Processor responsible for performing aggregation. TODO: evaluate throwing unchecked exceptions + * here. + * + * @param job + * @return + * @throws AggregationJobProcessException + * @throws ExecutionException + * @throws InterruptedException + */ @Override - public JobResult process(Job job) throws AggregationJobProcessException { + public JobResult process(Job job) + throws ExecutionException, InterruptedException, AggregationJobProcessException { Stopwatch processingStopwatch = stopwatches.createStopwatch("concurrent-" + toJobKeyString(job.jobKey())); processingStopwatch.start(); @@ -176,18 +189,14 @@ public JobResult process(Job job) throws AggregationJobProcessException { final Boolean debugRun = DebugSupportHelper.isDebugRun(job); final Optional debugPrivacyEpsilon = getPrivacyEpsilonForJob(job); - try { - if (debugPrivacyEpsilon.isPresent()) { - Double privacyEpsilonForJob = debugPrivacyEpsilon.get(); - if (!(privacyEpsilonForJob > 0d && privacyEpsilonForJob <= 64d)) { - return handleInvalidEpsilon(jobResultBuilder); - } + + if (debugPrivacyEpsilon.isPresent()) { + Double privacyEpsilonForJob = debugPrivacyEpsilon.get(); + if (!(privacyEpsilonForJob > 0d && privacyEpsilonForJob <= 64d)) { + throw new AggregationJobProcessException( + INVALID_JOB, + String.format("Failed Parsing Job parameters for %s", JOB_PARAM_DEBUG_PRIVACY_EPSILON)); } - } catch (Exception e) { - // TODO cleanup exception handling - logger.error( - String.format("Failed Parsing Job parameters for %s", JOB_PARAM_DEBUG_PRIVACY_EPSILON), - e); } ListenableFuture> outputDomain; @@ -214,20 +223,22 @@ public JobResult process(Job job) throws AggregationJobProcessException { dataShards = findShards(reportsLocation); if (dataShards.isEmpty()) { - throw new ConcurrentShardReadException( - new IllegalArgumentException( - "No report shards found for location: " + reportsLocation)); + throw new AggregationJobProcessException( + INPUT_DATA_READ_FAILED, "No report shards found for location: " + reportsLocation); } outputDomain = outputDomainLocation .map(outputDomainProcessor::readAndDedupDomain) .orElse(immediateFuture(ImmutableSet.of())); - } catch (ConcurrentShardReadException | DomainReadException e) { - // Error occurred in data client from the main thread. - // TODO(b/197999001) report exception in some monitoring counter - return jobResultForDataReadException(jobResultBuilder, e); + } catch (ConcurrentShardReadException e) { + throw new AggregationJobProcessException( + INPUT_DATA_READ_FAILED, "Exception while reading reports input data.", e); + } catch (DomainReadException e) { + throw new AggregationJobProcessException( + INPUT_DATA_READ_FAILED, "Exception while reading domain input data.", e); } + try { // List of futures, one for each data shard; a shard is read into a list of encrypted reports. ImmutableList>> shardReads = @@ -300,7 +311,6 @@ public JobResult process(Job job) throws AggregationJobProcessException { // Create error summary from the list of errors from decryption/validation ErrorSummary errorSummary = ErrorSummaryAggregator.createErrorSummary(invalidReports); - Optional debugPrivacyBudgetLimit = getPrivacyBudgetLimitForJob(job); ImmutableList missingPrivacyBudgetUnits = ImmutableList.of(); // Do not consume any privacy budget for debug-run jobs. @@ -316,37 +326,16 @@ public JobResult process(Job job) throws AggregationJobProcessException { /* budgetsToConsume= */ budgetsToConsume, /* attributionReportTo= */ job.requestInfo() .getJobParameters() - .get(JOB_PARAM_ATTRIBUTION_REPORT_TO), - /* debugPrivacyBudgetLimit= */ debugPrivacyBudgetLimit); + .get(JOB_PARAM_ATTRIBUTION_REPORT_TO)); } } catch (PrivacyBudgetingServiceBridgeException e) { - // Only fail the job if the worker is configured as such. At times during origin trial the - // job won't fail if there is an error reaching PBS (but still fails if budget is - // exhausted). This is done to lessen the impact of any instability in the privacy budget - // service. - return jobResultBuilder - .setResultInfo( - ResultInfo.newBuilder() - .setReturnCode(PRIVACY_BUDGET_ERROR.name()) - .setReturnMessage( - "Exception while consuming privacy budget: " + getStackTraceAsString(e)) - .setErrorSummary(ErrorSummary.getDefaultInstance()) - .setFinishedAt(ProtoUtil.toProtoTimestamp(Instant.now(clock))) - .build()) - .build(); + throw new AggregationJobProcessException( + PRIVACY_BUDGET_ERROR, "Exception while consuming privacy budget.", e); } if (!missingPrivacyBudgetUnits.isEmpty()) { - // Truncate the message in order to not overflow the result table. - return jobResultBuilder - .setResultInfo( - ResultInfo.newBuilder() - .setReturnCode(PRIVACY_BUDGET_EXHAUSTED.name()) - .setReturnMessage(PRIVACY_BUDGET_EXHAUSTED_ERROR_MESSAGE) - .setErrorSummary(ErrorSummary.getDefaultInstance()) - .setFinishedAt(ProtoUtil.toProtoTimestamp(Instant.now(clock))) - .build()) - .build(); + throw new AggregationJobProcessException( + PRIVACY_BUDGET_EXHAUSTED, PRIVACY_BUDGET_EXHAUSTED_ERROR_MESSAGE); } } @@ -371,66 +360,36 @@ public JobResult process(Job job) throws AggregationJobProcessException { .build()) .build(); } catch (ResultLogException e) { - // Error occurred in data write - // TODO(b/197999001) report exception in some monitoring counter - logger.error("Exception occurred during result data write. Reporting processing failure.", e); - return jobResultBuilder - .setResultInfo( - ResultInfo.newBuilder() - .setReturnCode(OUTPUT_DATAWRITE_FAILED.name()) - // TODO see if there's a better error message than the stack trace - .setReturnMessage(getStackTraceAsString(e)) - .setErrorSummary(ErrorSummary.getDefaultInstance()) - .setFinishedAt(ProtoUtil.toProtoTimestamp(Instant.now(clock))) - .build()) - .build(); + throw new AggregationJobProcessException( + RESULT_LOGGING_ERROR, "Result Logging exception.", e); + } catch (DomainProcessException e) { + throw new AggregationJobProcessException( + DOMAIN_PROCESS_EXCEPTION, "Exception in processing domain.", e); } catch (AccessControlException e) { - return handlePermissionException(e, jobResultBuilder); + throw new AggregationJobProcessException( + PERMISSION_ERROR, "Exception because of missing permission.", e); + } catch (ExecutionException e) { - if ((e.getCause() instanceof ConcurrentShardReadException) - || (e.getCause() instanceof DomainReadException)) { - // Error occurred in data read - // TODO(b/197999001) report exception in some monitoring counter - return jobResultForDataReadException(jobResultBuilder, e.getCause()); + // Error occurred in data read + // TODO(b/197999001) report exception in some monitoring counter + if ((e.getCause() instanceof ConcurrentShardReadException)) { + throw new AggregationJobProcessException( + INPUT_DATA_READ_FAILED, "Exception while reading reports input data.", e.getCause()); + } + if ((e.getCause() instanceof DomainReadException)) { + throw new AggregationJobProcessException( + INPUT_DATA_READ_FAILED, "Exception while reading domain input data.", e.getCause()); } + if (e.getCause() instanceof AccessControlException) { - return handlePermissionException((AccessControlException) e.getCause(), jobResultBuilder); + throw new AggregationJobProcessException( + PERMISSION_ERROR, "Exception because of missing permission.", e.getCause()); } - throw new AggregationJobProcessException(e); - } catch (InterruptedException e) { - throw new AggregationJobProcessException(e); + throw e; } } - private JobResult handleInvalidEpsilon(JobResult.Builder jobResultBuilder) { - return jobResultBuilder - .setResultInfo( - ResultInfo.newBuilder() - .setReturnCode(INVALID_JOB.name()) - .setReturnMessage( - String.format("%s should be > 0 and <= 64", JOB_PARAM_DEBUG_PRIVACY_EPSILON)) - .setErrorSummary(ErrorSummary.getDefaultInstance()) - .setFinishedAt(ProtoUtil.toProtoTimestamp(Instant.now(clock))) - .build()) - .build(); - } - - private JobResult handlePermissionException( - AccessControlException e, JobResult.Builder jobResultBuilder) { - logger.error("Exception occurred due to permission issues: " + e.getMessage()); - return jobResultBuilder - .setResultInfo( - ResultInfo.newBuilder() - .setReturnCode(PERMISSION_ERROR.name()) - // TODO see if there's a better error message than the stack trace - .setReturnMessage(getStackTraceAsString(e)) - .setErrorSummary(ErrorSummary.getDefaultInstance()) - .setFinishedAt(ProtoUtil.toProtoTimestamp(Instant.now(clock))) - .build()) - .build(); - } - private ImmutableList findShards(DataLocation reportsLocation) { try { ImmutableList shardBlobs = blobStorageClient.listBlobs(reportsLocation); @@ -630,31 +589,6 @@ private Void aggregateShard( return null; } - private JobResult jobResultForDataReadException(JobResult.Builder jobResultBuilder, Throwable e) { - logger.error("Exception occurred during input data read. Reporting processing failure.", e); - return jobResultBuilder - .setResultInfo( - ResultInfo.newBuilder() - .setReturnCode(INPUT_DATA_READ_FAILED.name()) - // TODO see if there's a better error message than the stack trace - .setReturnMessage(getStackTraceAsString(e)) - .setErrorSummary(ErrorSummary.getDefaultInstance()) - .setFinishedAt(ProtoUtil.toProtoTimestamp(Instant.now(clock))) - .build()) - .build(); - } - - /** Retrieve limit from nested optional fields */ - private Optional getPrivacyBudgetLimitForJob(Job job) { - if (job.requestInfo().getJobParameters().containsKey(JOB_PARAM_DEBUG_PRIVACY_BUDGET_LIMIT)) { - return Optional.of( - Integer.parseInt( - job.requestInfo().getJobParameters().get(JOB_PARAM_DEBUG_PRIVACY_BUDGET_LIMIT))); - } else { - return Optional.empty(); - } - } - /** Retrieve epsilon from nested optional fields */ private Optional getPrivacyEpsilonForJob(Job job) { Optional epsilonValueFromJobReq = Optional.empty(); @@ -672,18 +606,4 @@ private Optional getPrivacyEpsilonForJob(Job job) { } return epsilonValueFromJobReq; } - - static final class DomainProcessException extends RuntimeException { - - DomainProcessException(Throwable cause) { - super(cause); - } - } - - static final class ConcurrentShardReadException extends RuntimeException { - - ConcurrentShardReadException(Throwable cause) { - super(cause); - } - } } diff --git a/java/com/google/aggregate/adtech/worker/aggregation/domain/AvroOutputDomainProcessor.java b/java/com/google/aggregate/adtech/worker/aggregation/domain/AvroOutputDomainProcessor.java index e41bd559..b1a61248 100644 --- a/java/com/google/aggregate/adtech/worker/aggregation/domain/AvroOutputDomainProcessor.java +++ b/java/com/google/aggregate/adtech/worker/aggregation/domain/AvroOutputDomainProcessor.java @@ -20,6 +20,7 @@ import com.google.aggregate.adtech.worker.Annotations.BlockingThreadPool; import com.google.aggregate.adtech.worker.Annotations.NonBlockingThreadPool; +import com.google.aggregate.adtech.worker.exceptions.DomainReadException; import com.google.aggregate.perf.StopwatchRegistry; import com.google.aggregate.protocol.avro.AvroOutputDomainReader; import com.google.aggregate.protocol.avro.AvroOutputDomainReaderFactory; diff --git a/java/com/google/aggregate/adtech/worker/aggregation/domain/BUILD b/java/com/google/aggregate/adtech/worker/aggregation/domain/BUILD index a1954456..e837ec44 100644 --- a/java/com/google/aggregate/adtech/worker/aggregation/domain/BUILD +++ b/java/com/google/aggregate/adtech/worker/aggregation/domain/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( @@ -20,6 +22,7 @@ java_library( "OutputDomainProcessor.java", ], deps = [ + "//java/com/google/aggregate/adtech/worker/exceptions", "//java/com/google/aggregate/adtech/worker/util", "//java/com/google/aggregate/perf", "//java/external:clients_blobstorageclient", @@ -35,6 +38,7 @@ java_library( deps = [ ":domain", "//java/com/google/aggregate/adtech/worker", + "//java/com/google/aggregate/adtech/worker/exceptions", "//java/com/google/aggregate/adtech/worker/util", "//java/com/google/aggregate/perf", "//java/external:clients_blobstorageclient", @@ -52,6 +56,7 @@ java_library( deps = [ ":domain", "//java/com/google/aggregate/adtech/worker", + "//java/com/google/aggregate/adtech/worker/exceptions", "//java/com/google/aggregate/perf", "//java/com/google/aggregate/protocol/avro:avro_output_domain", "//java/com/google/aggregate/protocol/avro:avro_record_reader", diff --git a/java/com/google/aggregate/adtech/worker/aggregation/domain/OutputDomainProcessor.java b/java/com/google/aggregate/adtech/worker/aggregation/domain/OutputDomainProcessor.java index 6a475ab3..f0dfcc64 100644 --- a/java/com/google/aggregate/adtech/worker/aggregation/domain/OutputDomainProcessor.java +++ b/java/com/google/aggregate/adtech/worker/aggregation/domain/OutputDomainProcessor.java @@ -18,6 +18,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; +import com.google.aggregate.adtech.worker.exceptions.DomainReadException; import com.google.aggregate.perf.StopwatchRegistry; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; @@ -133,11 +134,4 @@ private ImmutableList listShards(DataLocation outputDomainLocation * @return the contents of the shard as a {@link ImmutableList} */ protected abstract ImmutableList readShard(DataLocation shardLocation); - - public final class DomainReadException extends RuntimeException { - - DomainReadException(Throwable cause) { - super(cause); - } - } } diff --git a/java/com/google/aggregate/adtech/worker/aggregation/domain/TextOutputDomainProcessor.java b/java/com/google/aggregate/adtech/worker/aggregation/domain/TextOutputDomainProcessor.java index 642e6d33..a2121702 100644 --- a/java/com/google/aggregate/adtech/worker/aggregation/domain/TextOutputDomainProcessor.java +++ b/java/com/google/aggregate/adtech/worker/aggregation/domain/TextOutputDomainProcessor.java @@ -21,6 +21,7 @@ import com.google.aggregate.adtech.worker.Annotations.BlockingThreadPool; import com.google.aggregate.adtech.worker.Annotations.NonBlockingThreadPool; +import com.google.aggregate.adtech.worker.exceptions.DomainReadException; import com.google.aggregate.adtech.worker.util.NumericConversions; import com.google.aggregate.perf.StopwatchRegistry; import com.google.common.base.Stopwatch; diff --git a/java/com/google/aggregate/adtech/worker/aggregation/engine/BUILD b/java/com/google/aggregate/adtech/worker/aggregation/engine/BUILD index b1cfdbe2..c64f5bf2 100644 --- a/java/com/google/aggregate/adtech/worker/aggregation/engine/BUILD +++ b/java/com/google/aggregate/adtech/worker/aggregation/engine/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/adtech/worker/aggregation/privacy/BUILD b/java/com/google/aggregate/adtech/worker/aggregation/privacy/BUILD index 7a772199..9189ddd0 100644 --- a/java/com/google/aggregate/adtech/worker/aggregation/privacy/BUILD +++ b/java/com/google/aggregate/adtech/worker/aggregation/privacy/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/adtech/worker/aggregation/privacy/FakePrivacyBudgetingServiceBridge.java b/java/com/google/aggregate/adtech/worker/aggregation/privacy/FakePrivacyBudgetingServiceBridge.java index cb7b70ad..accd7bff 100644 --- a/java/com/google/aggregate/adtech/worker/aggregation/privacy/FakePrivacyBudgetingServiceBridge.java +++ b/java/com/google/aggregate/adtech/worker/aggregation/privacy/FakePrivacyBudgetingServiceBridge.java @@ -34,7 +34,6 @@ public final class FakePrivacyBudgetingServiceBridge implements PrivacyBudgeting private boolean shouldThrow; private Optional> lastBudgetsToConsumeSent = Optional.empty(); private Optional lastAttributionReportToSent = Optional.empty(); - private Optional lastDebugPrivacyBudgetLimitSent = Optional.empty(); private final HashMap privacyBudgets; @@ -53,17 +52,14 @@ public void setShouldThrow() { @Override public ImmutableList consumePrivacyBudget( - ImmutableList budgetsToConsume, - String attributionReportTo, - Optional debugPrivacyBudgetLimit) + ImmutableList budgetsToConsume, String attributionReportTo) throws PrivacyBudgetingServiceBridgeException { if (shouldThrow) { - throw new PrivacyBudgetingServiceBridgeException(new IllegalStateException("Set to throw")); + throw new PrivacyBudgetingServiceBridgeException(); } lastBudgetsToConsumeSent = Optional.of(budgetsToConsume); lastAttributionReportToSent = Optional.of(attributionReportTo); - lastDebugPrivacyBudgetLimitSent = debugPrivacyBudgetLimit; ImmutableList insufficientPrivacyBudgetUnits = budgetsToConsume.stream() @@ -85,10 +81,6 @@ public Optional getLastAttributionReportToSent() { return lastAttributionReportToSent; } - public Optional getLastDebugPrivacyBudgetLimitSent() { - return lastDebugPrivacyBudgetLimitSent; - } - public Optional> getLastBudgetsToConsumeSent() { return lastBudgetsToConsumeSent; } diff --git a/java/com/google/aggregate/adtech/worker/aggregation/privacy/HttpPrivacyBudgetingServiceBridge.java b/java/com/google/aggregate/adtech/worker/aggregation/privacy/HttpPrivacyBudgetingServiceBridge.java index 3459c31b..139c15d4 100644 --- a/java/com/google/aggregate/adtech/worker/aggregation/privacy/HttpPrivacyBudgetingServiceBridge.java +++ b/java/com/google/aggregate/adtech/worker/aggregation/privacy/HttpPrivacyBudgetingServiceBridge.java @@ -24,7 +24,6 @@ import com.google.scp.operator.cpio.distributedprivacybudgetclient.DistributedPrivacyBudgetClient; import com.google.scp.operator.cpio.distributedprivacybudgetclient.DistributedPrivacyBudgetClient.DistributedPrivacyBudgetClientException; import com.google.scp.operator.cpio.distributedprivacybudgetclient.DistributedPrivacyBudgetClient.DistributedPrivacyBudgetServiceException; -import java.util.Optional; import javax.inject.Inject; /** HTTP privacy budgeting bridge which consumes privacy budget from an external HTTP service. */ @@ -42,9 +41,7 @@ public HttpPrivacyBudgetingServiceBridge( @Override public ImmutableList consumePrivacyBudget( - ImmutableList budgetsToConsume, - String attributionReportTo, - Optional debugPrivacyBudgetLimit) + ImmutableList budgetsToConsume, String attributionReportTo) throws PrivacyBudgetingServiceBridgeException { ConsumePrivacyBudgetRequest consumePrivacyBudgetRequest = ConsumePrivacyBudgetRequest.builder() @@ -53,7 +50,7 @@ public ImmutableList consumePrivacyBudget( budgetsToConsume.stream() .map(HttpPrivacyBudgetingServiceBridge::scpBudgetUnit) .collect(toImmutableList())) - .privacyBudgetLimit(debugPrivacyBudgetLimit.orElse(DEFAULT_PRIVACY_BUDGET_LIMIT)) + .privacyBudgetLimit(DEFAULT_PRIVACY_BUDGET_LIMIT) .build(); try { diff --git a/java/com/google/aggregate/adtech/worker/aggregation/privacy/PrivacyBudgetingServiceBridge.java b/java/com/google/aggregate/adtech/worker/aggregation/privacy/PrivacyBudgetingServiceBridge.java index 0b40efe5..fe9cff39 100644 --- a/java/com/google/aggregate/adtech/worker/aggregation/privacy/PrivacyBudgetingServiceBridge.java +++ b/java/com/google/aggregate/adtech/worker/aggregation/privacy/PrivacyBudgetingServiceBridge.java @@ -19,7 +19,6 @@ import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableList; import java.time.Instant; -import java.util.Optional; /** Interface for consuming privacy budgeting. */ public interface PrivacyBudgetingServiceBridge { @@ -33,9 +32,7 @@ public interface PrivacyBudgetingServiceBridge { * @return First few privacy budgeting units for which budget consumption failed. */ ImmutableList consumePrivacyBudget( - ImmutableList budgetsToConsume, - String attributionReportTo, - Optional debugPrivacyBudgetLimit) + ImmutableList budgetsToConsume, String attributionReportTo) throws PrivacyBudgetingServiceBridgeException; /** Identifier for an individual key of the privacy budget to be consumed. */ @@ -55,6 +52,10 @@ public static PrivacyBudgetUnit create(String privacyBudgetKey, Instant schedule /** Exception that may happen when consuming the privacy budget. */ final class PrivacyBudgetingServiceBridgeException extends Exception { + public PrivacyBudgetingServiceBridgeException() { + super(); + } + public PrivacyBudgetingServiceBridgeException(Throwable cause) { super(cause); } diff --git a/java/com/google/aggregate/adtech/worker/aggregation/privacy/UnlimitedPrivacyBudgetingServiceBridge.java b/java/com/google/aggregate/adtech/worker/aggregation/privacy/UnlimitedPrivacyBudgetingServiceBridge.java index 04c85885..91e6d476 100644 --- a/java/com/google/aggregate/adtech/worker/aggregation/privacy/UnlimitedPrivacyBudgetingServiceBridge.java +++ b/java/com/google/aggregate/adtech/worker/aggregation/privacy/UnlimitedPrivacyBudgetingServiceBridge.java @@ -17,7 +17,6 @@ package com.google.aggregate.adtech.worker.aggregation.privacy; import com.google.common.collect.ImmutableList; -import java.util.Optional; /** * Privacy budgeting service bridge with unlimited budget @@ -29,9 +28,7 @@ public final class UnlimitedPrivacyBudgetingServiceBridge implements PrivacyBudg @Override public ImmutableList consumePrivacyBudget( - ImmutableList budgetsToConsume, - String attributionReportTo, - Optional debugPrivacyBudgetLimit) { + ImmutableList budgetsToConsume, String attributionReportTo) { return ImmutableList.of(); } } diff --git a/java/com/google/aggregate/adtech/worker/configs/BUILD b/java/com/google/aggregate/adtech/worker/configs/BUILD index a02a472c..c079d354 100644 --- a/java/com/google/aggregate/adtech/worker/configs/BUILD +++ b/java/com/google/aggregate/adtech/worker/configs/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/adtech/worker/decryption/BUILD b/java/com/google/aggregate/adtech/worker/decryption/BUILD index 147a821d..e6f842cb 100644 --- a/java/com/google/aggregate/adtech/worker/decryption/BUILD +++ b/java/com/google/aggregate/adtech/worker/decryption/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/adtech/worker/decryption/hybrid/BUILD b/java/com/google/aggregate/adtech/worker/decryption/hybrid/BUILD index 388e46c4..d15784a8 100644 --- a/java/com/google/aggregate/adtech/worker/decryption/hybrid/BUILD +++ b/java/com/google/aggregate/adtech/worker/decryption/hybrid/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/adtech/worker/decryption/noop/BUILD b/java/com/google/aggregate/adtech/worker/decryption/noop/BUILD index c1acd3a2..80f69f7d 100644 --- a/java/com/google/aggregate/adtech/worker/decryption/noop/BUILD +++ b/java/com/google/aggregate/adtech/worker/decryption/noop/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/adtech/worker/exceptions/AggregationJobProcessException.java b/java/com/google/aggregate/adtech/worker/exceptions/AggregationJobProcessException.java new file mode 100644 index 00000000..be5714eb --- /dev/null +++ b/java/com/google/aggregate/adtech/worker/exceptions/AggregationJobProcessException.java @@ -0,0 +1,51 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.aggregate.adtech.worker.exceptions; + +import com.google.aggregate.adtech.worker.AggregationWorkerReturnCode; + +/** Exception to wrap Aggregation. */ +public class AggregationJobProcessException extends Exception { + + private AggregationWorkerReturnCode code; + + /** Builds a new AggregationJobProcessException with cause. */ + public AggregationJobProcessException(Throwable cause) { + super(cause); + } + + /** Builds a new AggregationJobProcessException with error code string and specified message. */ + public AggregationJobProcessException(AggregationWorkerReturnCode code, String message) { + super(message); + this.setCode(code); + } + + /** Builds a new AggregationJobProcessException with error code, cause and message. */ + public AggregationJobProcessException( + AggregationWorkerReturnCode code, String message, Throwable cause) { + super(message, cause); + this.setCode(code); + } + + public AggregationWorkerReturnCode getCode() { + return code; + } + + public void setCode(AggregationWorkerReturnCode code) { + this.code = code; + } +} diff --git a/java/com/google/aggregate/adtech/worker/exceptions/BUILD b/java/com/google/aggregate/adtech/worker/exceptions/BUILD new file mode 100644 index 00000000..eb677dd7 --- /dev/null +++ b/java/com/google/aggregate/adtech/worker/exceptions/BUILD @@ -0,0 +1,31 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_java//java:defs.bzl", "java_library") + +package(default_visibility = ["//visibility:public"]) + +java_library( + name = "exceptions", + srcs = [ + "AggregationJobProcessException.java", + "ConcurrentShardReadException.java", + "DomainProcessException.java", + "DomainReadException.java", + "ResultLogException.java", + ], + deps = [ + "//java/com/google/aggregate/adtech/worker:return_code", + ], +) diff --git a/java/com/google/aggregate/adtech/worker/exceptions/ConcurrentShardReadException.java b/java/com/google/aggregate/adtech/worker/exceptions/ConcurrentShardReadException.java new file mode 100644 index 00000000..dadda145 --- /dev/null +++ b/java/com/google/aggregate/adtech/worker/exceptions/ConcurrentShardReadException.java @@ -0,0 +1,26 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.aggregate.adtech.worker.exceptions; + +/** Exception caused when concurrent shard read fails. */ +public class ConcurrentShardReadException extends RuntimeException { + + /** Builds a new ConcurrentShardReadException with cause. */ + public ConcurrentShardReadException(Throwable cause) { + super(cause); + } +} diff --git a/java/com/google/aggregate/adtech/worker/exceptions/DomainProcessException.java b/java/com/google/aggregate/adtech/worker/exceptions/DomainProcessException.java new file mode 100644 index 00000000..f45ce7ba --- /dev/null +++ b/java/com/google/aggregate/adtech/worker/exceptions/DomainProcessException.java @@ -0,0 +1,26 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.aggregate.adtech.worker.exceptions; + +/** Exception caused when Domain processing fails. */ +public class DomainProcessException extends RuntimeException { + + /** Builds a new DomainProcessException with cause. */ + public DomainProcessException(Throwable cause) { + super(cause); + } +} diff --git a/java/com/google/aggregate/adtech/worker/exceptions/DomainReadException.java b/java/com/google/aggregate/adtech/worker/exceptions/DomainReadException.java new file mode 100644 index 00000000..6f104369 --- /dev/null +++ b/java/com/google/aggregate/adtech/worker/exceptions/DomainReadException.java @@ -0,0 +1,26 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.aggregate.adtech.worker.exceptions; + +/** Exception caused when Domain read fails. */ +public class DomainReadException extends RuntimeException { + + /** Builds a new DomainReadException with cause. */ + public DomainReadException(Throwable cause) { + super(cause); + } +} diff --git a/java/com/google/aggregate/adtech/worker/exceptions/ResultLogException.java b/java/com/google/aggregate/adtech/worker/exceptions/ResultLogException.java new file mode 100644 index 00000000..253f1902 --- /dev/null +++ b/java/com/google/aggregate/adtech/worker/exceptions/ResultLogException.java @@ -0,0 +1,26 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.aggregate.adtech.worker.exceptions; + +/** Exception caused when result logging fails. */ +public class ResultLogException extends RuntimeException { + + /** Builds a new ResultLogException with cause. */ + public ResultLogException(Throwable cause) { + super(cause); + } +} diff --git a/java/com/google/aggregate/adtech/worker/local/BUILD b/java/com/google/aggregate/adtech/worker/local/BUILD index b0ae7109..731cddd1 100644 --- a/java/com/google/aggregate/adtech/worker/local/BUILD +++ b/java/com/google/aggregate/adtech/worker/local/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/adtech/worker/model/BUILD b/java/com/google/aggregate/adtech/worker/model/BUILD index 143bdd11..47057c1b 100644 --- a/java/com/google/aggregate/adtech/worker/model/BUILD +++ b/java/com/google/aggregate/adtech/worker/model/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( @@ -37,7 +39,6 @@ java_library( "//java/external:autovalue", "//java/external:autovalue_annotations", "//java/external:clients_blobstorageclient_model", - "//java/external:dao_metadatadb_model", "//java/external:guava", "//java/external:jackson_annotations", "//java/external:jackson_core", diff --git a/java/com/google/aggregate/adtech/worker/model/SharedInfo.java b/java/com/google/aggregate/adtech/worker/model/SharedInfo.java index aa768451..b503e7a5 100644 --- a/java/com/google/aggregate/adtech/worker/model/SharedInfo.java +++ b/java/com/google/aggregate/adtech/worker/model/SharedInfo.java @@ -46,6 +46,7 @@ public abstract class SharedInfo { public static final String VERSION_0_1 = "0.1"; public static final String LATEST_VERSION = VERSION_0_1; public static final boolean DEFAULT_DEBUG_MODE = false; + public static final String PRIVACY_BUDGET_KEY_DELIMITER = "-"; public static Builder builder() { return new AutoValue_SharedInfo.Builder() @@ -86,13 +87,14 @@ public static Builder builder() { public abstract Optional reportId(); // String Debug mode value for writing json. + @JsonInclude(JsonInclude.Include.NON_EMPTY) @JsonProperty("debug_mode") - public abstract String reportDebugModeString(); + public abstract Optional reportDebugModeString(); // Convert the debugMode string field to boolean. @JsonIgnore public final boolean getReportDebugMode() { - return reportDebugModeString().equals("enabled"); + return reportDebugModeString().isPresent() && reportDebugModeString().get().equals("enabled"); } @AutoValue.Builder @@ -134,15 +136,15 @@ public static Builder builder() { public abstract Builder setReportDebugModeString(String value); /** - * Use boolean values for debug mode in the program and convert it to string values (enabled and - * disabled) for result json files. + * Use boolean values for debug mode in the program and convert it to string value enabled for + * result json files. */ @JsonIgnore public final Builder setReportDebugMode(Boolean value) { if (value) { return setReportDebugModeString("enabled"); } - return setReportDebugModeString("disabled"); + return this; } public abstract SharedInfo build(); // not public @@ -156,9 +158,13 @@ public String getPrivacyBudgetKey() { if (version().equals(VERSION_0_1)) { String privacyBudgetKeyHashInput = api().get() + + PRIVACY_BUDGET_KEY_DELIMITER + version() + + PRIVACY_BUDGET_KEY_DELIMITER + reportingOrigin() + + PRIVACY_BUDGET_KEY_DELIMITER + destination().get() + + PRIVACY_BUDGET_KEY_DELIMITER + sourceRegistrationTime().get(); return Hashing.sha256() .newHasher() diff --git a/java/com/google/aggregate/adtech/worker/model/serdes/BUILD b/java/com/google/aggregate/adtech/worker/model/serdes/BUILD index a23d6dd4..7b14f4b9 100644 --- a/java/com/google/aggregate/adtech/worker/model/serdes/BUILD +++ b/java/com/google/aggregate/adtech/worker/model/serdes/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/adtech/worker/model/serdes/cbor/BUILD b/java/com/google/aggregate/adtech/worker/model/serdes/cbor/BUILD index fd779ae5..a1f1ee32 100644 --- a/java/com/google/aggregate/adtech/worker/model/serdes/cbor/BUILD +++ b/java/com/google/aggregate/adtech/worker/model/serdes/cbor/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/adtech/worker/reader/avro/BUILD b/java/com/google/aggregate/adtech/worker/reader/avro/BUILD index 43d1381b..7a3a4cd9 100644 --- a/java/com/google/aggregate/adtech/worker/reader/avro/BUILD +++ b/java/com/google/aggregate/adtech/worker/reader/avro/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/adtech/worker/selector/BUILD b/java/com/google/aggregate/adtech/worker/selector/BUILD index 47455fd5..e68bd0bf 100644 --- a/java/com/google/aggregate/adtech/worker/selector/BUILD +++ b/java/com/google/aggregate/adtech/worker/selector/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/adtech/worker/testing/BUILD b/java/com/google/aggregate/adtech/worker/testing/BUILD index 1ba6504f..861ca582 100644 --- a/java/com/google/aggregate/adtech/worker/testing/BUILD +++ b/java/com/google/aggregate/adtech/worker/testing/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( @@ -35,6 +37,7 @@ java_library( srcs = ["NoopJobProcessor.java"], deps = [ "//java/com/google/aggregate/adtech/worker", + "//java/com/google/aggregate/adtech/worker/exceptions", "//java/com/google/aggregate/adtech/worker/model", "//java/external:clients_jobclient_model", ], @@ -117,6 +120,7 @@ java_library( ], deps = [ "//java/com/google/aggregate/adtech/worker", + "//java/com/google/aggregate/adtech/worker/exceptions", "//java/com/google/aggregate/adtech/worker/model", "//java/external:clients_blobstorageclient_model", "//java/external:clients_jobclient_model", @@ -152,6 +156,7 @@ java_library( deps = [ "//java/com/google/aggregate/adtech/worker", "//java/com/google/aggregate/adtech/worker:worker_runner", + "//java/com/google/aggregate/adtech/worker/exceptions", "//java/com/google/aggregate/adtech/worker/model", "//java/com/google/aggregate/adtech/worker/testing:in_memory_logger", "//java/external:guava", diff --git a/java/com/google/aggregate/adtech/worker/testing/FakeRecordDecrypter.java b/java/com/google/aggregate/adtech/worker/testing/FakeRecordDecrypter.java index 29ef2299..752cd341 100644 --- a/java/com/google/aggregate/adtech/worker/testing/FakeRecordDecrypter.java +++ b/java/com/google/aggregate/adtech/worker/testing/FakeRecordDecrypter.java @@ -16,6 +16,8 @@ package com.google.aggregate.adtech.worker.testing; +import static com.google.aggregate.adtech.worker.model.SharedInfo.DEFAULT_VERSION; + import com.google.aggregate.adtech.worker.decryption.RecordDecrypter; import com.google.aggregate.adtech.worker.model.EncryptedReport; import com.google.aggregate.adtech.worker.model.Report; @@ -44,7 +46,7 @@ public Report decryptSingleReport(EncryptedReport unused) throws DecryptionExcep throw new DecryptionException(new IllegalStateException("The decrypter was set to throw.")); } - return FakeReportGenerator.generate(idToGenerate); + return FakeReportGenerator.generateWithParam(idToGenerate, DEFAULT_VERSION); } public void setShouldThrow(boolean shouldThrowOnRead) { diff --git a/java/com/google/aggregate/adtech/worker/testing/FakeReportGenerator.java b/java/com/google/aggregate/adtech/worker/testing/FakeReportGenerator.java index 3493393a..399941a5 100644 --- a/java/com/google/aggregate/adtech/worker/testing/FakeReportGenerator.java +++ b/java/com/google/aggregate/adtech/worker/testing/FakeReportGenerator.java @@ -16,6 +16,7 @@ package com.google.aggregate.adtech.worker.testing; +import static com.google.aggregate.adtech.worker.model.SharedInfo.DEFAULT_VERSION; import static com.google.aggregate.adtech.worker.util.NumericConversions.createBucketFromInt; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.time.temporal.ChronoUnit.SECONDS; @@ -26,6 +27,7 @@ import com.google.aggregate.adtech.worker.model.SharedInfo; import com.google.common.collect.ImmutableList; import java.time.Instant; +import java.util.Optional; import java.util.UUID; import java.util.stream.IntStream; @@ -40,88 +42,148 @@ public static Fact generate(int bucket, int value) { } /** - * Given an id, returns the following Report(PrivacyBudgetKey: "1", Destination: "dummy", - * ReportingOrigin: "dummy", ScheduledReportTime: "1970-01-01 00:00:01", SourceRegistrationTime: - * "1970-01-01 00:00:01", ReportId: {UUID} Facts: facts + * Generates a Fake Report of the following format: + * + *

Payload: facts (Default Version) SharedInfo: reportingOrigin: "dummy", reportId: random + * UUID, privacyBudgetKey: "dummy", destination: "dummy", scheduledReportTime: 1 sec past epoch, + * sourceRegistrationTime: 1 sec past epoch + * + *

(Latest Version) SharedInfo: reportId: auto-generated from the following values, version: + * "0.1", api: "attribution-reporting", destination: "dummy", scheduledReportTime: 1 sec past + * epoch, sourceRegistrationTime: 1 sec past epoch + * + * @param facts Facts to populate the Report's Payload with + * @param reportVersion Version of the report to generate. Empty String for default version, + * anything else for latest. + * @return Fake Report */ - public static Report generate(ImmutableList facts) { - return Report.builder() - .setSharedInfo( - SharedInfo.builder() - .setPrivacyBudgetKey(String.valueOf("dummy")) - .setDestination("dummy") - .setReportingOrigin("dummy") - .setScheduledReportTime(Instant.EPOCH.plus(1, SECONDS)) - .setSourceRegistrationTime(Instant.EPOCH.plus(1, SECONDS)) - .setReportId(String.valueOf(UUID.randomUUID())) - .build()) - .setPayload(Payload.builder().addAllFact(facts).build()) - .build(); + public static Report generateWithFactList(ImmutableList facts, String reportVersion) { + return generate( + Optional.of(facts), + /* dummyValue */ Optional.empty(), + /* reportId */ Optional.empty(), + reportVersion); } /** - * Given an int param, returns the following if param=1, Report(PrivacyBudgetKey: "1", - * Destination: "1", ReportingOrigin: "1", ScheduledReportTime: "1970-01-01 00:00:01", - * SourceRegistrationTime: "1970-01-01 00:00:01", ReportId: {UUID} Facts: [Fact("1", 1)]) - * containing param number of Facts. + * Generates a Fake Report of the following format: + * + *

Payload: dummyValue number of Facts, each with bucket: dummyValue, value: dummyValue + * (Default Version) SharedInfo: reportingOrigin: dummyValue, reportId: random UUID, + * privacyBudgetKey: dummyValue, destination: dummyValue, scheduledReportTime: dummyValue sec past + * epoch, sourceRegistrationTime: dummyValue sec past epoch + * + *

(Latest Version) SharedInfo: reportId: auto-generated from the following values, version: + * "0.1", api: "attribution-reporting", destination: dummyValue, scheduledReportTime: dummyValue + * sec past epoch, sourceRegistrationTime: dummyValue sec past epoch + * + * @param dummyValue a dummy integer value that is set as various applicable values in the + * returned Report. See above for where it is specifically used. + * @param reportVersion Version of the report to generate. Empty String for default version, + * anything else for latest. + * @return */ - public static Report generate(int param) { - return Report.builder() - .setSharedInfo( - SharedInfo.builder() - .setPrivacyBudgetKey(String.valueOf(param)) - .setDestination(String.valueOf(param)) - .setReportingOrigin(String.valueOf(param)) - .setScheduledReportTime(Instant.EPOCH.plus(param, SECONDS)) - .setSourceRegistrationTime(Instant.EPOCH.plus(param, SECONDS)) - .setReportId(String.valueOf(UUID.randomUUID())) - .build()) - .setPayload( - Payload.builder() - .addAllFact( - IntStream.range(0, param) - .mapToObj( - i -> - Fact.builder() - .setBucket(createBucketFromInt(param)) - .setValue(param) - .build()) - .collect(toImmutableList())) - .build()) - .build(); + public static Report generateWithParam(int dummyValue, String reportVersion) { + return generate( + /* facts */ Optional.empty(), + Optional.of(dummyValue), + /* reportId */ Optional.empty(), + reportVersion); } + + /** + * Generates a Fake Report of the following format: + * + *

Payload: dummyValue number of Facts, each with bucket: dummyValue, value: dummyValue + * (Default Version) SharedInfo: reportingOrigin: dummyValue, reportId: reportVersion, + * privacyBudgetKey: dummyValue, destination: dummyValue, scheduledReportTime: dummyValue sec past + * epoch, sourceRegistrationTime: dummyValue sec past epoch + * + *

(Latest Version) SharedInfo: reportId: auto-generated from the following values, version: + * "0.1", api: "attribution-reporting", destination: dummyValue, scheduledReportTime: dummyValue + * sec past epoch, sourceRegistrationTime: dummyValue sec past epoch + * + * @param dummyValue a dummy integer value that is set as various applicable values in the + * returned Report. See above for where it is specifically used. + * @param reportId Value to set the report's ID to. + * @param reportVersion Version of the report to generate. Empty String for default version, + * anything else for latest. + * @return + */ + public static Report generateWithFixedReportId( + int dummyValue, String reportId, String reportVersion) { + return generate( + /* facts */ Optional.empty(), + Optional.of(dummyValue), + Optional.of(reportId), + reportVersion); + } + /** - * generateWithFixedReportId can be used to generate a report with fake fixed report id for the - * purpose of comparing reports. Other report fields are set based on param Given an int param, - * returns the following if param=1, Report(PrivacyBudgetKey: "1", Destination: "1", - * ReportingOrigin: "1", ScheduledReportTime: "1970-01-01 00:00:01", SourceRegistrationTime: - * "1970-01-01 00:00:01", ReportId: {UUID} Facts: [Fact("1", 1)]) containing param number of - * Facts. + * Internal Fake Report generator. Either facts OR dummyValue must be set, and only exactly one. + * If reportId is set, dummyValue must be too. See public generate() methods for specifics on the + * Fake Report that will be generated. + * + * @param facts Facts to place inside the Report's Payload. Will be a list of dummy facts if not + * set. + * @param dummyValue Integer value to set many integer-based Report variables to, namely the Time + * variables and the number of facts to generate for the payload if `facts` is not set. + * @param reportId The reportId of the generated Fake Report. If not specified, the Fake Report's + * id will be set to a random UUID. + * @param reportVersion Version of the Fake Report to generate. If set to the default version (aka + * an empty String), the privacyBudgetKey will be set to the String equivalent of dummyValue. + * Otherwise, it will be auto-generated from other SharedInfo variables. + * @return A Fake Report generated from the parameter data. */ - public static Report generateWithFixedReportId(int param, String reportId) { - return Report.builder() - .setSharedInfo( - SharedInfo.builder() - .setPrivacyBudgetKey(String.valueOf(param)) - .setDestination(String.valueOf(param)) - .setReportingOrigin(String.valueOf(param)) - .setScheduledReportTime(Instant.EPOCH.plus(param, SECONDS)) - .setSourceRegistrationTime(Instant.EPOCH.plus(param, SECONDS)) - .setReportId(reportId) - .build()) - .setPayload( - Payload.builder() - .addAllFact( - IntStream.range(0, param) - .mapToObj( - i -> - Fact.builder() - .setBucket(createBucketFromInt(param)) - .setValue(param) - .build()) - .collect(toImmutableList())) - .build()) - .build(); + private static Report generate( + Optional> facts, + Optional dummyValue, + Optional reportId, + String reportVersion) { + // Sanity check. Evaluates as XNOR to confirm only one of facts or dummyValue is present + if (!(facts.isPresent() ^ dummyValue.isPresent())) { + throw new IllegalStateException( + "Exactly one of the parameters facts or dummyValue must be set"); + } + + Optional paramString = dummyValue.map(number -> String.valueOf(number)); + int dummyValueActual = dummyValue.orElse(1); + String dummyStringActual = paramString.orElse("dummy"); + Instant dummyTime = Instant.EPOCH.plus(dummyValueActual, SECONDS); + + Report.Builder reportBuilder = + Report.builder() + .setPayload( + Payload.builder() + .addAllFact( + facts.orElse( + IntStream.range(0, dummyValueActual) + .mapToObj( + i -> + Fact.builder() + .setBucket(createBucketFromInt(dummyValueActual)) + .setValue(dummyValueActual) + .build()) + .collect(toImmutableList()))) + .build()); + + SharedInfo.Builder sharedInfoBuilder = + SharedInfo.builder() + .setDestination(dummyStringActual) + .setScheduledReportTime(dummyTime) + .setSourceRegistrationTime(dummyTime) + .setReportingOrigin(dummyStringActual) + .setReportId(reportId.orElse(String.valueOf(UUID.randomUUID()))); + + if (reportVersion.equals(DEFAULT_VERSION)) { + sharedInfoBuilder.setPrivacyBudgetKey(dummyStringActual); + } else { + /** SharedInfo in latest format */ + sharedInfoBuilder.setVersion("0.1").setApi("attribution-reporting"); + } + + reportBuilder.setSharedInfo(sharedInfoBuilder.build()); + return reportBuilder.build(); } private FakeReportGenerator() {} diff --git a/java/com/google/aggregate/adtech/worker/testing/InMemoryResultLogger.java b/java/com/google/aggregate/adtech/worker/testing/InMemoryResultLogger.java index de179a2f..6260e624 100644 --- a/java/com/google/aggregate/adtech/worker/testing/InMemoryResultLogger.java +++ b/java/com/google/aggregate/adtech/worker/testing/InMemoryResultLogger.java @@ -17,6 +17,7 @@ package com.google.aggregate.adtech.worker.testing; import com.google.aggregate.adtech.worker.ResultLogger; +import com.google.aggregate.adtech.worker.exceptions.ResultLogException; import com.google.aggregate.adtech.worker.model.AggregatedFact; import com.google.scp.operator.cpio.blobstorageclient.model.DataLocation; import com.google.scp.operator.cpio.blobstorageclient.model.DataLocation.BlobStoreDataLocation; diff --git a/java/com/google/aggregate/adtech/worker/testing/LocalAggregationWorkerRunner.java b/java/com/google/aggregate/adtech/worker/testing/LocalAggregationWorkerRunner.java index 92395e8a..dfbe8c24 100644 --- a/java/com/google/aggregate/adtech/worker/testing/LocalAggregationWorkerRunner.java +++ b/java/com/google/aggregate/adtech/worker/testing/LocalAggregationWorkerRunner.java @@ -20,7 +20,7 @@ import com.google.aggregate.adtech.worker.AggregationWorker; import com.google.aggregate.adtech.worker.AggregationWorkerArgs; import com.google.aggregate.adtech.worker.AggregationWorkerModule; -import com.google.aggregate.adtech.worker.ResultLogger.ResultLogException; +import com.google.aggregate.adtech.worker.exceptions.ResultLogException; import com.google.aggregate.adtech.worker.model.AggregatedFact; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ServiceManager; diff --git a/java/com/google/aggregate/adtech/worker/testing/NoopJobProcessor.java b/java/com/google/aggregate/adtech/worker/testing/NoopJobProcessor.java index 7d695e9f..9c9bfc40 100644 --- a/java/com/google/aggregate/adtech/worker/testing/NoopJobProcessor.java +++ b/java/com/google/aggregate/adtech/worker/testing/NoopJobProcessor.java @@ -34,9 +34,9 @@ public final class NoopJobProcessor implements JobProcessor { } @Override - public JobResult process(Job Job) throws AggregationJobProcessException { + public JobResult process(Job Job) throws IllegalStateException { if (shouldThrowException) { - throw new AggregationJobProcessException(new IllegalStateException("Was set to throw")); + throw new IllegalStateException("Was set to throw"); } lastProcessed = Optional.of(Job); return jobResultToReturn; diff --git a/java/com/google/aggregate/adtech/worker/util/BUILD b/java/com/google/aggregate/adtech/worker/util/BUILD index e932d962..54234457 100644 --- a/java/com/google/aggregate/adtech/worker/util/BUILD +++ b/java/com/google/aggregate/adtech/worker/util/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/adtech/worker/validation/BUILD b/java/com/google/aggregate/adtech/worker/validation/BUILD index c6d8152a..75290dfe 100644 --- a/java/com/google/aggregate/adtech/worker/validation/BUILD +++ b/java/com/google/aggregate/adtech/worker/validation/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/adtech/worker/writer/BUILD b/java/com/google/aggregate/adtech/worker/writer/BUILD index b66cc92d..2b304efe 100644 --- a/java/com/google/aggregate/adtech/worker/writer/BUILD +++ b/java/com/google/aggregate/adtech/worker/writer/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/adtech/worker/writer/avro/BUILD b/java/com/google/aggregate/adtech/worker/writer/avro/BUILD index 1553c02e..88fddcad 100644 --- a/java/com/google/aggregate/adtech/worker/writer/avro/BUILD +++ b/java/com/google/aggregate/adtech/worker/writer/avro/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/adtech/worker/writer/avro/LocalAvroDebugResultFileWriter.java b/java/com/google/aggregate/adtech/worker/writer/avro/LocalAvroDebugResultFileWriter.java index 888d355f..67bd1a38 100644 --- a/java/com/google/aggregate/adtech/worker/writer/avro/LocalAvroDebugResultFileWriter.java +++ b/java/com/google/aggregate/adtech/worker/writer/avro/LocalAvroDebugResultFileWriter.java @@ -17,6 +17,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; import com.google.aggregate.adtech.worker.model.AggregatedFact; import com.google.aggregate.adtech.worker.writer.LocalResultFileWriter; @@ -46,7 +47,8 @@ public final class LocalAvroDebugResultFileWriter implements LocalResultFileWrit public void writeLocalFile(Stream results, Path resultFilePath) throws FileWriteException { - try (OutputStream outputAvroStream = Files.newOutputStream(resultFilePath, CREATE); + try (OutputStream outputAvroStream = + Files.newOutputStream(resultFilePath, CREATE, TRUNCATE_EXISTING); AvroDebugResultsWriter avroDebugResultsWriter = writerFactory.create(outputAvroStream)) { ImmutableList metaData = ImmutableList.of(); Stream resultsRecords = diff --git a/java/com/google/aggregate/adtech/worker/writer/avro/LocalAvroResultFileWriter.java b/java/com/google/aggregate/adtech/worker/writer/avro/LocalAvroResultFileWriter.java index 33359bac..6acfbc46 100644 --- a/java/com/google/aggregate/adtech/worker/writer/avro/LocalAvroResultFileWriter.java +++ b/java/com/google/aggregate/adtech/worker/writer/avro/LocalAvroResultFileWriter.java @@ -16,8 +16,8 @@ package com.google.aggregate.adtech.worker.writer.avro; -import static java.nio.file.StandardOpenOption.APPEND; import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; import com.google.aggregate.adtech.worker.model.AggregatedFact; import com.google.aggregate.adtech.worker.util.NumericConversions; @@ -65,7 +65,8 @@ public void writeLocalFile(Stream results, Path resultFilePath) DatumWriter datumWriter = new GenericDatumWriter<>(schema); DataFileWriter dataFileWriter = new DataFileWriter<>(datumWriter); try { - dataFileWriter.create(schema, Files.newOutputStream(resultFilePath, CREATE, APPEND)); + dataFileWriter.create( + schema, Files.newOutputStream(resultFilePath, CREATE, TRUNCATE_EXISTING)); // Write all results to an Avro file. .append() call can throw IOExceptions so using an // Iterator is cleaner for exception handling. diff --git a/java/com/google/aggregate/adtech/worker/writer/json/BUILD b/java/com/google/aggregate/adtech/worker/writer/json/BUILD index 9c0441ce..e4ce02f3 100644 --- a/java/com/google/aggregate/adtech/worker/writer/json/BUILD +++ b/java/com/google/aggregate/adtech/worker/writer/json/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/perf/BUILD b/java/com/google/aggregate/perf/BUILD index ebaf4f5f..6d3c0ec0 100644 --- a/java/com/google/aggregate/perf/BUILD +++ b/java/com/google/aggregate/perf/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/perf/export/BUILD b/java/com/google/aggregate/perf/export/BUILD index 8cc17967..0043ab3c 100644 --- a/java/com/google/aggregate/perf/export/BUILD +++ b/java/com/google/aggregate/perf/export/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/privacy/budgeting/BUILD b/java/com/google/aggregate/privacy/budgeting/BUILD index 67d47018..8db41bf7 100644 --- a/java/com/google/aggregate/privacy/budgeting/BUILD +++ b/java/com/google/aggregate/privacy/budgeting/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/privacy/budgeting/converter/BUILD b/java/com/google/aggregate/privacy/budgeting/converter/BUILD index 9144cdbf..7094bc6b 100644 --- a/java/com/google/aggregate/privacy/budgeting/converter/BUILD +++ b/java/com/google/aggregate/privacy/budgeting/converter/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/privacy/budgeting/model/BUILD b/java/com/google/aggregate/privacy/budgeting/model/BUILD index 4c323342..86d6101a 100644 --- a/java/com/google/aggregate/privacy/budgeting/model/BUILD +++ b/java/com/google/aggregate/privacy/budgeting/model/BUILD @@ -14,6 +14,8 @@ # Exposes the models for interacting with the Privacy Budget Manager. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/privacy/budgeting/storage/BUILD b/java/com/google/aggregate/privacy/budgeting/storage/BUILD index f5afb29d..f3b7a37d 100644 --- a/java/com/google/aggregate/privacy/budgeting/storage/BUILD +++ b/java/com/google/aggregate/privacy/budgeting/storage/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/privacy/budgeting/testing/BUILD b/java/com/google/aggregate/privacy/budgeting/testing/BUILD index 6b9222d4..fe45d97d 100644 --- a/java/com/google/aggregate/privacy/budgeting/testing/BUILD +++ b/java/com/google/aggregate/privacy/budgeting/testing/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/privacy/noise/BUILD b/java/com/google/aggregate/privacy/noise/BUILD index 3244df29..692ba934 100644 --- a/java/com/google/aggregate/privacy/noise/BUILD +++ b/java/com/google/aggregate/privacy/noise/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/privacy/noise/model/BUILD b/java/com/google/aggregate/privacy/noise/model/BUILD index 7575e808..29e871cf 100644 --- a/java/com/google/aggregate/privacy/noise/model/BUILD +++ b/java/com/google/aggregate/privacy/noise/model/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/privacy/noise/proto/privacy_parameters.proto b/java/com/google/aggregate/privacy/noise/proto/privacy_parameters.proto index cc78478a..3fdba325 100644 --- a/java/com/google/aggregate/privacy/noise/proto/privacy_parameters.proto +++ b/java/com/google/aggregate/privacy/noise/proto/privacy_parameters.proto @@ -58,4 +58,4 @@ message NoiseSensitivity { // partitions (keys). Two values with difference <= l1_sensitivity that are // noised should produce roughly the same distribution. Aka global sensitivity optional int64 l1_sensitivity = 1; -} \ No newline at end of file +} diff --git a/java/com/google/aggregate/privacy/noise/testing/BUILD b/java/com/google/aggregate/privacy/noise/testing/BUILD index 53e51529..d24a88c7 100644 --- a/java/com/google/aggregate/privacy/noise/testing/BUILD +++ b/java/com/google/aggregate/privacy/noise/testing/BUILD @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library", "java_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + package(default_visibility = ["//visibility:public"]) proto_library( diff --git a/java/com/google/aggregate/privacy/noise/testing/integration/BUILD b/java/com/google/aggregate/privacy/noise/testing/integration/BUILD index 6cdffa25..87882ce7 100644 --- a/java/com/google/aggregate/privacy/noise/testing/integration/BUILD +++ b/java/com/google/aggregate/privacy/noise/testing/integration/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/privacy/noise/testing/noise_tester.proto b/java/com/google/aggregate/privacy/noise/testing/noise_tester.proto index e328db5f..b8f97bc4 100644 --- a/java/com/google/aggregate/privacy/noise/testing/noise_tester.proto +++ b/java/com/google/aggregate/privacy/noise/testing/noise_tester.proto @@ -55,4 +55,4 @@ message DpNoiseTesterParams { } optional NoiseValueType noise_value_type = 7; -} \ No newline at end of file +} diff --git a/java/com/google/aggregate/protocol/avro/BUILD b/java/com/google/aggregate/protocol/avro/BUILD index 9f890e61..db70dd03 100644 --- a/java/com/google/aggregate/protocol/avro/BUILD +++ b/java/com/google/aggregate/protocol/avro/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_binary", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/shared/BUILD b/java/com/google/aggregate/shared/BUILD new file mode 100644 index 00000000..c3f83de5 --- /dev/null +++ b/java/com/google/aggregate/shared/BUILD @@ -0,0 +1,29 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_java//java:defs.bzl", "java_library") + +package(default_visibility = ["//visibility:public"]) + +java_library( + name = "shared", + srcs = [ + "DependencyMetadata.java", + "LicenseUtil.java", + ], + deps = [ + "//java/external:autovalue", + "//java/external:autovalue_annotations", + ], +) diff --git a/java/com/google/aggregate/adtech/worker/DependencyMetadata.java b/java/com/google/aggregate/shared/DependencyMetadata.java similarity index 95% rename from java/com/google/aggregate/adtech/worker/DependencyMetadata.java rename to java/com/google/aggregate/shared/DependencyMetadata.java index 6ebac50b..5231b212 100644 --- a/java/com/google/aggregate/adtech/worker/DependencyMetadata.java +++ b/java/com/google/aggregate/shared/DependencyMetadata.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.google.aggregate.adtech.worker; +package com.google.aggregate.shared; import com.google.auto.value.AutoValue; diff --git a/java/com/google/aggregate/adtech/worker/LicenseUtil.java b/java/com/google/aggregate/shared/LicenseUtil.java similarity index 92% rename from java/com/google/aggregate/adtech/worker/LicenseUtil.java rename to java/com/google/aggregate/shared/LicenseUtil.java index 2769c7fd..052ef086 100644 --- a/java/com/google/aggregate/adtech/worker/LicenseUtil.java +++ b/java/com/google/aggregate/shared/LicenseUtil.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.google.aggregate.adtech.worker; +package com.google.aggregate.shared; import java.io.BufferedReader; import java.io.IOException; @@ -25,8 +25,8 @@ import java.util.List; /** - * Utility to print all licenses for dependencies on Aggregation Worker Binary. TODO(b/226980133) - * Add test for this utility if this solution is acceptable + * Utility to print all licenses for dependencies. TODO(b/226980133) Add test for this utility if + * this solution is acceptable. */ public final class LicenseUtil { diff --git a/java/com/google/aggregate/shared/mapper/BUILD b/java/com/google/aggregate/shared/mapper/BUILD index ee6616da..18645eed 100644 --- a/java/com/google/aggregate/shared/mapper/BUILD +++ b/java/com/google/aggregate/shared/mapper/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/testing/utils/BUILD b/java/com/google/aggregate/testing/utils/BUILD index fd42dc7d..943f2826 100644 --- a/java/com/google/aggregate/testing/utils/BUILD +++ b/java/com/google/aggregate/testing/utils/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/com/google/aggregate/tools/diff/BUILD b/java/com/google/aggregate/tools/diff/BUILD index a4fd8fc9..daddc8c4 100644 --- a/java/com/google/aggregate/tools/diff/BUILD +++ b/java/com/google/aggregate/tools/diff/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_binary", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( @@ -47,6 +49,7 @@ java_library( ":result_diff", "//java/com/google/aggregate/adtech/worker", "//java/com/google/aggregate/adtech/worker:worker_runner", + "//java/com/google/aggregate/adtech/worker/exceptions", "//java/com/google/aggregate/adtech/worker/model", "//java/com/google/aggregate/adtech/worker/testing:avro_results_file_reader", "//java/com/google/aggregate/adtech/worker/testing:local_aggregation_worker_runner", diff --git a/java/com/google/aggregate/tools/diff/DiffRunner.java b/java/com/google/aggregate/tools/diff/DiffRunner.java index ad14433b..d09afb64 100644 --- a/java/com/google/aggregate/tools/diff/DiffRunner.java +++ b/java/com/google/aggregate/tools/diff/DiffRunner.java @@ -19,7 +19,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import com.beust.jcommander.JCommander; -import com.google.aggregate.adtech.worker.ResultLogger.ResultLogException; +import com.google.aggregate.adtech.worker.exceptions.ResultLogException; import com.google.aggregate.adtech.worker.model.AggregatedFact; import com.google.aggregate.adtech.worker.testing.AvroResultsFileReader; import com.google.aggregate.adtech.worker.testing.LocalAggregationWorkerRunner; diff --git a/java/com/google/aggregate/tools/diff/README.md b/java/com/google/aggregate/tools/diff/README.md index ee1011dd..db136395 100644 --- a/java/com/google/aggregate/tools/diff/README.md +++ b/java/com/google/aggregate/tools/diff/README.md @@ -8,13 +8,13 @@ Use case: Check if there is any diff after a new code change. bazel build java/com/google/aggregate/tools/diff:DiffRunner ``` -2. Create the test input if you don't have one +1. Create the test input if you don't have one ```shell bazel build worker/testing/data:diff_1k ``` -3. Create the golden as a pre-change truth +1. Create the golden as a pre-change truth ```shell bazel-bin/java/com/google/aggregate/tools/diff/DiffRunner \ @@ -31,17 +31,16 @@ Example output: New golden file is written to: /tmp/diff_1k_golden.avro ``` -`update_golden` will make the **DiffRunner** create a new golden file with the -test inputs. +`update_golden` will make the **DiffRunner** create a new golden file with the test inputs. -4. Do some changes on the Aggregation worker code and recompile **DiffRunner** - to capture worker changes +1. Do some changes on the Aggregation worker code and recompile **DiffRunner** to capture worker + changes ```shell bazel build java/com/google/aggregate/tools/diff:DiffRunner ``` -5. Run diff between the pre-change truth and post-change result +1. Run diff between the pre-change truth and post-change result ```shell bazel-bin/java/com/google/aggregate/tools/diff/DiffRunner \ @@ -57,7 +56,7 @@ Found diffs between left(test) and right(golden). not equal: only on left={f=AggregatedFact{key=f, count=41, value=195}}: only on right={C=AggregatedFact{key=C, count=7, value=496}} ``` -Note: The **DiffRunner** is -using `java/com/google/aggregate/adtech/worker/testing/LocalAggregationWorkerRunner.java` -as worker runner. If you want to run diff checks with different worker flags, -you may have to make a local change on that class. \ No newline at end of file +Note: The **DiffRunner** is using +`java/com/google/aggregate/adtech/worker/testing/LocalAggregationWorkerRunner.java` as worker +runner. If you want to run diff checks with different worker flags, you may have to make a local +change on that class. diff --git a/java/com/google/aggregate/tools/generateinputs/BUILD b/java/com/google/aggregate/tools/generateinputs/BUILD index 52561aa6..5a039fd7 100644 --- a/java/com/google/aggregate/tools/generateinputs/BUILD +++ b/java/com/google/aggregate/tools/generateinputs/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_binary") + java_binary( name = "GenerateInputs", srcs = ["GenerateInputs.java"], diff --git a/java/com/google/aggregate/tools/shard/AvroShard.java b/java/com/google/aggregate/tools/shard/AvroShard.java index f924ff11..c5901b79 100644 --- a/java/com/google/aggregate/tools/shard/AvroShard.java +++ b/java/com/google/aggregate/tools/shard/AvroShard.java @@ -18,6 +18,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; @@ -44,7 +45,20 @@ import java.nio.file.Paths; import java.util.List; -/** A tool for sharding Avro report or domain inputs */ +/* + * This tool is for sharding Avro report or domain. + * The reports can be sharded with: + * bazel run //java/com/google/aggregate/tools/shard:AvroShard \ + * --input $PWD/20k_reports.avro \ + * --output_dir $PWD/20k_shards \ + * --num_shards 20 + * The domain can ba sharded with: + * bazel run //java/com/google/aggregate/tools/shard:AvroShard \ + * --input $PWD/20k_domain.avro \ + * --output_dir $PWD/20k_domain \ + * --num_shards 20 \ + * --domain + */ final class AvroShard { static Injector injector = Guice.createInjector(new Env()); @@ -112,7 +126,7 @@ private static void writeDomainRecords( AvroOutputDomainWriterFactory domainWriterFactory = injector.getInstance(AvroOutputDomainWriterFactory.class); - int runningShard = 0; + int runningShard = 1; for (List shard : Iterables.partition(records, shardSize)) { Path shardPath = outputDirPath.resolve( @@ -121,15 +135,11 @@ private static void writeDomainRecords( System.out.printf( "Writing domain shard %d at %s\n", runningShard, shardPath.toAbsolutePath()); - try { - if (!Files.exists(outputDirPath)) { - Files.createDirectories(outputDirPath); - } - OutputStream shardStream = Files.newOutputStream(shardPath, CREATE); - AvroOutputDomainWriter writer = domainWriterFactory.create(shardStream); + Files.createDirectories(outputDirPath); + + try (OutputStream shardStream = Files.newOutputStream(shardPath, CREATE, TRUNCATE_EXISTING); + AvroOutputDomainWriter writer = domainWriterFactory.create(shardStream)) { writer.writeRecords(/* metadata= */ ImmutableList.of(), ImmutableList.copyOf(shard)); - } catch (Exception ex) { - System.out.println(ex.getMessage()); } System.out.println("Domain shard written"); runningShard++; @@ -142,13 +152,16 @@ private static void writeReportRecords( AvroReportWriterFactory reportWriterFactory = injector.getInstance(AvroReportWriterFactory.class); - int runningShard = 0; + int runningShard = 1; for (List shard : Iterables.partition(records, shardSize)) { Path shardPath = - outputDirPath.resolve(String.format("shard-%d-of-%d.avro", runningShard, shardNum)); + outputDirPath.resolve( + String.format("shard-report-%d-of-%d.avro", runningShard, shardNum)); System.out.printf("Writing shard %d at %s\n", runningShard, shardPath.toAbsolutePath()); + Files.createDirectories(outputDirPath); + try (OutputStream shardStream = Files.newOutputStream(shardPath, CREATE); AvroReportWriter writer = reportWriterFactory.create(shardStream)) { writer.writeRecords(/* metadata= */ ImmutableList.of(), ImmutableList.copyOf(shard)); diff --git a/java/com/google/aggregate/tools/shard/BUILD b/java/com/google/aggregate/tools/shard/BUILD index c2993071..22700fca 100644 --- a/java/com/google/aggregate/tools/shard/BUILD +++ b/java/com/google/aggregate/tools/shard/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_binary", "java_library") + package(default_visibility = ["//visibility:public"]) java_library( diff --git a/java/external/BUILD b/java/external/BUILD index 3ba82358..dcf0a572 100644 --- a/java/external/BUILD +++ b/java/external/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library", "java_plugin") + package( default_visibility = ["//visibility:public"], ) @@ -410,11 +412,6 @@ alias( actual = "@com_google_adm_cloud_scp//java/com/google/scp/coordinator/privacy/budgeting/model", ) -alias( - name = "dao_metadatadb_model", - actual = "@com_google_adm_cloud_scp//java/com/google/scp/operator/shared/dao/metadatadb/model", -) - alias( name = "dao_metadatadb_aws_model", actual = "@com_google_adm_cloud_scp//java/com/google/scp/operator/shared/dao/metadatadb/aws/model", @@ -445,11 +442,6 @@ alias( actual = "@com_google_adm_cloud_scp//java/com/google/scp/operator/shared/dao/metadatadb/testing:fake_metadata_db", ) -alias( - name = "dao_jobqueue_model", - actual = "@com_google_adm_cloud_scp//java/com/google/scp/operator/shared/dao/jobqueue/model", -) - alias( name = "dao_jobqueue_aws", actual = "@com_google_adm_cloud_scp//java/com/google/scp/operator/shared/dao/jobqueue/aws", diff --git a/javatests/com/google/aggregate/adtech/worker/AggregationWorkerDiffTest.java b/javatests/com/google/aggregate/adtech/worker/AggregationWorkerDiffTest.java index addc80e9..2f7f8a2e 100644 --- a/javatests/com/google/aggregate/adtech/worker/AggregationWorkerDiffTest.java +++ b/javatests/com/google/aggregate/adtech/worker/AggregationWorkerDiffTest.java @@ -42,6 +42,11 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +/* + * Test aggregation worker generate the same results as local golden reports under specific + * condition (noising_epsilon and noising_l1_sensitivity). This test use sharded domain and reports + * as input. + */ @RunWith(JUnit4.class) public class AggregationWorkerDiffTest { @@ -51,20 +56,25 @@ public class AggregationWorkerDiffTest { @Inject private AvroResultsFileReader resultsFileReader; - private Path input; + private Path reportPath; + private Path domainPath; private Path key; private Path golden; private ImmutableList goldenFacts; - // Reports directory for the concurrent aggregator: the aggregator takes in all the files in the - // directory. + // The aggregator takes in all the files in the reportsDirectory for reports input and takes all + // the files in domainDirectory for domain input. private Path reportsDirectory; + private Path domainDirectory; @Before public void setUp() throws Exception { reportsDirectory = testWorkingDir.getRoot().toPath().resolve("reports"); + domainDirectory = testWorkingDir.getRoot().toPath().resolve("domain"); Files.createDirectory(reportsDirectory); - input = reportsDirectory.toAbsolutePath(); + Files.createDirectory(domainDirectory); + reportPath = reportsDirectory.toAbsolutePath(); + domainPath = domainDirectory.toAbsolutePath(); key = Paths.get("worker/testing/data/encryption_key.pb"); golden = Paths.get("worker/testing/data/1k/diff_1k_output.golden"); @@ -74,17 +84,17 @@ public void setUp() throws Exception { } /** - * Copy the shards from the provided directory to the reportsDirectory for use with the concurrent + * Copy the shards from the provided directory to the target directory for use with the concurrent * processor. * *

NOTE: This method assumes there are no subdirectories in the directory provided, only files. */ - public void copyShards(Path directory) throws Exception { - Files.list(directory) + public void copyShards(Path fromDir, Path toDir) throws Exception { + Files.list(fromDir) .forEach( file -> { try { - Files.copy(file, reportsDirectory.resolve(file.getFileName())); + Files.copy(file, toDir.resolve(file.getFileName())); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -92,16 +102,25 @@ public void copyShards(Path directory) throws Exception { } // To update the golden: - // bazel build worker/testing/data/1k:diff_1k java/com/google/aggregate/tools/diff:DiffRunner - // bazel-bin/java/com/google/aggregate/tools/diff/DiffRunner \ - // --test_input bazel-bin/worker/testing/data/1k/diff_1k.avro \ - // --test_key worker/testing/data/encryption_key.pb \ - // --test_golden worker/testing/data/1k/diff_1k_output.golden \ + // bazel build worker/testing/data/1k:diff_1k_cbor + // bazel build worker/testing/data/1k/sharded:diff_1k_cbor + // bazel build worker/testing/data/1k/sharded:diff_1k_cbor_domain + // bazel run java/com/google/aggregate/tools/diff:DiffRunner -- \ + // --test_input $PWD/bazel-bin/worker/testing/data/1k/sharded/diff_1k_cbor_shards \ + // --test_output_domain_dir + // $PWD/bazel-bin/worker/testing/data/1k/sharded/diff_1k_cbor_domain_shards \ + // --test_key $PWD/worker/testing/data/encryption_key.pb \ + // --test_golden $PWD/worker/testing/data/1k/diff_1k_output.golden \ + // --noising_epsilon 0.1 \ + // --noising_l1_sensitivity 4 \ // --update_golden + // Note: The domain generated in diff_1k_cbor_domain is fully overlapped with reports keys. @Test public void diffTestConstantNoising() throws Exception { - copyShards(Paths.get("worker/testing/data/1k/sharded/diff_1k_cbor_shards/")); + copyShards(Paths.get("worker/testing/data/1k/sharded/diff_1k_cbor_shards/"), reportsDirectory); + copyShards( + Paths.get("worker/testing/data/1k/sharded/diff_1k_cbor_domain_shards/"), domainDirectory); MapDifference diff = diff(); @@ -116,11 +135,11 @@ private MapDifference diff() throws Exception { ImmutableList goldenFacts = resultsFileReader.readAvroResultsFile(golden); LocalAggregationWorkerRunner workerRunner = LocalAggregationWorkerRunner.create(testWorkingDir.getRoot().toPath()); - // TODO(b/228085828): Add tests without flag --domain_optional. + workerRunner.updateArgs( new String[] { "--local_file_single_puller_path", - input.toString(), + reportPath.toString(), "--local_file_decryption_key_path", key.toString(), // Use noise parameters that match those provided when the golden output were generated @@ -129,8 +148,7 @@ private MapDifference diff() throws Exception { "--noising_l1_sensitivity", "4", "--local_output_domain_path", - "", - "--domain_optional" + domainPath.toString(), }); workerRunner.run(); diff --git a/javatests/com/google/aggregate/adtech/worker/AggregationWorkerHermeticTest.java b/javatests/com/google/aggregate/adtech/worker/AggregationWorkerHermeticTest.java index 096dd419..3153f0f0 100644 --- a/javatests/com/google/aggregate/adtech/worker/AggregationWorkerHermeticTest.java +++ b/javatests/com/google/aggregate/adtech/worker/AggregationWorkerHermeticTest.java @@ -19,7 +19,6 @@ import static com.google.aggregate.adtech.worker.util.NumericConversions.createBucketFromInt; import static com.google.common.truth.Truth.assertThat; import static java.nio.file.StandardOpenOption.CREATE; -import static org.junit.Assert.assertThrows; import com.beust.jcommander.JCommander; import com.google.acai.Acai; @@ -29,6 +28,7 @@ import com.google.aggregate.adtech.worker.testing.MaterializedAggregationResults; import com.google.aggregate.privacy.noise.testing.ConstantNoiseModule.ConstantNoiseApplier; import com.google.aggregate.privacy.noise.testing.FakeNoiseApplierSupplier; +import com.google.aggregate.protocol.avro.AvroOutputDomainWriterFactory; import com.google.aggregate.protocol.avro.AvroRecordWriter.MetadataElement; import com.google.aggregate.protocol.avro.AvroReportRecord; import com.google.aggregate.protocol.avro.AvroReportWriter; @@ -62,6 +62,12 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +/** + * AggregationWorkerHermeticTest covers tests with different worker args and noises. The threshold + * used in AggregationWorkerHermeticTest is calculated from formula in {@code ThresholdSupplier}. + * Given noising_epsilon, noising_delta and noising_l1_sensitivity in provided args, the threshold + * is round 4.57 in this test. + */ @RunWith(JUnit4.class) public class AggregationWorkerHermeticTest { @@ -88,6 +94,7 @@ public class AggregationWorkerHermeticTest { @Rule public final Acai acai = new Acai(TestEnv.class); @Inject private AvroReportWriterFactory writerFactory; + @Inject private AvroOutputDomainWriterFactory domainWriterFactory; @Before public void setUp() throws Exception { @@ -99,12 +106,25 @@ public void setUp() throws Exception { resultFile = testWorkingDirPath.resolve("results.json"); domainAvro = testWorkingDirPath.resolve("domain.avro"); domainShardsDir = testWorkingDirPath.resolve("domain_shards"); + // Result of summing the values of the keys: 22:56, 33:33, 44:123, 55:3 + simulationInputFileLines = + ImmutableList.of( + "22:1,33:2", + "22:10,55:3,33:10", + "22:15", + "22:10", + "22:20,33:15", + "44:123,33:5", + "33:1"); metadata = ImmutableList.of( MetadataElement.create(/* key= */ "foo", /* value= */ "bar"), MetadataElement.create(/* key= */ "abc", /* value= */ "xyz")); - args = getLocalAggregationWorkerArgs(); - + // Set domain_optional arg to true for most test cases in order to bypass creating domain files. + // Otherwise, the aggregated result would be empty if domain is required without domain files. + args = + getLocalAggregationWorkerArgs( + /* noEncryption= */ false, /* outputDomain= */ false, /* domainOptional= */ true); Files.createDirectory(reportShardsDir); Files.createDirectory(domainShardsDir); } @@ -117,15 +137,6 @@ public void tearDown() throws Exception { @Test public void localTestNoNoising() throws Exception { - simulationInputFileLines = - ImmutableList.of( - "22:1,33:2", - "22:10,55:3,33:10", - "22:15", - "22:10", - "22:20,33:15", - "44:123,33:5", - "33:1"); AwsHermeticTestHelper.generateAvroReportsFromTextList( SimulationTestParams.builder() .setHybridKey(hybridKey) @@ -153,16 +164,9 @@ public void localTestNoNoising() throws Exception { @Test public void localTestNoNoisingNoEncryption() throws Exception { - args = getLocalAggregationWorkerArgs(/* noEncryption= */ true, /* outputDomain= */ false); - simulationInputFileLines = - ImmutableList.of( - "22:1,33:2", - "22:10,55:3,33:10", - "22:15", - "22:10", - "22:20,33:15", - "44:123,33:5", - "33:1"); + args = + getLocalAggregationWorkerArgs( + /* noEncryption= */ true, /* outputDomain= */ false, /* domainOptional= */ true); AwsHermeticTestHelper.generateAvroReportsFromTextList( SimulationTestParams.builder() .setHybridKey(hybridKey) @@ -192,15 +196,6 @@ public void localTestNoNoisingNoEncryption() throws Exception { @Test public void localTestConstantNoising() throws Exception { - simulationInputFileLines = - ImmutableList.of( - "55:1,66:2", - "55:10,77:3,66:10", - "55:15", - "55:10", - "55:20,66:15", - "88:123,66:5", - "66:1"); AwsHermeticTestHelper.generateAvroReportsFromTextList( SimulationTestParams.builder() .setHybridKey(hybridKey) @@ -220,16 +215,16 @@ public void localTestConstantNoising() throws Exception { // Nothing gets filtered out now because everything meets the threshold. AggregatedFact expectedFact1 = AggregatedFact.create( - /* key= */ createBucketFromInt(55), /* metric= */ 66, /* unnoisedMetric= */ 56L); + /* key= */ createBucketFromInt(22), /* metric= */ 66, /* unnoisedMetric= */ 56L); AggregatedFact expectedFact2 = AggregatedFact.create( - /* key= */ createBucketFromInt(66), /* metric= */ 43, /* unnoisedMetric= */ 33L); + /* key= */ createBucketFromInt(33), /* metric= */ 43, /* unnoisedMetric= */ 33L); AggregatedFact expectedFact3 = AggregatedFact.create( - /* key= */ createBucketFromInt(88), /* metric= */ 133, /* unnoisedMetric= */ 123L); + /* key= */ createBucketFromInt(44), /* metric= */ 133, /* unnoisedMetric= */ 123L); AggregatedFact expectedFact4 = AggregatedFact.create( - /* key= */ createBucketFromInt(77), /* metric= */ 13, /* unnoisedMetric= */ 3L); + /* key= */ createBucketFromInt(55), /* metric= */ 13, /* unnoisedMetric= */ 3L); assertThat(factList) .containsExactly(expectedFact1, expectedFact2, expectedFact3, expectedFact4); // The name of the job is empty on both sides of '|' because this is a fully local job. The @@ -289,28 +284,20 @@ public void localTestConstantNoising_shardedReport() throws Exception { @Test public void localTestConstantNoising_shardedDomain() throws Exception { - args = getLocalAggregationWorkerArgs(/* noEncryption= */ false, /* outputDomain= */ true); - simulationInputFileLines = - ImmutableList.of( - "55:1,66:2", - "55:10,77:3,66:10", - "55:15", - "55:10", - "55:20,66:15", - "88:123,66:5", - "66:1"); + args = + getLocalAggregationWorkerArgs( + /* noEncryption= */ false, /* outputDomain= */ true, /* domainOptional= */ true); AwsHermeticTestHelper.generateAvroReportsFromTextList( SimulationTestParams.builder() .setHybridKey(hybridKey) .setReportsAvro(reportsAvro) .setSimulationInputFileLines(simulationInputFileLines) - .setOutputDomainPath(domainAvro) - .setWriteOutputDomain(true) - .setOutputDomainSize(2) .build()); Files.copy(reportsAvro, reportShardsDir.resolve("shard.avro")); - Files.copy(domainAvro, domainShardsDir.resolve("shard_1.avro")); - Files.copy(domainAvro, domainShardsDir.resolve("shard_2.avro")); + AwsHermeticTestHelper.writeDomainAvroFile(domainWriterFactory, domainAvro, "55"); + Files.move(domainAvro, domainShardsDir.resolve("shard_1.avro")); + AwsHermeticTestHelper.writeDomainAvroFile(domainWriterFactory, domainAvro, "11"); + Files.move(domainAvro, domainShardsDir.resolve("shard_2.avro")); setupLocalAggregationWorker(args); Injector injector = worker.getInjector(); @@ -324,23 +311,58 @@ public void localTestConstantNoising_shardedDomain() throws Exception { // Nothing gets filtered out now because everything meets the threshold. AggregatedFact expectedFact1 = AggregatedFact.create( - /* key= */ createBucketFromInt(55), /* metric= */ 66, /* unnoisedMetric= */ 56L); + /* key= */ createBucketFromInt(22), /* metric= */ 66, /* unnoisedMetric= */ 56L); AggregatedFact expectedFact2 = AggregatedFact.create( - /* key= */ createBucketFromInt(66), /* metric= */ 43, /* unnoisedMetric= */ 33L); + /* key= */ createBucketFromInt(33), /* metric= */ 43, /* unnoisedMetric= */ 33L); AggregatedFact expectedFact3 = AggregatedFact.create( - /* key= */ createBucketFromInt(88), /* metric= */ 133, /* unnoisedMetric= */ 123L); + /* key= */ createBucketFromInt(44), /* metric= */ 133, /* unnoisedMetric= */ 123L); AggregatedFact expectedFact4 = AggregatedFact.create( - /* key= */ createBucketFromInt(77), /* metric= */ 13, /* unnoisedMetric= */ 3L); + /* key= */ createBucketFromInt(55), /* metric= */ 13, /* unnoisedMetric= */ 3L); + AggregatedFact expectedFact5 = + AggregatedFact.create( + /* key= */ createBucketFromInt(11), /* metric= */ 10, /* unnoisedMetric= */ 0L); assertThat(factList) - .containsAtLeastElementsIn( - ImmutableList.of(expectedFact1, expectedFact2, expectedFact3, expectedFact4)); - assertThat(factList.size()).isAtMost(6); + .containsExactly(expectedFact1, expectedFact2, expectedFact3, expectedFact4, expectedFact5); + } + + @Test + public void localTestConstantNoising_DomainRequired() throws Exception { + args = + getLocalAggregationWorkerArgs( + /* noEncryption= */ false, /* outputDomain= */ true, /* domainOptional= */ false); + AwsHermeticTestHelper.generateAvroReportsFromTextList( + SimulationTestParams.builder() + .setHybridKey(hybridKey) + .setReportsAvro(reportsAvro) + .setSimulationInputFileLines(simulationInputFileLines) + .build()); + Files.copy(reportsAvro, reportShardsDir.resolve("shard.avro")); + AwsHermeticTestHelper.writeDomainAvroFile(domainWriterFactory, domainAvro, "22"); + Files.copy(domainAvro, domainShardsDir.resolve("shard.avro")); + + setupLocalAggregationWorker(args); + Injector injector = worker.getInjector(); + FakeNoiseApplierSupplier noiserSupplier = injector.getInstance(FakeNoiseApplierSupplier.class); + noiserSupplier.setFakeNoiseApplier(new ConstantNoiseApplier(10)); + + runWorker(); + ImmutableList factList = waitForAggregation(); + + // All values have 10 added due to constant noising. + // Only key 22 is in the result because only the key in domain would be output when domain is + // required. + AggregatedFact expectedFact = + AggregatedFact.create( + /* key= */ createBucketFromInt(22), /* metric= */ 66, /* unnoisedMetric= */ 56L); + + assertThat(factList).containsExactly(expectedFact); } + // TODO(b/260642993): Fix sharedInfo and encryption/decryption issues @Test public void localTestUsingCustomizedAvroReport() throws Exception { ImmutableList avroReportRecords = @@ -388,6 +410,7 @@ public void localTestUsingCustomizedAvroReport() throws Exception { .build()); } + // TODO(b/260642993): Fix sharedInfo and encryption/decryption issues @Test public void localTestUsingSameKeyAvroReports() throws Exception { ImmutableList avroReportRecords = @@ -461,6 +484,7 @@ public void localTestUsingEmptyAvroReports() throws Exception { .build()); } + // TODO(b/260642993): Fix sharedInfo and encryption/decryption issues @Test public void localTestUsingAllEmptyBytesReports() throws Exception { ImmutableList avroReportRecords = @@ -504,6 +528,7 @@ public void localTestUsingAllEmptyBytesReports() throws Exception { .build()); } + // TODO(b/260642993): Fix sharedInfo and encryption/decryption issues @Test public void localTestUsingAvroReportsWithOneKeyEmpty() throws Exception { ImmutableList avroReportRecords = @@ -549,6 +574,7 @@ public void localTestUsingAvroReportsWithOneKeyEmpty() throws Exception { .build()); } + // TODO(b/260642993): Fix sharedInfo and encryption/decryption issues @Test public void localTestUsingAvroReportsWithSpecialBytes() throws Exception { ImmutableList avroReportRecords = @@ -596,76 +622,52 @@ public void localTestUsingAvroReportsWithSpecialBytes() throws Exception { .build()); } - @Test - public void localTestUsingMalformattedAvroReport() throws Exception { - // generate mal-formatted Avro report - try (OutputStream outputAvroStream = Files.newOutputStream(reportsAvro, CREATE)) { - outputAvroStream.write(new byte[] {0x02, 0x03}); - } - Files.copy(reportsAvro, reportShardsDir.resolve("shard.avro")); - setupLocalAggregationWorker(args); - - runWorker(); - - assertThrows(TimeoutException.class, () -> waitForAggregation()); - String actualResultSerialized = Files.readString(resultFile); - ResultInfo.Builder builder = ResultInfo.newBuilder(); - JSON_PARSER.merge(actualResultSerialized, builder); - ResultInfo actualResult = builder.build(); - - assertThat(actualResult.getReturnCode()) - .isEqualTo(ReturnCode.INPUT_DATA_READ_FAILED.toString()); - assertThat(actualResult.getReturnMessage()) - .contains("java.io.IOException: Not an Avro data file."); - } - - private String[] getLocalAggregationWorkerArgs() throws Exception { - return getLocalAggregationWorkerArgs(/* noEncryption= */ false, /* outputDomain= */ false); - } - - private String[] getLocalAggregationWorkerArgs(boolean noEncryption, boolean outputDomain) - throws Exception { + private String[] getLocalAggregationWorkerArgs( + boolean noEncryption, boolean outputDomain, boolean domainOptional) throws Exception { // Create the local key HybridConfig.register(); - - return new String[] { - "--local_file_decryption_key_path", - hybridKey.toAbsolutePath().toString(), - "--job_client", - "LOCAL_FILE", - "--blob_storage_client", - "LOCAL_FS_CLIENT", - "--result_working_directory_path", - "/tmp/newton", - "--local_file_single_puller_path", - reportShardsDir.toAbsolutePath().toString(), - "--local_file_job_info_path", - resultFile.toAbsolutePath().toString(), - "--local_output_domain_path", - outputDomain ? domainShardsDir.toAbsolutePath().toString() : "", - "--decryption_key_service", - "LOCAL_FILE_DECRYPTION_KEY_SERVICE", - "--record_reader", - "LOCAL_NIO_AVRO", - "--decryption", - noEncryption ? "NOOP" : "HYBRID", - "--result_logger", - "IN_MEMORY", - "--noising", - "CONSTANT_NOISING", - "--timer_exporter", - "PLAIN_FILE", - "--timer_exporter_file_path", - stopwatchFile.toAbsolutePath().toString(), - "--simulation_inputs", - "--noising_epsilon", - "64", - "--noising_delta", - "1e-4", - "--noising_l1_sensitivity", - "4", - "--domain_optional" - }; + ImmutableList.Builder argsBuilder = + ImmutableList.builder() + .add( + "--local_file_decryption_key_path", + hybridKey.toAbsolutePath().toString(), + "--job_client", + "LOCAL_FILE", + "--blob_storage_client", + "LOCAL_FS_CLIENT", + "--result_working_directory_path", + "/tmp/newton", + "--local_file_single_puller_path", + reportShardsDir.toAbsolutePath().toString(), + "--local_file_job_info_path", + resultFile.toAbsolutePath().toString(), + "--local_output_domain_path", + outputDomain ? domainShardsDir.toAbsolutePath().toString() : "", + "--decryption_key_service", + "LOCAL_FILE_DECRYPTION_KEY_SERVICE", + "--record_reader", + "LOCAL_NIO_AVRO", + "--decryption", + noEncryption ? "NOOP" : "HYBRID", + "--result_logger", + "IN_MEMORY", + "--noising", + "CONSTANT_NOISING", + "--timer_exporter", + "PLAIN_FILE", + "--timer_exporter_file_path", + stopwatchFile.toAbsolutePath().toString(), + "--simulation_inputs", + "--noising_epsilon", + "64", + "--noising_delta", + "1e-4", + "--noising_l1_sensitivity", + "4"); + if (domainOptional) { + argsBuilder.add("--domain_optional"); + } + return argsBuilder.build().toArray(String[]::new); } private void setupLocalAggregationWorker(String[] args) { diff --git a/javatests/com/google/aggregate/adtech/worker/AwsWorkerAutoScalingTest.java b/javatests/com/google/aggregate/adtech/worker/AwsWorkerAutoScalingTest.java index 950c9631..520968eb 100644 --- a/javatests/com/google/aggregate/adtech/worker/AwsWorkerAutoScalingTest.java +++ b/javatests/com/google/aggregate/adtech/worker/AwsWorkerAutoScalingTest.java @@ -98,8 +98,7 @@ private CreateJobRequest createE2EJob(Integer jobCount) throws Exception { DATA_BUCKET, outputDataPath, Optional.of(DATA_BUCKET), - Optional.of(INPUT_DOMAIN_PATH), - /* debugPrivacyBudgetLimit= */ Optional.empty()); + Optional.of(INPUT_DOMAIN_PATH)); submitJob(createJobRequest, SUBMIT_JOB_TIMEOUT, false); return createJobRequest; } diff --git a/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousDiffTest.java b/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousDiffTest.java index 94a21010..19a656e1 100644 --- a/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousDiffTest.java +++ b/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousDiffTest.java @@ -40,6 +40,7 @@ import java.time.Duration; import java.time.temporal.ChronoUnit; import java.util.Optional; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -54,81 +55,71 @@ public class AwsWorkerContinuousDiffTest { private static final Duration completionTimeout = Duration.of(60, ChronoUnit.MINUTES); - private static final String DATA_BUCKET = "aggregation-service-testing"; - - // Input data generated with the following command: - // bazel run java/com/google/aggregate/simulation:SimluationRunner -- \ - // --aggregatable_report_file_path $PWD/1m_staging_2022_05_21.avro \ - // --num_reports 1000000 \ - // --num_encryption_keys 3 \ - // --encryption_key_service CLOUD \ - // --public_key_vending_uri \ - // https://jykzugjj3g.execute-api.us-west-2.amazonaws.com/stage/v1alpha/publicKeys \ - // --distribution FILE \ - // --distribution_file_path $PWD/1m_staging_1.txt - // Where the distribution file used is - // s3://aggregation-service-testing/testdata/1m_staging_1.txt - // Data then sharded with: - // Report: - // bazel run //java/com/google/aggregate/tools/shard:AvroShard -- \ - // --input $PWD/1m_staging_2022_05_21.avro \ - // --output_dir $PWD/1m_staging_2022_05_21_sharded \ - // --num_shards 20 - // Domain: - // bazel run //java/com/google/aggregate/tools/shard:AvroShard -- \ - // --input $PWD/1m_staging_2022_08_08_domain.avro \ - // --output_dir $PWD/1m_staging_2022_0808_sharded_domain \ - // --num_shards 20 \ - // --domain - - private static final String INPUT_DATA_PREFIX = "testdata/1m_staging_2022_05_21_sharded/shard"; - private static final String OUTPUT_DOMAIN_PREFIX = - "testdata/1m_staging_2022_08_08_sharded_domain/shard"; + private static final String DEFAULT_TEST_DATA_BUCKET = "aggregation-service-testing"; + + // Input data is generated in shared_e2e.sh + private static final String inputKey = + String.format("%s/test-inputs/10k_diff_test_input_sharded/", KOKORO_BUILD_ID); + private static final String domainKey = + String.format("%s/test-inputs/diff_test_domain_sharded/", KOKORO_BUILD_ID); + private static final String outputKey = + String.format("%s/test-outputs/10k_diff_test_output.avro", KOKORO_BUILD_ID); @Inject S3BlobStorageClient s3BlobStorageClient; @Inject AvroResultsFileReader avroResultsFileReader; + @Before + public void checkBuildEnv() { + if (KOKORO_BUILD_ID == null) { + throw new IllegalStateException("KOKORO_BUILD_ID env var must be set."); + } + } + + /** + * TEST_DATA_BUCKET is used for storing the input data and the output results. If TEST_DATA_BUCKET + * is not set in the environment variable, DEFAULT_TEST_DATA_BUCKET is used. + */ + private static String getTestDataBucket() { + if (System.getenv("TEST_DATA_BUCKET") != null) { + return System.getenv("TEST_DATA_BUCKET"); + } + return DEFAULT_TEST_DATA_BUCKET; + } + @Test public void e2eDiffTest() throws Exception { - // End to end diff testing: - // Starts with a createJob request to API gateway with the test inputs pre-uploaded to s3 - // bucket. - // Ends by calling getJob API to retrieve result information. - // Assertions are made on result status (SUCCESS) and result avro comparison with golden - // which sits in the testing bucket. // To update golden: - // 1. Log onto aws console + // 1. Run this test // 2. Find the latest test output under - // s3://aggregation-service-testing/e2e_test_outputs//1m_staging_1_sharded.avro.test + // s3://aggregation-service-testing//test-outputs/10k_diff_test_output.avro // 3. Copy/Move above file to - // s3://aggregation-service-testing/testdata/golden//1m_staging_1.avro.golden + // s3://aggregation-service-testing/testdata/golden//10k_diff_test.avro.golden // 4. Update the value of 'goldenLocation' below to the new path - // 20 shards test - // Golden output is the output from e2e_test_outputs/3a73a3c1a29ea67ed0f89b98669cf399 - String goldenLocation = "testdata/golden/2022_08_25/1m_staging_1.avro.golden"; - String outputDataPath = - String.format("e2e_test_outputs/%s/%s", KOKORO_BUILD_ID, "1m_staging_1_sharded.avro.test"); + String goldenLocation = "testdata/golden/2022_10_18/10k_diff_test.avro.golden"; CreateJobRequest createJobRequest = createJobRequest( - DATA_BUCKET, - INPUT_DATA_PREFIX, - DATA_BUCKET, - outputDataPath, - Optional.of(DATA_BUCKET), - Optional.of(OUTPUT_DOMAIN_PREFIX), - /* debugPrivacyBudgetLimit= */ Optional.of(String.valueOf(Integer.MAX_VALUE))); + getTestDataBucket(), + inputKey, + getTestDataBucket(), + outputKey, + Optional.of(getTestDataBucket()), + Optional.of(domainKey)); JsonNode result = submitJobAndWaitForResult(createJobRequest, completionTimeout); assertThat(result.get("result_info").get("return_code").asText()).isEqualTo(SUCCESS.name()); + assertThat(result.get("result_info").get("error_summary").get("error_counts").isEmpty()) + .isTrue(); // Read output avro from s3. ImmutableList aggregatedFacts = - readResultsFromS3(s3BlobStorageClient, avroResultsFileReader, DATA_BUCKET, outputDataPath); + readResultsFromS3( + s3BlobStorageClient, avroResultsFileReader, getTestDataBucket(), outputKey); ImmutableList goldenAggregatedFacts = - readResultsFromS3(s3BlobStorageClient, avroResultsFileReader, DATA_BUCKET, goldenLocation); + readResultsFromS3( + s3BlobStorageClient, avroResultsFileReader, getTestDataBucket(), goldenLocation); MapDifference diffs = ResultDiffer.diffResults(aggregatedFacts.stream(), goldenAggregatedFacts.stream()); diff --git a/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousInvalidCredentialsTest.java b/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousInvalidCredentialsTest.java index b908367d..4e354c20 100644 --- a/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousInvalidCredentialsTest.java +++ b/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousInvalidCredentialsTest.java @@ -48,7 +48,7 @@ public class AwsWorkerContinuousInvalidCredentialsTest { private static final String TESTING_BUCKET = "aggregation-service-testing"; // Input data generated with the following command: - // bazel run java/com/google/aggregate/simulation:SimluationRunner -- \ + // bazel run java/com/google/aggregate/simulation:SimulationRunner -- \ // --aggregatable_report_file_path $PWD/10k_staging_2022_05_20.avro \ // --num_reports 10000 \ // --num_encryption_keys 3 \ @@ -84,8 +84,7 @@ public void e2ePerfTest() throws Exception { TESTING_BUCKET, outputDataPath, Optional.of(TESTING_BUCKET), - Optional.of(INPUT_DOMAIN_PATH), - /* debugPrivacyBudgetLimit= */ Optional.of(String.valueOf(Integer.MAX_VALUE))); + Optional.of(INPUT_DOMAIN_PATH)); JsonNode result = submitJob(createJobRequest, completionTimeout, /* waitForCompletion= */ false); diff --git a/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousNegativePrivacyBudgetTest.java b/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousNegativePrivacyBudgetTest.java index 21629c8c..d0a4f1a7 100644 --- a/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousNegativePrivacyBudgetTest.java +++ b/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousNegativePrivacyBudgetTest.java @@ -62,7 +62,7 @@ public class AwsWorkerContinuousNegativePrivacyBudgetTest { private static final Duration COMPLETION_TIMEOUT = Duration.of(10, ChronoUnit.MINUTES); // Input data generated with the following command: - // bazel run java/com/google/aggregate/simulation:SimluationRunner -- \ + // bazel run java/com/google/aggregate/simulation:SimulationRunner -- \ // --aggregatable_report_file_path $PWD/10k_staging_2022_05_20.avro \ // --num_reports 10000 \ // --num_encryption_keys 3 \ @@ -137,8 +137,7 @@ public void createJobE2ETest() throws Exception { outputBucket, outputKey, /* outputDomainBucketName= */ Optional.of(domainBucket), - /* outputDomainPrefix= */ Optional.of(domainKey), - /* debugPrivacyBudgetLimit= */ Optional.of("1")); + /* outputDomainPrefix= */ Optional.of(domainKey)); JsonNode result = submitJobAndWaitForResult(createJobRequest, COMPLETION_TIMEOUT); assertThat(result.get("result_info").get("return_code").asText()) diff --git a/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousPerfTest.java b/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousPerfTest.java index b0bfa36e..42c48022 100644 --- a/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousPerfTest.java +++ b/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousPerfTest.java @@ -51,7 +51,7 @@ public class AwsWorkerContinuousPerfTest { private static final Path stopwatchWriteLocation = Path.of(System.getenv("STOPWATCH_TEMP_FILE")); // Input data generated with the following command: - // bazel run java/com/google/aggregate/simulation:SimluationRunner -- \ + // bazel run java/com/google/aggregate/simulation:SimulationRunner -- \ // --aggregatable_report_file_path $PWD/1m_staging_2022_05_21.avro \ // --num_reports 1000000 \ // --num_encryption_keys 3 \ @@ -102,8 +102,7 @@ public void e2ePerfTest() throws Exception { TESTING_BUCKET, outputDataPath, /* outputDomainBucketName= */ Optional.of(TESTING_BUCKET), - /* outputDomainPrefix= */ Optional.of(OUTPUT_DOMAIN_PREFIX), - /* debugPrivacyBudgetLimit= */ Optional.of(String.valueOf(Integer.MAX_VALUE))); + /* outputDomainPrefix= */ Optional.of(OUTPUT_DOMAIN_PREFIX)); JsonNode result = submitJobAndWaitForResult(createJobRequest, completionTimeout); diff --git a/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousSmokeTest.java b/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousSmokeTest.java index 263e6eb0..d660000b 100644 --- a/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousSmokeTest.java +++ b/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousSmokeTest.java @@ -25,8 +25,6 @@ import static com.google.aggregate.adtech.worker.util.DebugSupportHelper.getDebugFilePrefix; import static com.google.common.truth.Truth.assertThat; import static com.google.scp.operator.protos.frontend.api.v1.ReturnCodeProto.ReturnCode.SUCCESS; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import com.google.acai.Acai; @@ -66,8 +64,9 @@ *

  • Multiple report input files with default version of reports. Each file follows the naming * format 10k_test_input_${number}.avro. Each test below that relies on default version * reports is supposed to use one of the above input files each - *
  • One report input file with report version 0.1. The file follows the naming format - * 10k_test_input_v0_1.avro + *
  • One report input file with report version 0.0 which is the report version before 0.1 and + * can be generated with --generated_report_version ""` in SimulationRunner. The file follows + * the naming format 10k_test_input_v0_0.avro *
  • One report input file with default version of reports and debug mode enabled on generated * reports. The file follows the naming format 10k_test_input_debug.avro * @@ -79,8 +78,8 @@ *
  • Multiple domain files for default version of reports. Each file follows the naming format * 10k_test_domain_${number}.avro. Each test below that relies on default version reports is * supposed to use one of the above domain files each that matches the report input. - *
  • One domain file for reports with vesion 0.1. The file follows the naming format - * 10k_test_domain_v0_1.avro + *
  • One domain file for reports with version 0.0 which is the report version before 0.1. The + * file follows the naming format 10k_test_domain_v0_0.avro *
  • One domain file for reports in default version with debug mode enabled on the reports. The * file follows the naming format 10k_test_domain_debug.avro * @@ -92,7 +91,7 @@ * *
      *
    • "s3://aggregation-service-testing/$KOKORO_BUILD_ID/test-outputs/10k_test_output_${number}.avro" - *
    • "s3://aggregation-service-testing/$KOKORO_BUILD_ID/test-outputs/10k_test_output_v0_1.avro" + *
    • "s3://aggregation-service-testing/$KOKORO_BUILD_ID/test-outputs/10k_test_output_v0_0.avro" *
    • "s3://aggregation-service-testing/$KOKORO_BUILD_ID/test-outputs/10k_test_output_debug_nodebug.avro" *
    • "s3://aggregation-service-testing/$KOKORO_BUILD_ID/test-outputs/10k_test_output_nodebug_nodebug.avro" *
    • "s3://aggregation-service-testing/$KOKORO_BUILD_ID/test-outputs/10k_test_output_debug_debug.avro" @@ -150,12 +149,12 @@ public void createJobE2ETest() throws Exception { getTestDataBucket(), outputKey, /* outputDomainBucketName= */ Optional.of(getTestDataBucket()), - /* outputDomainPrefix= */ Optional.of(domainKey), - /* debugPrivacyBudgetLimit= */ Optional.empty()); + /* outputDomainPrefix= */ Optional.of(domainKey)); JsonNode result = submitJobAndWaitForResult(createJobRequest, COMPLETION_TIMEOUT); assertThat(result.get("result_info").get("return_code").asText()).isEqualTo(SUCCESS.name()); - assertTrue(result.get("result_info").get("error_summary").get("error_counts").isEmpty()); + assertThat(result.get("result_info").get("error_summary").get("error_counts").isEmpty()) + .isTrue(); // Read output avro from s3. ImmutableList aggregatedFacts = @@ -168,14 +167,14 @@ public void createJobE2ETest() throws Exception { } @Test - public void createJobE2EVersionZeroDotOneTest() throws Exception { + public void createJobE2EVersionZeroDotZeroTest() throws Exception { // End to end testing: - // Follows same idea as createJobE2ETest but uses report with version 0.1 and - // new format shared info + // Follows same idea as createJobE2ETest but uses report with version 0.0 (the report version + // before v0.1) and old format shared info. - var inputKey = String.format("%s/test-inputs/10k_test_input_v0_1.avro", KOKORO_BUILD_ID); - var domainKey = String.format("%s/test-inputs/10k_test_domain_v0_1.avro", KOKORO_BUILD_ID); - var outputKey = String.format("%s/test-outputs/10k_test_output_v0_1.avro", KOKORO_BUILD_ID); + var inputKey = String.format("%s/test-inputs/10k_test_input_v0_0.avro", KOKORO_BUILD_ID); + var domainKey = String.format("%s/test-inputs/10k_test_domain_v0_0.avro", KOKORO_BUILD_ID); + var outputKey = String.format("%s/test-outputs/10k_test_output_v0_0.avro", KOKORO_BUILD_ID); CreateJobRequest createJobRequest = AwsWorkerContinuousTestHelper.createJobRequest( @@ -184,12 +183,12 @@ public void createJobE2EVersionZeroDotOneTest() throws Exception { getTestDataBucket(), outputKey, /* outputDomainBucketName= */ Optional.of(getTestDataBucket()), - /* outputDomainPrefix= */ Optional.of(domainKey), - /* debugPrivacyBudgetLimit= */ Optional.empty()); + /* outputDomainPrefix= */ Optional.of(domainKey)); JsonNode result = submitJobAndWaitForResult(createJobRequest, COMPLETION_TIMEOUT); assertThat(result.get("result_info").get("return_code").asText()).isEqualTo(SUCCESS.name()); - assertTrue(result.get("result_info").get("error_summary").get("error_counts").isEmpty()); + assertThat(result.get("result_info").get("error_summary").get("error_counts").isEmpty()) + .isTrue(); // Read output avro from s3. ImmutableList aggregatedFacts = @@ -220,8 +219,7 @@ public void createNotDebugJobE2EReportDebugEnabledTest() throws Exception { outputKey, /* debugRun= */ false, /* outputDomainBucketName= */ Optional.of(getTestDataBucket()), - /* outputDomainPrefix= */ Optional.of(domainKey), - /* debugPrivacyBudgetLimit= */ Optional.empty()); + /* outputDomainPrefix= */ Optional.of(domainKey)); JsonNode result = submitJobAndWaitForResult(createJobRequest, COMPLETION_TIMEOUT); assertThat(result.get("result_info").get("return_code").asText()).isEqualTo(SUCCESS.name()); @@ -238,9 +236,10 @@ public void createNotDebugJobE2EReportDebugEnabledTest() throws Exception { // The "isAtLeast" assert is set here to accommodate both conditions. assertThat(aggregatedFacts.size()).isAtLeast(DEBUG_DOMAIN_KEY_SIZE); // The debug file shouldn't exist because it's not debug run - assertFalse( - AwsWorkerContinuousTestHelper.checkS3FileExists( - s3BlobStorageClient, getTestDataBucket(), getDebugFilePrefix(outputKey))); + assertThat( + AwsWorkerContinuousTestHelper.checkS3FileExists( + s3BlobStorageClient, getTestDataBucket(), getDebugFilePrefix(outputKey))) + .isFalse(); } /** This test includes sending a debug job and aggregatable reports with debug mode enabled. */ @@ -260,8 +259,7 @@ public void createDebugJobE2EReportDebugModeEnabledTest() throws Exception { outputKey, /* debugRun= */ true, /* outputDomainBucketName= */ Optional.of(getTestDataBucket()), - /* outputDomainPrefix= */ Optional.of(domainKey), - /* debugPrivacyBudgetLimit= */ Optional.empty()); + /* outputDomainPrefix= */ Optional.of(domainKey)); JsonNode result = submitJobAndWaitForResult(createJobRequest, COMPLETION_TIMEOUT); assertThat(result.get("result_info").get("return_code").asText()).isEqualTo(SUCCESS.name()); @@ -306,8 +304,7 @@ public void createDebugJobE2EReportDebugModeDisabledTest() throws Exception { outputKey, /* debugRun= */ true, /* outputDomainBucketName= */ Optional.of(getTestDataBucket()), - /* outputDomainPrefix= */ Optional.of(domainKey), - /* debugPrivacyBudgetLimit= */ Optional.empty()); + /* outputDomainPrefix= */ Optional.of(domainKey)); JsonNode result = submitJobAndWaitForResult(createJobRequest, COMPLETION_TIMEOUT); assertThat(result.get("result_info").get("return_code").asText()).isEqualTo(SUCCESS.name()); @@ -361,12 +358,12 @@ public void createJobE2ETestPrivacyBudgetExhausted() throws Exception { getTestDataBucket(), outputKey, /* outputDomainBucketName= */ Optional.of(getTestDataBucket()), - /* outputDomainPrefix= */ Optional.of(domainKey), - /* debugPrivacyBudgetLimit= */ Optional.empty()); + /* outputDomainPrefix= */ Optional.of(domainKey)); JsonNode result = submitJobAndWaitForResult(createJobRequest1, COMPLETION_TIMEOUT); assertThat(result.get("result_info").get("return_code").asText()).isEqualTo(SUCCESS.name()); - assertTrue(result.get("result_info").get("error_summary").get("error_counts").isEmpty()); + assertThat(result.get("result_info").get("error_summary").get("error_counts").isEmpty()) + .isTrue(); CreateJobRequest createJobRequest2 = createJobRequest1.toBuilder().setJobRequestId(UUID.randomUUID().toString()).build(); @@ -375,7 +372,8 @@ public void createJobE2ETestPrivacyBudgetExhausted() throws Exception { assertThat(result.get("result_info").get("return_code").asText()) .isEqualTo(PRIVACY_BUDGET_EXHAUSTED.name()); - assertTrue(result.get("result_info").get("error_summary").get("error_counts").isEmpty()); + assertThat(result.get("result_info").get("error_summary").get("error_counts").isEmpty()) + .isTrue(); } private static class TestEnv extends AbstractModule { diff --git a/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousSmokeTestChromeReports.java b/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousSmokeTestChromeReports.java index 0c5f53ff..b96c0502 100644 --- a/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousSmokeTestChromeReports.java +++ b/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousSmokeTestChromeReports.java @@ -106,8 +106,7 @@ public void createJobE2ETest() throws Exception { outputBucket, outputKey, Optional.of(outputDomainBucket), - Optional.of(outputDomainKey), - /* debugPrivacyBudgetLimit= */ Optional.of(String.valueOf(Integer.MAX_VALUE))); + Optional.of(outputDomainKey)); JsonNode result = submitJobAndWaitForResult(createJobRequest, COMPLETION_TIMEOUT); assertThat(result.get("result_info").get("return_code").asText()).isEqualTo(SUCCESS.name()); @@ -155,8 +154,7 @@ public void createDebugJobE2ETest() throws Exception { outputKey, /* debugRun= */ true, Optional.of(outputDomainBucket), - Optional.of(outputDomainKey), - /* debugPrivacyBudgetLimit= */ Optional.of(String.valueOf(Integer.MAX_VALUE))); + Optional.of(outputDomainKey)); JsonNode result = submitJobAndWaitForResult(createJobRequest, COMPLETION_TIMEOUT); assertThat(result.get("result_info").get("return_code").asText()).isEqualTo(SUCCESS.name()); diff --git a/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousTestHelper.java b/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousTestHelper.java index 74cfe97d..b76f6860 100644 --- a/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousTestHelper.java +++ b/javatests/com/google/aggregate/adtech/worker/AwsWorkerContinuousTestHelper.java @@ -125,13 +125,10 @@ public static CreateJobRequest createJobRequest( String outputDataBlobBucket, String outputDataBlobPrefix, Optional outputDomainBucketName, - Optional outputDomainPrefix, - Optional debugPrivacyBudgetLimit) { + Optional outputDomainPrefix) { return createDefaultJobRequestBuilder( inputDataBlobBucket, inputDataBlobPrefix, outputDataBlobBucket, outputDataBlobPrefix) - .putAllJobParameters( - getJobParams( - false, outputDomainBucketName, outputDomainPrefix, debugPrivacyBudgetLimit)) + .putAllJobParameters(getJobParams(false, outputDomainBucketName, outputDomainPrefix)) .build(); } @@ -142,13 +139,10 @@ public static CreateJobRequest createJobRequest( String outputDataBlobPrefix, Boolean debugRun, Optional outputDomainBucketName, - Optional outputDomainPrefix, - Optional debugPrivacyBudgetLimit) { + Optional outputDomainPrefix) { return createDefaultJobRequestBuilder( inputDataBlobBucket, inputDataBlobPrefix, outputDataBlobBucket, outputDataBlobPrefix) - .putAllJobParameters( - getJobParams( - debugRun, outputDomainBucketName, outputDomainPrefix, debugPrivacyBudgetLimit)) + .putAllJobParameters(getJobParams(debugRun, outputDomainBucketName, outputDomainPrefix)) .build(); } @@ -177,17 +171,13 @@ private static CreateJobRequest.Builder createDefaultJobRequestBuilder( private static ImmutableMap getJobParams( Boolean debugRun, Optional outputDomainBucketName, - Optional outputDomainPrefix, - Optional debugPrivacyBudgetLimit) { + Optional outputDomainPrefix) { ImmutableMap.Builder jobParams = ImmutableMap.builder(); jobParams.put("attribution_report_to", getAttributionReportTo()); if (debugRun) { jobParams.put("debug_run", "true"); } - if (debugPrivacyBudgetLimit.isPresent()) { - jobParams.put("debug_privacy_budget_limit", debugPrivacyBudgetLimit.get()); - } if (outputDomainPrefix.isPresent() && outputDomainBucketName.isPresent()) { jobParams.put("output_domain_blob_prefix", outputDomainPrefix.get()); jobParams.put("output_domain_bucket_name", outputDomainBucketName.get()); diff --git a/javatests/com/google/aggregate/adtech/worker/BUILD b/javatests/com/google/aggregate/adtech/worker/BUILD index 9e909d4e..dd953c16 100644 --- a/javatests/com/google/aggregate/adtech/worker/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_library", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/adtech/worker/ErrorSummaryAggregatorTest.java b/javatests/com/google/aggregate/adtech/worker/ErrorSummaryAggregatorTest.java index 5797910f..b19ccae5 100644 --- a/javatests/com/google/aggregate/adtech/worker/ErrorSummaryAggregatorTest.java +++ b/javatests/com/google/aggregate/adtech/worker/ErrorSummaryAggregatorTest.java @@ -62,7 +62,9 @@ public void shouldAggregateResultsWithErrors() { .setDetailedErrorMessage("fizz") .build()) .build(), - DecryptionValidationResult.builder().setReport(FakeReportGenerator.generate(0)).build(), + DecryptionValidationResult.builder() + .setReport(FakeReportGenerator.generateWithParam(0, /* reportVersion */ "")) + .build(), DecryptionValidationResult.builder() .addErrorMessage( ErrorMessage.builder() diff --git a/javatests/com/google/aggregate/adtech/worker/ReportDecrypterAndValidatorTest.java b/javatests/com/google/aggregate/adtech/worker/ReportDecrypterAndValidatorTest.java index ae9ff248..f633e429 100644 --- a/javatests/com/google/aggregate/adtech/worker/ReportDecrypterAndValidatorTest.java +++ b/javatests/com/google/aggregate/adtech/worker/ReportDecrypterAndValidatorTest.java @@ -80,7 +80,14 @@ public void testBasic() { assertThat(decryptionValidationResult.report()) .hasValue( FakeReportGenerator.generateWithFixedReportId( - 1, decryptionValidationResult.report().get().sharedInfo().reportId().get())); + 1, + decryptionValidationResult + .report() + .get() + .sharedInfo() + .reportId() + .get(), /* reportVersion */ + "")); } @Test diff --git a/javatests/com/google/aggregate/adtech/worker/aggregation/concurrent/BUILD b/javatests/com/google/aggregate/adtech/worker/aggregation/concurrent/BUILD index 5d2d552f..e2d43d8b 100644 --- a/javatests/com/google/aggregate/adtech/worker/aggregation/concurrent/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/aggregation/concurrent/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( @@ -29,6 +31,7 @@ java_test( "//java/com/google/aggregate/adtech/worker/configs", "//java/com/google/aggregate/adtech/worker/decryption", "//java/com/google/aggregate/adtech/worker/decryption/hybrid", + "//java/com/google/aggregate/adtech/worker/exceptions", "//java/com/google/aggregate/adtech/worker/model", "//java/com/google/aggregate/adtech/worker/model/serdes", "//java/com/google/aggregate/adtech/worker/model/serdes/cbor", @@ -60,6 +63,5 @@ java_test( "//java/external:operator_protos", "//java/external:scp_shared_proto", "//java/external:shared_model", - "@com_google_adm_cloud_scp//java/com/google/scp/operator/shared/dao/metadatadb/model", ], ) diff --git a/javatests/com/google/aggregate/adtech/worker/aggregation/concurrent/ConcurrentAggregationProcessorTest.java b/javatests/com/google/aggregate/adtech/worker/aggregation/concurrent/ConcurrentAggregationProcessorTest.java index af86e7cc..7d1f0d28 100644 --- a/javatests/com/google/aggregate/adtech/worker/aggregation/concurrent/ConcurrentAggregationProcessorTest.java +++ b/javatests/com/google/aggregate/adtech/worker/aggregation/concurrent/ConcurrentAggregationProcessorTest.java @@ -16,12 +16,13 @@ package com.google.aggregate.adtech.worker.aggregation.concurrent; +import static com.google.aggregate.adtech.worker.AggregationWorkerReturnCode.INPUT_DATA_READ_FAILED; import static com.google.aggregate.adtech.worker.AggregationWorkerReturnCode.INVALID_JOB; import static com.google.aggregate.adtech.worker.AggregationWorkerReturnCode.PERMISSION_ERROR; import static com.google.aggregate.adtech.worker.AggregationWorkerReturnCode.PRIVACY_BUDGET_ERROR; import static com.google.aggregate.adtech.worker.AggregationWorkerReturnCode.PRIVACY_BUDGET_EXHAUSTED; +import static com.google.aggregate.adtech.worker.AggregationWorkerReturnCode.RESULT_LOGGING_ERROR; import static com.google.aggregate.adtech.worker.aggregation.concurrent.ConcurrentAggregationProcessor.JOB_PARAM_ATTRIBUTION_REPORT_TO; -import static com.google.aggregate.adtech.worker.aggregation.concurrent.ConcurrentAggregationProcessor.JOB_PARAM_DEBUG_PRIVACY_BUDGET_LIMIT; import static com.google.aggregate.adtech.worker.aggregation.concurrent.ConcurrentAggregationProcessor.JOB_PARAM_DEBUG_PRIVACY_EPSILON; import static com.google.aggregate.adtech.worker.aggregation.concurrent.ConcurrentAggregationProcessor.JOB_PARAM_DEBUG_RUN; import static com.google.aggregate.adtech.worker.aggregation.concurrent.ConcurrentAggregationProcessor.JOB_PARAM_OUTPUT_DOMAIN_BLOB_PREFIX; @@ -34,8 +35,6 @@ import static com.google.common.truth.Truth8.assertThat; import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; import static com.google.scp.operator.protos.shared.backend.JobErrorCategoryProto.JobErrorCategory.GENERAL_ERROR; -import static com.google.scp.operator.protos.shared.backend.ReturnCodeProto.ReturnCode.INPUT_DATA_READ_FAILED; -import static com.google.scp.operator.protos.shared.backend.ReturnCodeProto.ReturnCode.OUTPUT_DATAWRITE_FAILED; import static com.google.scp.operator.protos.shared.backend.ReturnCodeProto.ReturnCode.SUCCESS; import static java.nio.charset.StandardCharsets.US_ASCII; import static java.nio.file.StandardOpenOption.CREATE; @@ -51,7 +50,6 @@ import com.google.aggregate.adtech.worker.Annotations.FailJobOnPbsException; import com.google.aggregate.adtech.worker.Annotations.NonBlockingThreadPool; import com.google.aggregate.adtech.worker.ResultLogger; -import com.google.aggregate.adtech.worker.ResultLogger.ResultLogException; import com.google.aggregate.adtech.worker.aggregation.domain.OutputDomainProcessor; import com.google.aggregate.adtech.worker.aggregation.domain.TextOutputDomainProcessor; import com.google.aggregate.adtech.worker.aggregation.engine.AggregationEngine; @@ -67,6 +65,8 @@ import com.google.aggregate.adtech.worker.decryption.DeserializingReportDecrypter; import com.google.aggregate.adtech.worker.decryption.RecordDecrypter; import com.google.aggregate.adtech.worker.decryption.hybrid.HybridDecryptionModule; +import com.google.aggregate.adtech.worker.exceptions.AggregationJobProcessException; +import com.google.aggregate.adtech.worker.exceptions.ResultLogException; import com.google.aggregate.adtech.worker.model.AggregatedFact; import com.google.aggregate.adtech.worker.model.AvroRecordEncryptedReportConverter; import com.google.aggregate.adtech.worker.model.DebugBucketAnnotation; @@ -140,7 +140,6 @@ public class ConcurrentAggregationProcessorTest { private static final Instant FIXED_TIME = Instant.parse("2021-01-01T00:00:00Z"); - private static final Integer DEBUG_PRIVACY_BUDGET_LIMIT = 5; @Rule public final Acai acai = new Acai(TestEnv.class); @Rule public final TemporaryFolder testWorkingDir = new TemporaryFolder(); @@ -189,7 +188,7 @@ private static void assertJobResultsEqualsReturnCode(JobResult actual, JobResult private EncryptedReport generateEncryptedReport(int param) { String keyId = UUID.randomUUID().toString(); - Report report = FakeReportGenerator.generate(param); + Report report = FakeReportGenerator.generateWithParam(param, /* reportVersion */ ""); String sharedInfoString1 = sharedInfoSerdes.reverse().convert(Optional.of(report.sharedInfo())); try { ByteSource firstReportBytes = @@ -225,12 +224,7 @@ public void setUp() throws Exception { RequestInfo requestInfo = ctx.requestInfo(); RequestInfo newRequestInfo = requestInfo.toBuilder() - .putAllJobParameters( - combineJobParams( - requestInfo.getJobParameters(), - ImmutableMap.of( - JOB_PARAM_DEBUG_PRIVACY_BUDGET_LIMIT, - DEBUG_PRIVACY_BUDGET_LIMIT.toString()))) + .putAllJobParameters(requestInfo.getJobParameters()) // Simulating 2 shards of input. .setInputDataBucketName(reportsDirectory.toAbsolutePath().toString()) .setInputDataBlobPrefix("") @@ -423,9 +417,9 @@ public void aggregate_withOutputDomain_thresholding_debugEpsilonOutOfRange() thr fakeValidator.setNextShouldReturnError(ImmutableList.of(false, false, false, false).iterator()); fakeNoiseApplierSupplier.setFakeNoiseApplier(new ConstantNoiseApplier(-3)); - JobResult jobResultProcessor = processor.process(ctx); - - assertThat(jobResultProcessor.resultInfo().getReturnCode()).isEqualTo(INVALID_JOB.toString()); + AggregationJobProcessException ex = + assertThrows(AggregationJobProcessException.class, () -> processor.process(ctx)); + assertThat(ex.getCode()).isEqualTo(INVALID_JOB); } @Test @@ -678,17 +672,10 @@ public void aggregate_withOutputDomain_domainNotReadable() throws Exception { fakeValidator.setNextShouldReturnError(ImmutableList.of(false, false, false, false).iterator()); fakeNoiseApplierSupplier.setFakeNoiseApplier(new ConstantNoiseApplier(0)); - JobResult jobResultProcessor = processor.process(ctx); - - JobResult expectedJobResult = - this.expectedJobResult.toBuilder() - .setResultInfo( - resultInfoBuilder - .setReturnCode(INPUT_DATA_READ_FAILED.name()) - .setReturnMessage("") - .build()) - .build(); - assertJobResultsEqualsIgnoreReturnMessage(jobResultProcessor, expectedJobResult); + AggregationJobProcessException ex = + assertThrows(AggregationJobProcessException.class, () -> processor.process(ctx)); + assertThat(ex.getCode()).isEqualTo(INPUT_DATA_READ_FAILED); + assertThat(ex.getMessage()).contains("Exception while reading domain input data."); } @Test @@ -786,18 +773,10 @@ public void process_inputReadFailedCodeWhenBadShardThrows() throws Exception { Path badDataShard = reportsDirectory.resolve("reports_bad.avro"); Files.writeString(badDataShard, "Bad data", US_ASCII, WRITE, CREATE); fakeValidator.setNextShouldReturnError(ImmutableList.of(false, false, false, false).iterator()); - - JobResult jobResultProcessor = processor.process(ctx); - - JobResult expectedJobResult = - this.expectedJobResult.toBuilder() - .setResultInfo( - resultInfoBuilder - .setReturnCode(INPUT_DATA_READ_FAILED.name()) - .setReturnMessage("") - .build()) - .build(); - assertJobResultsEqualsIgnoreReturnMessage(jobResultProcessor, expectedJobResult); + AggregationJobProcessException ex = + assertThrows(AggregationJobProcessException.class, () -> processor.process(ctx)); + assertThat(ex.getCode()).isEqualTo(INPUT_DATA_READ_FAILED); + assertThat(ex.getMessage()).contains("Exception while reading reports input data."); } @Test @@ -805,18 +784,9 @@ public void process_outputWriteFailedCodeWhenResultLoggerThrows() throws Excepti fakeValidator.setNextShouldReturnError(ImmutableList.of(false, false, false, false).iterator()); fakeNoiseApplierSupplier.setFakeNoiseApplier(new ConstantNoiseApplier(0)); resultLogger.setShouldThrow(true); - - JobResult jobResultProcessor = processor.process(ctx); - - JobResult expectedJobResult = - this.expectedJobResult.toBuilder() - .setResultInfo( - resultInfoBuilder - .setReturnCode(OUTPUT_DATAWRITE_FAILED.name()) - .setReturnMessage("") - .build()) - .build(); - assertJobResultsEqualsIgnoreReturnMessage(jobResultProcessor, expectedJobResult); + AggregationJobProcessException ex = + assertThrows(AggregationJobProcessException.class, () -> processor.process(ctx)); + assertThat(ex.getCode()).isEqualTo(RESULT_LOGGING_ERROR); } @Test @@ -824,18 +794,9 @@ public void process_decryptionKeyFetchFailedWithPermissionDeniedReason() throws fakeValidator.setNextShouldReturnError(ImmutableList.of(false, false, false, false).iterator()); fakeNoiseApplierSupplier.setFakeNoiseApplier(new ConstantNoiseApplier(0)); fakeDecryptionKeyService.setShouldThrowPermissionException(true); - - JobResult actualJobResult = processor.process(ctx); - - JobResult expectedJobResult = - this.expectedJobResult.toBuilder() - .setResultInfo( - resultInfoBuilder - .setReturnCode(PERMISSION_ERROR.name()) - .setReturnMessage("") - .build()) - .build(); - assertJobResultsEqualsReturnCode(actualJobResult, expectedJobResult); + AggregationJobProcessException ex = + assertThrows(AggregationJobProcessException.class, () -> processor.process(ctx)); + assertThat(ex.getCode()).isEqualTo(PERMISSION_ERROR); } @Test @@ -854,30 +815,10 @@ public void process_decryptionKeyFetchFailedOtherReasons() throws Exception { assertJobResultsEqualsReturnCode(actualJobResult, expectedJobResult); } - /** - * Test to ensure that if a job is reprocessed by the same worker then no errors should occur. - * This scenario can when exceptions occur in processing, preventing the stopwatch from being - * stopped. - */ - @Test - public void process_moreThanOnceShouldNotCauseError() throws Exception { - // Create an error on the first run, starting a stopwatch but not stopping it - Path badDataShard = reportsDirectory.resolve("reports_bad.avro"); - Files.writeString(badDataShard, "Bad data", US_ASCII, WRITE, CREATE); - fakeValidator.setNextShouldReturnError(ImmutableList.of(false, false, false, false).iterator()); - processor.process(ctx); - - // No exception should be thrown when the job is processed again - fakeValidator.setNextShouldReturnError(ImmutableList.of(false, false, false, false).iterator()); - processor.process(ctx); - - // Test passes if no exception is thrown - } - @Test public void processingWithWrongSharedInfo() throws Exception { String keyId = UUID.randomUUID().toString(); - Report report = FakeReportGenerator.generate(1); + Report report = FakeReportGenerator.generateWithParam(1, /* reportVersion */ ""); // Encrypt with a different sharedInfo than what is provided with the report so that decryption // fails String sharedInfoForEncryption = "foobarbaz"; @@ -934,23 +875,9 @@ public void aggregate_withPrivacyBudgeting_noBudget() throws Exception { fakePrivacyBudgetingServiceBridge); fakeValidator.setNextShouldReturnError(ImmutableList.of(false, false, false, false).iterator()); fakeNoiseApplierSupplier.setFakeNoiseApplier(new ConstantNoiseApplier(0)); - - JobResult jobResultProcessor = processor.process(ctx); - - JobResult expectedJobResult = - this.expectedJobResult.toBuilder() - .setResultInfo( - resultInfoBuilder - .setReturnCode(PRIVACY_BUDGET_EXHAUSTED.name()) - .setReturnMessage( - "Insufficient privacy budget for " - + "one or more aggregatable reports." - + " No aggregatable report can" - + " appear in more than one batch or " - + "contribute to more than one summary report.") - .build()) - .build(); - assertThat(jobResultProcessor).isEqualTo(expectedJobResult); + AggregationJobProcessException ex = + assertThrows(AggregationJobProcessException.class, () -> processor.process(ctx)); + assertThat(ex.getCode()).isEqualTo(PRIVACY_BUDGET_EXHAUSTED); } @Test @@ -978,8 +905,6 @@ public void aggregate_withPrivacyBudgeting() throws Exception { // Check that the right attributionReportTo and debugPrivacyBudgetLimit were sent to the bridge assertThat(fakePrivacyBudgetingServiceBridge.getLastAttributionReportToSent()) .hasValue(ctx.requestInfo().getJobParameters().get(JOB_PARAM_ATTRIBUTION_REPORT_TO)); - assertThat(fakePrivacyBudgetingServiceBridge.getLastDebugPrivacyBudgetLimitSent()) - .hasValue(DEBUG_PRIVACY_BUDGET_LIMIT); } /** @@ -996,13 +921,9 @@ public void aggregate_withPrivacyBudgeting_exception_failJobOnPbsException() thr fakePrivacyBudgetingServiceBridge); fakeValidator.setNextShouldReturnError(ImmutableList.of(false, false, false, false).iterator()); fakeNoiseApplierSupplier.setFakeNoiseApplier(new ConstantNoiseApplier(0)); - - JobResult jobResultProcessor = processor.process(ctx); - - assertThat(jobResultProcessor.resultInfo().getReturnCode()) - .isEqualTo(PRIVACY_BUDGET_ERROR.name()); - assertThat(jobResultProcessor.resultInfo().getReturnMessage()) - .contains("Exception while consuming privacy budget:"); + AggregationJobProcessException ex = + assertThrows(AggregationJobProcessException.class, () -> processor.process(ctx)); + assertThat(ex.getCode()).isEqualTo(PRIVACY_BUDGET_ERROR); } @Test @@ -1016,23 +937,52 @@ public void aggregate_withPrivacyBudgeting_oneBudgetMissing() throws Exception { fakePrivacyBudgetingServiceBridge); fakeValidator.setNextShouldReturnError(ImmutableList.of(false, false, false, false).iterator()); fakeNoiseApplierSupplier.setFakeNoiseApplier(new ConstantNoiseApplier(0)); + AggregationJobProcessException ex = + assertThrows(AggregationJobProcessException.class, () -> processor.process(ctx)); + assertThat(ex.getCode()).isEqualTo(PRIVACY_BUDGET_EXHAUSTED); + } - JobResult jobResultProcessor = processor.process(ctx); + /** + * Test that the worker fails the job if an exception occurs when the reports bucket is + * nonexistent. + */ + @Test + public void aggregate_withNonExistentBucket() throws Exception { + Path nonExistentReportsDirectory = + testWorkingDir.getRoot().toPath().resolve("nonExistentBucket"); + ctx = + ctx.toBuilder() + .setRequestInfo( + ctx.requestInfo().toBuilder() + .setInputDataBucketName(nonExistentReportsDirectory.toAbsolutePath().toString()) + .setInputDataBlobPrefix("") + .build()) + .build(); + // TODO(b/258078789): Passing nonexistent reports folder should throw + // TODO(b/258082317): Add assertion on return message. + AggregationJobProcessException ex = + assertThrows(AggregationJobProcessException.class, () -> processor.process(ctx)); + assertThat(ex.getCode()).isEqualTo(INPUT_DATA_READ_FAILED); + } - JobResult expectedJobResult = - this.expectedJobResult.toBuilder() - .setResultInfo( - resultInfoBuilder - .setReturnCode(PRIVACY_BUDGET_EXHAUSTED.name()) - .setReturnMessage( - "Insufficient privacy budget for " - + "one or more aggregatable reports." - + " No aggregatable report can" - + " appear in more than one batch or " - + "contribute to more than one summary report.") + /** + * Test that the worker fails the job if an exception occurs when the report file path is + * nonexistent. + */ + @Test + public void aggregate_withNonExistentReportFile() throws Exception { + ctx = + ctx.toBuilder() + .setRequestInfo( + ctx.requestInfo().toBuilder() + .setInputDataBucketName(reportsDirectory.toAbsolutePath().toString()) + .setInputDataBlobPrefix("nonExistentReport.avro") .build()) .build(); - assertThat(jobResultProcessor).isEqualTo(expectedJobResult); + AggregationJobProcessException ex = + assertThrows(AggregationJobProcessException.class, () -> processor.process(ctx)); + assertThat(ex.getCode()).isEqualTo(INPUT_DATA_READ_FAILED); + assertThat(ex.getMessage()).contains("No report shards found for location"); } private void writeOutputDomain(Path outputDomainPath, String... keys) throws IOException { @@ -1078,12 +1028,9 @@ private void setPrivacyBudgetingServiceBridgeImpl(PrivacyBudgetingServiceBridge @Override public ImmutableList consumePrivacyBudget( - ImmutableList budgetsToConsume, - String attributionReportTo, - Optional debugPrivacyBudgetLimit) + ImmutableList budgetsToConsume, String attributionReportTo) throws PrivacyBudgetingServiceBridgeException { - return wrappedImpl.consumePrivacyBudget( - budgetsToConsume, attributionReportTo, debugPrivacyBudgetLimit); + return wrappedImpl.consumePrivacyBudget(budgetsToConsume, attributionReportTo); } } diff --git a/javatests/com/google/aggregate/adtech/worker/aggregation/domain/AvroOutputDomainProcessorTest.java b/javatests/com/google/aggregate/adtech/worker/aggregation/domain/AvroOutputDomainProcessorTest.java index 702fed97..df333317 100644 --- a/javatests/com/google/aggregate/adtech/worker/aggregation/domain/AvroOutputDomainProcessorTest.java +++ b/javatests/com/google/aggregate/adtech/worker/aggregation/domain/AvroOutputDomainProcessorTest.java @@ -24,7 +24,7 @@ import com.google.acai.Acai; import com.google.aggregate.adtech.worker.Annotations.BlockingThreadPool; import com.google.aggregate.adtech.worker.Annotations.NonBlockingThreadPool; -import com.google.aggregate.adtech.worker.aggregation.domain.OutputDomainProcessor.DomainReadException; +import com.google.aggregate.adtech.worker.exceptions.DomainReadException; import com.google.aggregate.protocol.avro.AvroOutputDomainRecord; import com.google.aggregate.protocol.avro.AvroOutputDomainWriter; import com.google.aggregate.protocol.avro.AvroOutputDomainWriterFactory; diff --git a/javatests/com/google/aggregate/adtech/worker/aggregation/domain/BUILD b/javatests/com/google/aggregate/adtech/worker/aggregation/domain/BUILD index e241287f..29bfbe6e 100644 --- a/javatests/com/google/aggregate/adtech/worker/aggregation/domain/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/aggregation/domain/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( @@ -21,6 +23,7 @@ java_test( "//java/com/google/aggregate/adtech/worker", "//java/com/google/aggregate/adtech/worker/aggregation/domain", "//java/com/google/aggregate/adtech/worker/aggregation/domain:text_domain", + "//java/com/google/aggregate/adtech/worker/exceptions", "//java/com/google/aggregate/adtech/worker/model", "//java/com/google/aggregate/adtech/worker/util", "//java/external:acai", @@ -39,6 +42,7 @@ java_test( "//java/com/google/aggregate/adtech/worker", "//java/com/google/aggregate/adtech/worker/aggregation/domain", "//java/com/google/aggregate/adtech/worker/aggregation/domain:avro_domain", + "//java/com/google/aggregate/adtech/worker/exceptions", "//java/com/google/aggregate/adtech/worker/model", "//java/com/google/aggregate/protocol/avro:avro_output_domain", "//java/external:acai", diff --git a/javatests/com/google/aggregate/adtech/worker/aggregation/domain/TextOutputDomainProcessorTest.java b/javatests/com/google/aggregate/adtech/worker/aggregation/domain/TextOutputDomainProcessorTest.java index 6932c49d..e6fb48c7 100644 --- a/javatests/com/google/aggregate/adtech/worker/aggregation/domain/TextOutputDomainProcessorTest.java +++ b/javatests/com/google/aggregate/adtech/worker/aggregation/domain/TextOutputDomainProcessorTest.java @@ -28,7 +28,7 @@ import com.google.acai.Acai; import com.google.aggregate.adtech.worker.Annotations.BlockingThreadPool; import com.google.aggregate.adtech.worker.Annotations.NonBlockingThreadPool; -import com.google.aggregate.adtech.worker.aggregation.domain.OutputDomainProcessor.DomainReadException; +import com.google.aggregate.adtech.worker.exceptions.DomainReadException; import com.google.common.base.Ticker; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; diff --git a/javatests/com/google/aggregate/adtech/worker/aggregation/engine/AggregationEngineTest.java b/javatests/com/google/aggregate/adtech/worker/aggregation/engine/AggregationEngineTest.java index ec85606a..17b2282b 100644 --- a/javatests/com/google/aggregate/adtech/worker/aggregation/engine/AggregationEngineTest.java +++ b/javatests/com/google/aggregate/adtech/worker/aggregation/engine/AggregationEngineTest.java @@ -47,7 +47,7 @@ public void setUp() { @Test public void oneReportOneFact() { - Report report = FakeReportGenerator.generate(/* bucket= */ 1); + Report report = FakeReportGenerator.generateWithParam(/* bucket= */ 1, /* reportVersion */ ""); engine.accept(report); ImmutableMap aggregation = engine.makeAggregation(); @@ -61,8 +61,10 @@ public void oneReportOneFact() { @Test public void twoReportNoFacts() { ImmutableList factList = ImmutableList.of(); - Report firstReport = FakeReportGenerator.generate(/* facts= */ factList); - Report secondReport = FakeReportGenerator.generate(/* facts= */ factList); + Report firstReport = + FakeReportGenerator.generateWithFactList(/* facts= */ factList, /* reportVersion */ ""); + Report secondReport = + FakeReportGenerator.generateWithFactList(/* facts= */ factList, /* reportVersion */ ""); engine.accept(firstReport); engine.accept(secondReport); @@ -76,9 +78,11 @@ public void twoReportSameFactKey() { Fact firstReportFact = FakeFactGenerator.generate(/* bucket= */ 2, /* value= */ 2); Fact secondReportFact = FakeFactGenerator.generate(/* bucket= */ 2, /* value= */ 5); Report firstReport = - FakeReportGenerator.generate(/* facts= */ ImmutableList.of(firstReportFact)); + FakeReportGenerator.generateWithFactList( + /* facts= */ ImmutableList.of(firstReportFact), /* reportVersion */ ""); Report secondReport = - FakeReportGenerator.generate(/* facts= */ ImmutableList.of(secondReportFact)); + FakeReportGenerator.generateWithFactList( + /* facts= */ ImmutableList.of(secondReportFact), /* reportVersion */ ""); engine.accept(firstReport); engine.accept(secondReport); @@ -95,8 +99,9 @@ public void oneReportMultipleFacts() { Fact secondFact = FakeFactGenerator.generate(/* bucket= */ 1, /* value= */ 5); Fact thirdFact = FakeFactGenerator.generate(/* bucket= */ 2, /* value= */ 10); Report report = - FakeReportGenerator.generate( - /* facts= */ ImmutableList.of(firstFact, secondFact, thirdFact)); + FakeReportGenerator.generateWithFactList( + /* facts= */ ImmutableList.of(firstFact, secondFact, thirdFact), /* reportVersion */ + ""); engine.accept(report); ImmutableMap aggregation = engine.makeAggregation(); @@ -116,13 +121,19 @@ public void multipleReportsMultipleFacts() { Fact secondReportSecondFact = FakeFactGenerator.generate(/* bucket= */ 3, /* value= */ 10); Fact secondReportThirdFact = FakeFactGenerator.generate(/* bucket= */ 4, /* value= */ 20); Report firstReport = - FakeReportGenerator.generate( + FakeReportGenerator.generateWithFactList( /* facts= */ ImmutableList.of( - firstReportFirstFact, firstReportSecondFact, firstReportThirdFact)); + firstReportFirstFact, + firstReportSecondFact, + firstReportThirdFact), /* reportVersion */ + ""); Report secondReport = - FakeReportGenerator.generate( + FakeReportGenerator.generateWithFactList( /* facts= */ ImmutableList.of( - secondReportFirstFact, secondReportSecondFact, secondReportThirdFact)); + secondReportFirstFact, + secondReportSecondFact, + secondReportThirdFact), /* reportVersion */ + ""); engine.accept(firstReport); engine.accept(secondReport); @@ -142,10 +153,13 @@ public void multipleReportsMultipleFacts() { @Test public void privacyBudgetUnits() { - Report report = FakeReportGenerator.generate(/* bucket= */ 1); - Report reportDuplicate = FakeReportGenerator.generate(/* bucket= */ 1); - Report secondReport = FakeReportGenerator.generate(/* bucket= */ 4000); - Report thirdReport = FakeReportGenerator.generate(/* bucket= */ 100); + Report report = FakeReportGenerator.generateWithParam(/* bucket= */ 1, /* reportVersion */ ""); + Report reportDuplicate = + FakeReportGenerator.generateWithParam(/* bucket= */ 1, /* reportVersion */ ""); + Report secondReport = + FakeReportGenerator.generateWithParam(/* bucket= */ 4000, /* reportVersion */ ""); + Report thirdReport = + FakeReportGenerator.generateWithParam(/* bucket= */ 100, /* reportVersion */ ""); engine.accept(report); engine.accept(reportDuplicate); diff --git a/javatests/com/google/aggregate/adtech/worker/aggregation/engine/BUILD b/javatests/com/google/aggregate/adtech/worker/aggregation/engine/BUILD index 82f7ed72..affb4d5e 100644 --- a/javatests/com/google/aggregate/adtech/worker/aggregation/engine/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/aggregation/engine/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/adtech/worker/aggregation/privacy/BUILD b/javatests/com/google/aggregate/adtech/worker/aggregation/privacy/BUILD index c99dca78..b819e03c 100644 --- a/javatests/com/google/aggregate/adtech/worker/aggregation/privacy/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/aggregation/privacy/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/adtech/worker/aggregation/privacy/FakePrivacyBudgetingServiceBridgeTest.java b/javatests/com/google/aggregate/adtech/worker/aggregation/privacy/FakePrivacyBudgetingServiceBridgeTest.java index b6721f5f..aaebef3b 100644 --- a/javatests/com/google/aggregate/adtech/worker/aggregation/privacy/FakePrivacyBudgetingServiceBridgeTest.java +++ b/javatests/com/google/aggregate/adtech/worker/aggregation/privacy/FakePrivacyBudgetingServiceBridgeTest.java @@ -22,7 +22,6 @@ import com.google.aggregate.adtech.worker.aggregation.privacy.PrivacyBudgetingServiceBridge.PrivacyBudgetUnit; import com.google.common.collect.ImmutableList; import java.time.Instant; -import java.util.Optional; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -34,7 +33,6 @@ public class FakePrivacyBudgetingServiceBridgeTest { private FakePrivacyBudgetingServiceBridge privacyBudgetingService; private String attributionReportTo = "foo.com"; - private Optional debugPrivacyBudgetLimit = Optional.of(6); // Not relevant for the test private final PrivacyBudgetUnit firstId = PrivacyBudgetUnit.create("foo", Instant.ofEpochMilli(1000)); @@ -52,13 +50,11 @@ public void noBudget() throws Exception { ImmutableList missingBudget = privacyBudgetingService.consumePrivacyBudget( - ImmutableList.of(firstId), attributionReportTo, debugPrivacyBudgetLimit); + ImmutableList.of(firstId), attributionReportTo); assertThat(missingBudget).containsExactly(firstId); assertThat(privacyBudgetingService.getLastAttributionReportToSent()) .hasValue(attributionReportTo); - assertThat(privacyBudgetingService.getLastDebugPrivacyBudgetLimitSent()) - .isEqualTo(debugPrivacyBudgetLimit); } @Test @@ -67,13 +63,11 @@ public void oneBudgetMissing() throws Exception { ImmutableList missingBudget = privacyBudgetingService.consumePrivacyBudget( - ImmutableList.of(firstId, secondId), attributionReportTo, debugPrivacyBudgetLimit); + ImmutableList.of(firstId, secondId), attributionReportTo); assertThat(missingBudget).containsExactly(secondId); assertThat(privacyBudgetingService.getLastAttributionReportToSent()) .hasValue(attributionReportTo); - assertThat(privacyBudgetingService.getLastDebugPrivacyBudgetLimitSent()) - .isEqualTo(debugPrivacyBudgetLimit); } @Test @@ -83,13 +77,11 @@ public void success() throws Exception { ImmutableList missingBudget = privacyBudgetingService.consumePrivacyBudget( - ImmutableList.of(firstId, secondId), attributionReportTo, debugPrivacyBudgetLimit); + ImmutableList.of(firstId, secondId), attributionReportTo); assertThat(missingBudget).isEmpty(); assertThat(privacyBudgetingService.getLastAttributionReportToSent()) .hasValue(attributionReportTo); - assertThat(privacyBudgetingService.getLastDebugPrivacyBudgetLimitSent()) - .isEqualTo(debugPrivacyBudgetLimit); } @Test @@ -99,10 +91,10 @@ public void budgetDepleted() throws Exception { ImmutableList missingBudgetFirst = privacyBudgetingService.consumePrivacyBudget( - ImmutableList.of(firstId, secondId), attributionReportTo, debugPrivacyBudgetLimit); + ImmutableList.of(firstId, secondId), attributionReportTo); ImmutableList missingBudgetSecond = privacyBudgetingService.consumePrivacyBudget( - ImmutableList.of(firstId, secondId), attributionReportTo, debugPrivacyBudgetLimit); + ImmutableList.of(firstId, secondId), attributionReportTo); assertThat(missingBudgetFirst).isEmpty(); assertThat(missingBudgetSecond).containsExactly(firstId); @@ -115,11 +107,11 @@ public void budgetRestored() throws Exception { ImmutableList missingBudgetFirst = privacyBudgetingService.consumePrivacyBudget( - ImmutableList.of(firstId, secondId), attributionReportTo, debugPrivacyBudgetLimit); + ImmutableList.of(firstId, secondId), attributionReportTo); privacyBudgetingService.setPrivacyBudget(firstId, 1); ImmutableList missingBudgetSecond = privacyBudgetingService.consumePrivacyBudget( - ImmutableList.of(firstId, secondId), attributionReportTo, debugPrivacyBudgetLimit); + ImmutableList.of(firstId, secondId), attributionReportTo); assertThat(missingBudgetFirst).isEmpty(); assertThat(missingBudgetFirst).isEmpty(); diff --git a/javatests/com/google/aggregate/adtech/worker/aggregation/privacy/HttpPrivacyBudgetingServiceBridgeTest.java b/javatests/com/google/aggregate/adtech/worker/aggregation/privacy/HttpPrivacyBudgetingServiceBridgeTest.java index 5c5756e8..8a067223 100644 --- a/javatests/com/google/aggregate/adtech/worker/aggregation/privacy/HttpPrivacyBudgetingServiceBridgeTest.java +++ b/javatests/com/google/aggregate/adtech/worker/aggregation/privacy/HttpPrivacyBudgetingServiceBridgeTest.java @@ -28,7 +28,6 @@ import com.google.scp.coordinator.privacy.budgeting.model.ConsumePrivacyBudgetResponse; import com.google.scp.operator.cpio.distributedprivacybudgetclient.DistributedPrivacyBudgetClient; import java.time.Instant; -import java.util.Optional; import javax.inject.Inject; import org.junit.Rule; import org.junit.Test; @@ -58,9 +57,7 @@ public void noBudget() throws Exception { ImmutableList missingBudget = privacyBudgetingService.consumePrivacyBudget( - ImmutableList.of(firstId, secondId), - attributionReportTo, - /* debugPrivacyBudgetLimit= */ Optional.empty()); + ImmutableList.of(firstId, secondId), attributionReportTo); assertThat(missingBudget).containsExactly(firstId, secondId); assertThat(fakeHttpPrivacyBudgetingServiceClient.lastRequestSent) @@ -80,9 +77,7 @@ public void oneBudgetMissing() throws Exception { ImmutableList missingBudget = privacyBudgetingService.consumePrivacyBudget( - ImmutableList.of(firstId, secondId), - attributionReportTo, - /* debugPrivacyBudgetLimit= */ Optional.empty()); + ImmutableList.of(firstId, secondId), attributionReportTo); assertThat(missingBudget).containsExactly(firstId); assertThat(fakeHttpPrivacyBudgetingServiceClient.lastRequestSent) @@ -101,9 +96,7 @@ public void success() throws Exception { ImmutableList missingBudget = privacyBudgetingService.consumePrivacyBudget( - ImmutableList.of(firstId, secondId), - attributionReportTo, - /* debugPrivacyBudgetLimit= */ Optional.empty()); + ImmutableList.of(firstId, secondId), attributionReportTo); assertThat(missingBudget).isEmpty(); assertThat(fakeHttpPrivacyBudgetingServiceClient.lastRequestSent) @@ -116,26 +109,6 @@ public void success() throws Exception { .build()); } - @Test - public void usesDebugLimitIfPresent() throws Exception { - fakeHttpPrivacyBudgetingServiceClient.setExhaustedUnits(ImmutableList.of()); - Optional debugPrivacyBudgetLimit = Optional.of(5); - - ImmutableList missingBudget = - privacyBudgetingService.consumePrivacyBudget( - ImmutableList.of(firstId, secondId), attributionReportTo, debugPrivacyBudgetLimit); - - assertThat(missingBudget).isEmpty(); - assertThat(fakeHttpPrivacyBudgetingServiceClient.lastRequestSent) - .isEqualTo( - ConsumePrivacyBudgetRequest.builder() - .privacyBudgetUnits( - ImmutableList.of(workerToScpUnit(firstId), workerToScpUnit(secondId))) - .attributionReportTo(attributionReportTo) - .privacyBudgetLimit(debugPrivacyBudgetLimit.get()) - .build()); - } - private static com.google.scp.coordinator.privacy.budgeting.model.PrivacyBudgetUnit workerToScpUnit(PrivacyBudgetUnit unit) { return com.google.scp.coordinator.privacy.budgeting.model.PrivacyBudgetUnit.builder() diff --git a/javatests/com/google/aggregate/adtech/worker/configs/BUILD b/javatests/com/google/aggregate/adtech/worker/configs/BUILD index 79064772..e26b78e0 100644 --- a/javatests/com/google/aggregate/adtech/worker/configs/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/configs/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/adtech/worker/decryption/BUILD b/javatests/com/google/aggregate/adtech/worker/decryption/BUILD index f3e74d43..d22e768c 100644 --- a/javatests/com/google/aggregate/adtech/worker/decryption/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/decryption/BUILD @@ -12,4 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) diff --git a/javatests/com/google/aggregate/adtech/worker/decryption/DeserializingReportDecrypterTest.java b/javatests/com/google/aggregate/adtech/worker/decryption/DeserializingReportDecrypterTest.java index fe7daff9..81381ce4 100644 --- a/javatests/com/google/aggregate/adtech/worker/decryption/DeserializingReportDecrypterTest.java +++ b/javatests/com/google/aggregate/adtech/worker/decryption/DeserializingReportDecrypterTest.java @@ -75,7 +75,7 @@ public class DeserializingReportDecrypterTest { @Before public void setUp() throws Exception { - report = FakeReportGenerator.generate(1); + report = FakeReportGenerator.generateWithParam(1, /* reportVersion */ ""); sharedInfo = sharedInfoSerdes.reverse().convert(Optional.of(report.sharedInfo())); encryptReport(); } diff --git a/javatests/com/google/aggregate/adtech/worker/decryption/hybrid/BUILD b/javatests/com/google/aggregate/adtech/worker/decryption/hybrid/BUILD index 33ee1813..846c5905 100644 --- a/javatests/com/google/aggregate/adtech/worker/decryption/hybrid/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/decryption/hybrid/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/adtech/worker/model/BUILD b/javatests/com/google/aggregate/adtech/worker/model/BUILD index 2443ecd0..ff302110 100644 --- a/javatests/com/google/aggregate/adtech/worker/model/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/model/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/adtech/worker/model/DecryptionValidationResultTest.java b/javatests/com/google/aggregate/adtech/worker/model/DecryptionValidationResultTest.java index c2cfc631..344b5fec 100644 --- a/javatests/com/google/aggregate/adtech/worker/model/DecryptionValidationResultTest.java +++ b/javatests/com/google/aggregate/adtech/worker/model/DecryptionValidationResultTest.java @@ -31,7 +31,8 @@ public class DecryptionValidationResultTest { @Test public void testValidationCanSetReport() { DecryptionValidationResult.Builder resultBuilder = - DecryptionValidationResult.builder().setReport(FakeReportGenerator.generate(1)); + DecryptionValidationResult.builder() + .setReport(FakeReportGenerator.generateWithParam(1, /* reportVersion */ "")); resultBuilder.build(); @@ -61,7 +62,7 @@ public void testValidationCanSetErrors() { public void testValidationErrorThrownIfBothSet() { DecryptionValidationResult.Builder resultBuilder = DecryptionValidationResult.builder() - .setReport(FakeReportGenerator.generate(1)) + .setReport(FakeReportGenerator.generateWithParam(1, /* reportVersion */ "")) .addErrorMessage( ErrorMessage.builder() .setCategory(GENERAL_ERROR.name()) diff --git a/javatests/com/google/aggregate/adtech/worker/model/SharedInfoTest.java b/javatests/com/google/aggregate/adtech/worker/model/SharedInfoTest.java index 01237906..bce73047 100644 --- a/javatests/com/google/aggregate/adtech/worker/model/SharedInfoTest.java +++ b/javatests/com/google/aggregate/adtech/worker/model/SharedInfoTest.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertTrue; import java.time.Instant; +import java.util.Optional; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -53,14 +54,14 @@ public class SharedInfoTest { * source_registration_time */ private static final String PRIVACY_BUDGET_KEY_2 = - "23f305727d53223a10d0a5a0e2132ea261a5a266524d5c14c4a61bd3f3e60a40"; + "7b16c743c6ffe44bc559561b4f457fd3dcf91b797446ed6dc6b01d9fb32d3565"; /** * Privacy Budget Key generated based on Chrome Golden Report - * https://source.chromium.org/chromium/chromium/src/+/main:content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_1.json */ private static final String PRIVACY_BUDGET_KEY_CHROME_GOLDEN_REPORT = - "b8f9b95599410466fb20e1daeb33ac83ffe05450a43c25e1c3ba9857f4d13063"; + "399bd3cd2282959381e4ad6858c5f434285ec70252b5a446808815780d36140f"; /** Test to verify Privacy Budget Key is pickup correctly from Shared Info */ @Test @@ -147,7 +148,7 @@ public void testSetAndGetReportDebugModeEnabled() { SharedInfo si = sharedInfoBuilder.build(); - assertEquals(si.reportDebugModeString(), "enabled"); + assertEquals(si.reportDebugModeString().get(), "enabled"); assertTrue(si.getReportDebugMode()); } @@ -164,7 +165,7 @@ public void testSetAndGetReportDebugModeDisabled() { SharedInfo si = sharedInfoBuilder.build(); - assertEquals(si.reportDebugModeString(), "disabled"); + assertEquals(si.reportDebugModeString(), Optional.empty()); assertFalse(si.getReportDebugMode()); } @@ -180,7 +181,7 @@ public void testSetAndGetReportDebugModeDefault() { SharedInfo si = sharedInfoBuilder.build(); - assertEquals(si.reportDebugModeString(), "disabled"); + assertEquals(si.reportDebugModeString(), Optional.empty()); assertFalse(si.getReportDebugMode()); } @@ -210,31 +211,4 @@ public void testSetReportDebugModeEnabledTwoTypes() { assertEquals(si1, si2); } - - /** - * Test to verify setReportDebugModeString has the same result as setReportDebugMode when debug - * mode is disabled - */ - @Test - public void testSetReportDebugModeDisabledTwoTypes() { - SharedInfo.Builder sharedInfoBuilder1 = - SharedInfo.builder() - .setVersion(DEFAULT_VERSION) - .setPrivacyBudgetKey(PRIVACY_BUDGET_KEY_1) - .setScheduledReportTime(FIXED_TIME) - .setReportingOrigin(REPORTING_ORIGIN) - .setReportDebugModeString("disabled"); - SharedInfo si1 = sharedInfoBuilder1.build(); - - SharedInfo.Builder sharedInfoBuilder2 = - SharedInfo.builder() - .setVersion(DEFAULT_VERSION) - .setPrivacyBudgetKey(PRIVACY_BUDGET_KEY_1) - .setScheduledReportTime(FIXED_TIME) - .setReportingOrigin(REPORTING_ORIGIN) - .setReportDebugMode(false); - SharedInfo si2 = sharedInfoBuilder2.build(); - - assertEquals(si1, si2); - } } diff --git a/javatests/com/google/aggregate/adtech/worker/model/serdes/BUILD b/javatests/com/google/aggregate/adtech/worker/model/serdes/BUILD index 696f98be..810aa8a2 100644 --- a/javatests/com/google/aggregate/adtech/worker/model/serdes/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/model/serdes/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/adtech/worker/model/serdes/SharedInfoSerdesTest.java b/javatests/com/google/aggregate/adtech/worker/model/serdes/SharedInfoSerdesTest.java index 5b8d47a8..42504810 100644 --- a/javatests/com/google/aggregate/adtech/worker/model/serdes/SharedInfoSerdesTest.java +++ b/javatests/com/google/aggregate/adtech/worker/model/serdes/SharedInfoSerdesTest.java @@ -363,6 +363,29 @@ public void testSerializeAndDeserializeWithoutDebugMode() { assertThat(deserialized).hasValue(sharedInfoWithoutDebugMode); } + /** + * Test if setting debug_mode false in SharedInfo object doesn't add "debug_mode" string to + * serialized SharedInfo string + */ + @Test + public void testSerializeSharedInfoDebugModeDisabled() { + // No setup + SharedInfo sharedInfoDebugModeDisabled = + SharedInfo.builder() + .setVersion("") + .setReportId(SAMPLE_REPORT_ID) + .setScheduledReportTime(FIXED_TIME) + .setReportingOrigin("bar.com") + .setPrivacyBudgetKey("foo") + .setReportDebugMode(false) + .build(); + + String serialized = + sharedInfoSerdes.reverse().convert(Optional.ofNullable(sharedInfoDebugModeDisabled)); + + assertFalse(serialized.contains("debug_mode")); + } + /** Serialize share-info for simulation. Check @JsonIgnore and @JsonProperty setting correctly */ @Test public void testSerializeSharedInfoIgnoreField() { diff --git a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/BUILD b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/BUILD index e5119738..687de0f7 100644 --- a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/CborPayloadSerdesTest.java b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/CborPayloadSerdesTest.java index c2483aaa..f6e74593 100644 --- a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/CborPayloadSerdesTest.java +++ b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/CborPayloadSerdesTest.java @@ -59,7 +59,7 @@ public void setUp() { public void testDeserializeFromCborBytes_report1() throws Exception { Payload expectedPayload = Payload.builder() - .addFact(Fact.builder().setBucket(BigInteger.valueOf(0x4d2)).setValue(5).build()) + .addFact(Fact.builder().setBucket(BigInteger.valueOf(0x1)).setValue(2).build()) .build(); readCborBytesFromFileAndAssert( @@ -70,7 +70,7 @@ public void testDeserializeFromCborBytes_report1() throws Exception { public void testDeserializeFromCborBytes_report2() throws Exception { Payload expectedPayload = Payload.builder() - .addFact(Fact.builder().setBucket(BigInteger.valueOf(0x4d2)).setValue(500).build()) + .addFact(Fact.builder().setBucket(BigInteger.valueOf(0x1)).setValue(2).build()) .build(); readCborBytesFromFileAndAssert( @@ -81,7 +81,8 @@ public void testDeserializeFromCborBytes_report2() throws Exception { public void testDeserializeFromCborBytes_report3() throws Exception { Payload expectedPayload = Payload.builder() - .addFact(Fact.builder().setBucket(BigInteger.valueOf(0x45b352)).setValue(123).build()) + .addFact(Fact.builder().setBucket(BigInteger.valueOf(0x1)).setValue(2).build()) + .addFact(Fact.builder().setBucket(BigInteger.valueOf(0x3)).setValue(4).build()) .build(); readCborBytesFromFileAndAssert( @@ -92,11 +93,8 @@ public void testDeserializeFromCborBytes_report3() throws Exception { public void testDeserializeFromCborBytes_report4() throws Exception { Payload expectedPayload = Payload.builder() - .addFact( - Fact.builder() - .setBucket(BigInteger.valueOf(1).shiftLeft(120)) /* = 2^120 */ - .setValue(2) - .build()) + .addFact(Fact.builder().setBucket(BigInteger.valueOf(0x1)).setValue(2).build()) + .addFact(Fact.builder().setBucket(BigInteger.valueOf(0x3)).setValue(4).build()) .build(); readCborBytesFromFileAndAssert( @@ -113,7 +111,7 @@ public void testDeserializeFromCborBytes_report5() throws Exception { BigInteger.valueOf(1) .shiftLeft(128) .subtract(BigInteger.valueOf(1) /* = 2^128-1 */)) - .setValue(345) + .setValue(1000) .build()) .build(); @@ -125,7 +123,11 @@ public void testDeserializeFromCborBytes_report5() throws Exception { public void testDeserializeFromCborBytes_report6() throws Exception { Payload expectedPayload = Payload.builder() - .addFact(Fact.builder().setBucket(BigInteger.valueOf(0)).setValue(345).build()) + .addFact( + Fact.builder() + .setBucket(new BigInteger("340282366920938463463374607431768211455")) + .setValue(1000) + .build()) .build(); readCborBytesFromFileAndAssert( diff --git a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/README.md b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/README.md index 5991ee63..f075b20d 100644 --- a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/README.md +++ b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/README.md @@ -2,16 +2,15 @@ Contains tests and test input files for cbor deserialization & serialization -`report.cbor` is a CBOR serialized report, generated from the contents -of `report.json` using the `json2cbor.rb` utility -from https://github.com/cabo/cbor-diag. `report.json` is a JSON representation -of the reports that Chrome produces. +Chrome Json reports are picked from +[here](https://source.chromium.org/chromium/chromium/src/+/main:content/test/data/attribution_reporting/aggregatable_report_goldens/latest/). -Spec is defined -here: https://github.com/WICG/conversion-measurement-api/blob/main/AGGREGATE.md#aggregate-attribution-reports +`report_x_cleartext_payloads.json` has base64 encoded cleartest payload that is passed through +base64 decoder to get cbor, for example: -If needed, the file can be regenerated with this command: - -``` - cat report.json | json2cbor.rb > report.cbor +```sh + base64 -d report_1_cleartext_payloads.json > report1.cbor ``` + +CborPayloadSerdesTest reads the cbor files from reportx.cbor files, deserializes the cbor payload +and compares it with manually constructed test payload. diff --git a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report1.cbor b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report1.cbor index e6caabe2..26fb67eb 100644 Binary files a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report1.cbor and b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report1.cbor differ diff --git a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report2.cbor b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report2.cbor index 8814e9fc..26fb67eb 100644 Binary files a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report2.cbor and b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report2.cbor differ diff --git a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report3.cbor b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report3.cbor index 067418cf..271d8e81 100644 Binary files a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report3.cbor and b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report3.cbor differ diff --git a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report4.cbor b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report4.cbor index 7a32af5a..271d8e81 100644 Binary files a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report4.cbor and b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report4.cbor differ diff --git a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report5.cbor b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report5.cbor index 3ab98f21..04764260 100644 Binary files a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report5.cbor and b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report5.cbor differ diff --git a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report6.cbor b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report6.cbor index b4bc19a2..04764260 100644 Binary files a/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report6.cbor and b/javatests/com/google/aggregate/adtech/worker/model/serdes/cbor/resources/report6.cbor differ diff --git a/javatests/com/google/aggregate/adtech/worker/reader/avro/BUILD b/javatests/com/google/aggregate/adtech/worker/reader/avro/BUILD index 490fc75e..5c940c9d 100644 --- a/javatests/com/google/aggregate/adtech/worker/reader/avro/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/reader/avro/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/adtech/worker/testing/BUILD b/javatests/com/google/aggregate/adtech/worker/testing/BUILD index ff2661b5..490b3a30 100644 --- a/javatests/com/google/aggregate/adtech/worker/testing/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/testing/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + java_test( name = "FakeJobResultGeneratorTest", srcs = ["FakeJobResultGeneratorTest.java"], @@ -32,6 +34,7 @@ java_test( srcs = ["NoopJobProcessorTest.java"], deps = [ "//java/com/google/aggregate/adtech/worker", + "//java/com/google/aggregate/adtech/worker/exceptions", "//java/com/google/aggregate/adtech/worker/model", "//java/com/google/aggregate/adtech/worker/testing:fake_job_result_generator", "//java/com/google/aggregate/adtech/worker/testing:noop_job_processor", @@ -118,6 +121,7 @@ java_test( srcs = ["InMemoryResultLoggerTest.java"], deps = [ "//java/com/google/aggregate/adtech/worker", + "//java/com/google/aggregate/adtech/worker/exceptions", "//java/com/google/aggregate/adtech/worker/model", "//java/com/google/aggregate/adtech/worker/testing:in_memory_logger", "//java/external:clients_jobclient_aws", diff --git a/javatests/com/google/aggregate/adtech/worker/testing/FakeRecordDecrypterTest.java b/javatests/com/google/aggregate/adtech/worker/testing/FakeRecordDecrypterTest.java index 1d2c1e59..e6ec308e 100644 --- a/javatests/com/google/aggregate/adtech/worker/testing/FakeRecordDecrypterTest.java +++ b/javatests/com/google/aggregate/adtech/worker/testing/FakeRecordDecrypterTest.java @@ -62,6 +62,8 @@ public void fakeDecryption() throws Exception { Report report = decrypter.decryptSingleReport(encryptedReport); assertThat(report) - .isEqualTo(generateWithFixedReportId(2, report.sharedInfo().reportId().get())); + .isEqualTo( + generateWithFixedReportId( + 2, report.sharedInfo().reportId().get(), /* reportVersion */ "")); } } diff --git a/javatests/com/google/aggregate/adtech/worker/testing/FakeReportGeneratorTest.java b/javatests/com/google/aggregate/adtech/worker/testing/FakeReportGeneratorTest.java index d1346718..fa8ed823 100644 --- a/javatests/com/google/aggregate/adtech/worker/testing/FakeReportGeneratorTest.java +++ b/javatests/com/google/aggregate/adtech/worker/testing/FakeReportGeneratorTest.java @@ -27,6 +27,7 @@ import com.google.aggregate.adtech.worker.testing.FakeReportGenerator.FakeFactGenerator; import com.google.common.collect.ImmutableList; import java.time.Instant; +import java.util.Optional; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -34,6 +35,8 @@ @RunWith(JUnit4.class) public class FakeReportGeneratorTest { + private static final String VERSION_0_1 = "0.1"; + @Test public void testGenerateFakeFact() { int id = 1; @@ -46,7 +49,7 @@ public void testGenerateFakeFact() { } @Test - public void testGenerateFakeReportListInput() { + public void testGenerate_FakeReportList_version_default() { // Setup. int id1 = 1; int val1 = 1; @@ -59,7 +62,8 @@ public void testGenerateFakeReportListInput() { FakeFactGenerator.generate(id1, val1), FakeFactGenerator.generate(id2, val2)); // Invocation. - Report generatedReport = FakeReportGenerator.generate(factList); + Report generatedReport = + FakeReportGenerator.generateWithFactList(factList, /* reportVersion */ ""); // Assert. assertThat(generatedReport) @@ -83,10 +87,10 @@ public void testGenerateFakeReportListInput() { } @Test - public void testGenerate() { + public void testGenerate_version_default() { int id = 2; - Report generatedReport = FakeReportGenerator.generate(id); + Report generatedReport = FakeReportGenerator.generateWithParam(id, /* reportVersion */ ""); assertThat(generatedReport) .isEqualTo( @@ -107,4 +111,80 @@ public void testGenerate() { .build()) .build()); } + + @Test + public void testGenerate_reportId_version_default() { + String reportId = "My Report ID"; + int dummyValue = 2; + + Report generatedReport = + FakeReportGenerator.generateWithFixedReportId(dummyValue, reportId, /* reportVersion */ ""); + + assertThat(generatedReport.sharedInfo().reportId()).isEqualTo(Optional.of(reportId)); + } + + @Test + public void testGenerate_FakeReportList_version_0_1() { + // Setup. + int id1 = 1; + int val1 = 1; + + int id2 = 2; + int val2 = 2; + + ImmutableList factList = + ImmutableList.of( + FakeFactGenerator.generate(id1, val1), FakeFactGenerator.generate(id2, val2)); + + // Invocation. + Report generatedReport = FakeReportGenerator.generateWithFactList(factList, VERSION_0_1); + + // Assert. + assertThat(generatedReport) + .isEqualTo( + Report.builder() + .setSharedInfo( + SharedInfo.builder() + .setVersion("0.1") + .setApi("attribution-reporting") + .setDestination("dummy") + .setReportingOrigin("dummy") + .setScheduledReportTime(Instant.EPOCH.plus(1, SECONDS)) + .setSourceRegistrationTime(Instant.EPOCH.plus(1, SECONDS)) + .setReportId(generatedReport.sharedInfo().reportId().get()) + .build()) + .setPayload( + Payload.builder() + .addFact(FakeFactGenerator.generate(id1, val1)) + .addFact(FakeFactGenerator.generate(id2, val2)) + .build()) + .build()); + } + + @Test + public void testGenerate_version_0_1() { + int id = 2; + + Report generatedReport = FakeReportGenerator.generateWithParam(id, VERSION_0_1); + + assertThat(generatedReport) + .isEqualTo( + Report.builder() + .setSharedInfo( + SharedInfo.builder() + .setVersion("0.1") + .setApi("attribution-reporting") + .setDestination(String.valueOf(id)) + .setReportingOrigin(String.valueOf(id)) + .setScheduledReportTime(Instant.EPOCH.plus(id, SECONDS)) + .setSourceRegistrationTime(Instant.EPOCH.plus(id, SECONDS)) + .setReportId(generatedReport.sharedInfo().reportId().get()) + .build()) + .setPayload( + Payload.builder() + .addFact(FakeFactGenerator.generate(id, id)) + .addFact(FakeFactGenerator.generate(id, id)) + .build()) + .build()); + } } diff --git a/javatests/com/google/aggregate/adtech/worker/testing/FakeValidatorTest.java b/javatests/com/google/aggregate/adtech/worker/testing/FakeValidatorTest.java index 49d9329e..eccf38c1 100644 --- a/javatests/com/google/aggregate/adtech/worker/testing/FakeValidatorTest.java +++ b/javatests/com/google/aggregate/adtech/worker/testing/FakeValidatorTest.java @@ -42,7 +42,7 @@ public class FakeValidatorTest { @Before public void setUp() { - report = FakeReportGenerator.generate(1); + report = FakeReportGenerator.generateWithParam(1, /* reportVersion */ ""); ctx = FakeJobGenerator.generate("foo"); fakeValidator = new FakeValidator(); diff --git a/javatests/com/google/aggregate/adtech/worker/testing/InMemoryResultLoggerTest.java b/javatests/com/google/aggregate/adtech/worker/testing/InMemoryResultLoggerTest.java index aa47c68c..fb15b36f 100644 --- a/javatests/com/google/aggregate/adtech/worker/testing/InMemoryResultLoggerTest.java +++ b/javatests/com/google/aggregate/adtech/worker/testing/InMemoryResultLoggerTest.java @@ -19,7 +19,7 @@ import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; -import com.google.aggregate.adtech.worker.ResultLogger.ResultLogException; +import com.google.aggregate.adtech.worker.exceptions.ResultLogException; import com.google.aggregate.adtech.worker.model.AggregatedFact; import com.google.scp.operator.cpio.jobclient.model.Job; import com.google.scp.operator.cpio.jobclient.testing.FakeJobGenerator; diff --git a/javatests/com/google/aggregate/adtech/worker/testing/NoopJobProcessorTest.java b/javatests/com/google/aggregate/adtech/worker/testing/NoopJobProcessorTest.java index cbcbe40a..897ee77a 100644 --- a/javatests/com/google/aggregate/adtech/worker/testing/NoopJobProcessorTest.java +++ b/javatests/com/google/aggregate/adtech/worker/testing/NoopJobProcessorTest.java @@ -20,7 +20,6 @@ import static com.google.common.truth.Truth8.assertThat; import static org.junit.Assert.assertThrows; -import com.google.aggregate.adtech.worker.JobProcessor.AggregationJobProcessException; import com.google.scp.operator.cpio.jobclient.model.Job; import com.google.scp.operator.cpio.jobclient.model.JobResult; import com.google.scp.operator.cpio.jobclient.testing.FakeJobGenerator; @@ -65,6 +64,6 @@ public void throwsWhenSetTo() throws Exception { Job item = FakeJobGenerator.generate("foo"); processor.setShouldThrowException(true); - assertThrows(AggregationJobProcessException.class, () -> processor.process(item)); + assertThrows(IllegalStateException.class, () -> processor.process(item)); } } diff --git a/javatests/com/google/aggregate/adtech/worker/util/BUILD b/javatests/com/google/aggregate/adtech/worker/util/BUILD index 2d4bcd78..819d5ac0 100644 --- a/javatests/com/google/aggregate/adtech/worker/util/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/util/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + java_test( name = "NumericConversionsTest", srcs = ["NumericConversionsTest.java"], @@ -34,7 +36,6 @@ java_test( "//java/external:google_truth", "//java/external:operator_protos", "@com_google_adm_cloud_scp//java/com/google/scp/operator/cpio/jobclient:model", - "@com_google_adm_cloud_scp//java/com/google/scp/operator/shared/dao/metadatadb/model", "@com_google_adm_cloud_scp//java/com/google/scp/operator/shared/model", ], ) diff --git a/javatests/com/google/aggregate/adtech/worker/validation/BUILD b/javatests/com/google/aggregate/adtech/worker/validation/BUILD index d5394aa0..d5e36e16 100644 --- a/javatests/com/google/aggregate/adtech/worker/validation/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/validation/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/adtech/worker/writer/avro/BUILD b/javatests/com/google/aggregate/adtech/worker/writer/avro/BUILD index 24e788a5..12638b06 100644 --- a/javatests/com/google/aggregate/adtech/worker/writer/avro/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/writer/avro/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/adtech/worker/writer/json/BUILD b/javatests/com/google/aggregate/adtech/worker/writer/json/BUILD index b4cdc637..ef8d886c 100644 --- a/javatests/com/google/aggregate/adtech/worker/writer/json/BUILD +++ b/javatests/com/google/aggregate/adtech/worker/writer/json/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/perf/BUILD b/javatests/com/google/aggregate/perf/BUILD index 4150b0ce..3196d861 100644 --- a/javatests/com/google/aggregate/perf/BUILD +++ b/javatests/com/google/aggregate/perf/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/privacy/budgeting/converter/BUILD b/javatests/com/google/aggregate/privacy/budgeting/converter/BUILD index 14fa68e3..c7a05d7d 100644 --- a/javatests/com/google/aggregate/privacy/budgeting/converter/BUILD +++ b/javatests/com/google/aggregate/privacy/budgeting/converter/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/privacy/budgeting/testing/BUILD b/javatests/com/google/aggregate/privacy/budgeting/testing/BUILD index d72c7587..87758ae1 100644 --- a/javatests/com/google/aggregate/privacy/budgeting/testing/BUILD +++ b/javatests/com/google/aggregate/privacy/budgeting/testing/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/privacy/noise/BUILD b/javatests/com/google/aggregate/privacy/noise/BUILD index a70e24a0..3615b4b2 100644 --- a/javatests/com/google/aggregate/privacy/noise/BUILD +++ b/javatests/com/google/aggregate/privacy/noise/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/privacy/noise/testing/BUILD b/javatests/com/google/aggregate/privacy/noise/testing/BUILD index 22bac1db..d9a05b68 100644 --- a/javatests/com/google/aggregate/privacy/noise/testing/BUILD +++ b/javatests/com/google/aggregate/privacy/noise/testing/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/privacy/noise/testing/integration/BUILD b/javatests/com/google/aggregate/privacy/noise/testing/integration/BUILD index f87cb045..a87b03ea 100644 --- a/javatests/com/google/aggregate/privacy/noise/testing/integration/BUILD +++ b/javatests/com/google/aggregate/privacy/noise/testing/integration/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/protocol/avro/BUILD b/javatests/com/google/aggregate/protocol/avro/BUILD index a6192810..d424eb93 100644 --- a/javatests/com/google/aggregate/protocol/avro/BUILD +++ b/javatests/com/google/aggregate/protocol/avro/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/shared/mapper/BUILD b/javatests/com/google/aggregate/shared/mapper/BUILD index 12f41bcb..69aa61d2 100644 --- a/javatests/com/google/aggregate/shared/mapper/BUILD +++ b/javatests/com/google/aggregate/shared/mapper/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + package(default_visibility = ["//visibility:public"]) java_test( diff --git a/javatests/com/google/aggregate/tools/shard/AvroShardTest.java b/javatests/com/google/aggregate/tools/shard/AvroShardTest.java index a08ed69b..810bad1a 100644 --- a/javatests/com/google/aggregate/tools/shard/AvroShardTest.java +++ b/javatests/com/google/aggregate/tools/shard/AvroShardTest.java @@ -18,6 +18,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import com.google.aggregate.protocol.avro.AvroOutputDomainReader; import com.google.aggregate.protocol.avro.AvroOutputDomainReaderFactory; @@ -63,22 +64,26 @@ public void setUp() throws IOException { @Test public void testReportShard() throws IOException { int numShard = 2; + Path outputReportShardsDir = outPutDirectory.resolve("reportShards"); Path reportPath = baseDirectory.resolve("input_set_0/batch.avro"); List reportShards = new ArrayList<>(); String[] cli = new String[] { "--input", reportPath.toString(), - "--output_dir", outPutDirectory.toString(), + "--output_dir", outputReportShardsDir.toString(), "--num_shards", String.valueOf(numShard) }; avroShard.main(cli); - List reportShardPaths = Arrays.asList(outPutDirectory.toFile().list()); + + assertTrue(Files.exists(outputReportShardsDir)); + + List reportShardPaths = Arrays.asList(outputReportShardsDir.toFile().list()); assertEquals(reportShardPaths.size(), numShard); for (String reportShardPath : reportShardPaths) { - reportShards.addAll(readReport(outPutDirectory.resolve(reportShardPath))); + reportShards.addAll(readReport(outputReportShardsDir.resolve(reportShardPath))); } assertEquals(reportShards.size(), readReport(reportPath).size()); @@ -87,23 +92,27 @@ public void testReportShard() throws IOException { @Test public void testDomainShard() throws IOException { int numShard = 2; + Path outputDomainShardsDir = outPutDirectory.resolve("domainShards"); Path domainPath = baseDirectory.resolve("input_set_0/domain.avro"); List outputDomainShards = new ArrayList<>(); String[] cli = new String[] { "--input", domainPath.toString(), - "--output_dir", outPutDirectory.toString(), + "--output_dir", outputDomainShardsDir.toString(), "--num_shards", String.valueOf(numShard), "--domain" }; avroShard.main(cli); - List domainShardPaths = Arrays.asList(outPutDirectory.toFile().list()); + + assertTrue(Files.exists(outputDomainShardsDir)); + + List domainShardPaths = Arrays.asList(outputDomainShardsDir.toFile().list()); assertEquals(domainShardPaths.size(), numShard); for (String domainShardPath : domainShardPaths) { - outputDomainShards.addAll(readDomain(outPutDirectory.resolve(domainShardPath))); + outputDomainShards.addAll(readDomain(outputDomainShardsDir.resolve(domainShardPath))); } assertEquals(outputDomainShards.size(), readDomain(domainPath).size()); diff --git a/javatests/com/google/aggregate/tools/shard/BUILD b/javatests/com/google/aggregate/tools/shard/BUILD index f81c08b5..2e6dfb5d 100644 --- a/javatests/com/google/aggregate/tools/shard/BUILD +++ b/javatests/com/google/aggregate/tools/shard/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@rules_java//java:defs.bzl", "java_test") + java_test( name = "AvroShardTest", srcs = ["AvroShardTest.java"], diff --git a/javatests/testData/aggregatable_report_goldens/latest/report_1.json b/javatests/testData/aggregatable_report_goldens/latest/report_1.json new file mode 100644 index 00000000..5b624e04 --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/latest/report_1.json @@ -0,0 +1,10 @@ +{ + "aggregation_service_payloads": [ { + "debug_cleartext_payload": "omRkYXRhgaJldmFsdWVEAAAAAmZidWNrZXRQAAAAAAAAAAAAAAAAAAAAAWlvcGVyYXRpb25paGlzdG9ncmFt", + "key_id": "example_id", + "payload": "+3Q4+JVUsuSEaoct8xQpBqBFZGtfWQvsTTq142kGIRfO99j+JHuI87CLyPKguIEdC7fWjQ75d4HmQ9D/WXzVSMNPbXNjY67MMnkwN8/a0rISl8YGVyLyvmGtnJ36zmJvMMMHZmFVNp2LP9o1LbFo" + } ], + "shared_info": "{\"api\":\"attribution-reporting\",\"attribution_destination\":\"https://conversion.test\",\"debug_mode\":\"enabled\",\"report_id\":\"21abd97f-73e8-4b88-9389-a9fee6abda5e\",\"reporting_origin\":\"https://report.test\",\"scheduled_report_time\":\"1234486400\",\"source_registration_time\":\"1234483200\",\"version\":\"0.1\"}", + "source_debug_key": "123", + "trigger_debug_key": "456" +} diff --git a/javatests/testData/aggregatable_report_goldens/latest/report_1_cleartext_payloads.json b/javatests/testData/aggregatable_report_goldens/latest/report_1_cleartext_payloads.json new file mode 100644 index 00000000..bc85911f --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/latest/report_1_cleartext_payloads.json @@ -0,0 +1,3 @@ +[ + "omRkYXRhgaJldmFsdWVEAAAAAmZidWNrZXRQAAAAAAAAAAAAAAAAAAAAAWlvcGVyYXRpb25paGlzdG9ncmFt" +] \ No newline at end of file diff --git a/javatests/testData/aggregatable_report_goldens/latest/report_2.json b/javatests/testData/aggregatable_report_goldens/latest/report_2.json new file mode 100644 index 00000000..a1dbb336 --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/latest/report_2.json @@ -0,0 +1,7 @@ +{ + "aggregation_service_payloads": [ { + "key_id": "example_id", + "payload": "dCYizN7JKoWWZDpuozgMKeByzoe/IpsVMhelP8l2pmad+TDXg2ozkr7wTGtrnzzV9QO6IpC1HD1fE4zmVK887kGj3yKKFDffRtB61g5DwMwW9ZrBJYvYP5doz4JV/R5QKwpgM5gmvAEa+PXT1R7h" + } ], + "shared_info": "{\"api\":\"attribution-reporting\",\"attribution_destination\":\"https://conversion.test\",\"report_id\":\"21abd97f-73e8-4b88-9389-a9fee6abda5e\",\"reporting_origin\":\"https://report.test\",\"scheduled_report_time\":\"1234486400\",\"source_registration_time\":\"1234483200\",\"version\":\"0.1\"}" +} diff --git a/javatests/testData/aggregatable_report_goldens/latest/report_2_cleartext_payloads.json b/javatests/testData/aggregatable_report_goldens/latest/report_2_cleartext_payloads.json new file mode 100644 index 00000000..bc85911f --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/latest/report_2_cleartext_payloads.json @@ -0,0 +1,3 @@ +[ + "omRkYXRhgaJldmFsdWVEAAAAAmZidWNrZXRQAAAAAAAAAAAAAAAAAAAAAWlvcGVyYXRpb25paGlzdG9ncmFt" +] \ No newline at end of file diff --git a/javatests/testData/aggregatable_report_goldens/latest/report_3.json b/javatests/testData/aggregatable_report_goldens/latest/report_3.json new file mode 100644 index 00000000..fe2f3da5 --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/latest/report_3.json @@ -0,0 +1,11 @@ +{ + "aggregation_service_payloads": [ { + "debug_cleartext_payload": "omRkYXRhgqJldmFsdWVEAAAAAmZidWNrZXRQAAAAAAAAAAAAAAAAAAAAAaJldmFsdWVEAAAABGZidWNrZXRQAAAAAAAAAAAAAAAAAAAAA2lvcGVyYXRpb25paGlzdG9ncmFt", + "key_id": "example_id", + "payload": "YgUBhtua0jsHvbFxd8ROeg97SI5Qelu4ykXRBIdoUSW2SH7ES5Dq4VrlFJL28J2Ly0tzlbMfzvdKqTLcEATjUaWaQ9Ncg8HQ8tnoghdxzql5vS/MTwbgiY/TDqi3EgbOIiCd//7pnH5dAD+WIgpstlZZUBiJQKlu5C0YDSWOuoLwK+VTs+Tc9SYFq4zdJSGVA7tA" + } ], + "shared_info": "{\"api\":\"attribution-reporting\",\"attribution_destination\":\"https://conversion.test\",\"debug_mode\":\"enabled\",\"report_id\":\"21abd97f-73e8-4b88-9389-a9fee6abda5e\",\"reporting_origin\":\"https://report.test\",\"scheduled_report_time\":\"1234486500\",\"source_registration_time\":\"1234483200\",\"version\":\"0.1\"}", + "source_debug_key": "123", + "trigger_debug_key": "456" +} + \ No newline at end of file diff --git a/javatests/testData/aggregatable_report_goldens/latest/report_3_cleartext_payloads.json b/javatests/testData/aggregatable_report_goldens/latest/report_3_cleartext_payloads.json new file mode 100644 index 00000000..c3578b04 --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/latest/report_3_cleartext_payloads.json @@ -0,0 +1,3 @@ +[ + "omRkYXRhgqJldmFsdWVEAAAAAmZidWNrZXRQAAAAAAAAAAAAAAAAAAAAAaJldmFsdWVEAAAABGZidWNrZXRQAAAAAAAAAAAAAAAAAAAAA2lvcGVyYXRpb25paGlzdG9ncmFt" +] \ No newline at end of file diff --git a/javatests/testData/aggregatable_report_goldens/latest/report_4.json b/javatests/testData/aggregatable_report_goldens/latest/report_4.json new file mode 100644 index 00000000..ecaf02f3 --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/latest/report_4.json @@ -0,0 +1,7 @@ +{ + "aggregation_service_payloads": [ { + "key_id": "example_id", + "payload": "LiavspqLWGHPnexW5xtuLXVMb2oNNBrrhULpOrBR4TGKnKCxL9qdKTyqzSJlkztvG8lDG2eQnxMwRl0lwKu4T157wwYHLpt0VxWBArFUeNh18QP5WihW7z3+UYkNQN3PJ7ddwoDe60X5UiQ6fxdUKT7F0mOIb/bAjKYKvUd8QukUBCvxTLdGmrWj22+OR239nRAO" + } ], + "shared_info": "{\"api\":\"attribution-reporting\",\"attribution_destination\":\"https://conversion.test\",\"report_id\":\"21abd97f-73e8-4b88-9389-a9fee6abda5e\",\"reporting_origin\":\"https://report.test\",\"scheduled_report_time\":\"1234486500\",\"source_registration_time\":\"1234483200\",\"version\":\"0.1\"}" +} diff --git a/javatests/testData/aggregatable_report_goldens/latest/report_4_cleartext_payloads.json b/javatests/testData/aggregatable_report_goldens/latest/report_4_cleartext_payloads.json new file mode 100644 index 00000000..c3578b04 --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/latest/report_4_cleartext_payloads.json @@ -0,0 +1,3 @@ +[ + "omRkYXRhgqJldmFsdWVEAAAAAmZidWNrZXRQAAAAAAAAAAAAAAAAAAAAAaJldmFsdWVEAAAABGZidWNrZXRQAAAAAAAAAAAAAAAAAAAAA2lvcGVyYXRpb25paGlzdG9ncmFt" +] \ No newline at end of file diff --git a/javatests/testData/aggregatable_report_goldens/latest/report_5.json b/javatests/testData/aggregatable_report_goldens/latest/report_5.json new file mode 100644 index 00000000..beee9df5 --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/latest/report_5.json @@ -0,0 +1,10 @@ +{ + "aggregation_service_payloads": [ { + "debug_cleartext_payload": "omRkYXRhgaJldmFsdWVEAAAD6GZidWNrZXRQ/////////////////////2lvcGVyYXRpb25paGlzdG9ncmFt", + "key_id": "example_id", + "payload": "s6tBRv9QuDDKxi4/yT2/JGys+gxnzqVic33u8ungNBo03u/kQhhZd4tGiOubxUHQwliP+PPiSymQwapcWuDvPM2+ajGXY/sY6fQXuMg0KIuYUOkguNPM5ZwcIxCiCJ2Brbv8MtGy2ZuCHQ176xPh" + } ], + "shared_info": "{\"api\":\"attribution-reporting\",\"attribution_destination\":\"https://conversion.test\",\"debug_mode\":\"enabled\",\"report_id\":\"21abd97f-73e8-4b88-9389-a9fee6abda5e\",\"reporting_origin\":\"https://report.test\",\"scheduled_report_time\":\"1234486600\",\"source_registration_time\":\"1234483200\",\"version\":\"0.1\"}", + "source_debug_key": "123", + "trigger_debug_key": "456" +} diff --git a/javatests/testData/aggregatable_report_goldens/latest/report_5_cleartext_payloads.json b/javatests/testData/aggregatable_report_goldens/latest/report_5_cleartext_payloads.json new file mode 100644 index 00000000..26be6ded --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/latest/report_5_cleartext_payloads.json @@ -0,0 +1,3 @@ +[ + "omRkYXRhgaJldmFsdWVEAAAD6GZidWNrZXRQ/////////////////////2lvcGVyYXRpb25paGlzdG9ncmFt" +] \ No newline at end of file diff --git a/javatests/testData/aggregatable_report_goldens/latest/report_6.json b/javatests/testData/aggregatable_report_goldens/latest/report_6.json new file mode 100644 index 00000000..520fed79 --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/latest/report_6.json @@ -0,0 +1,7 @@ +{ + "aggregation_service_payloads": [ { + "key_id": "example_id", + "payload": "IQuLGiKuR9tcUrq3nWuJ7Qy6VrtQSKNDUgflzyuIkT1R6kagnjxsYVMhrw3j9euUQaSlK9jYfg070Z+GMIGYjRwsVUGnCbmTROH30zrB1ar2hupppnvl0Hz2Xkbv9ZjoVCmuV97RuQP506JqYyj7" + } ], + "shared_info": "{\"api\":\"attribution-reporting\",\"attribution_destination\":\"https://conversion.test\",\"report_id\":\"21abd97f-73e8-4b88-9389-a9fee6abda5e\",\"reporting_origin\":\"https://report.test\",\"scheduled_report_time\":\"1234486600\",\"source_registration_time\":\"1234483200\",\"version\":\"0.1\"}" +} diff --git a/javatests/testData/aggregatable_report_goldens/latest/report_6_cleartext_payloads.json b/javatests/testData/aggregatable_report_goldens/latest/report_6_cleartext_payloads.json new file mode 100644 index 00000000..26be6ded --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/latest/report_6_cleartext_payloads.json @@ -0,0 +1,3 @@ +[ + "omRkYXRhgaJldmFsdWVEAAAD6GZidWNrZXRQ/////////////////////2lvcGVyYXRpb25paGlzdG9ncmFt" +] \ No newline at end of file diff --git a/javatests/testData/aggregatable_report_goldens/version_/report_1.json b/javatests/testData/aggregatable_report_goldens/version_/report_1.json new file mode 100644 index 00000000..dbea77de --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/version_/report_1.json @@ -0,0 +1,13 @@ +{ + "aggregation_service_payloads": [ { + "debug_cleartext_payload": "omRkYXRhgaJldmFsdWVEAAAAAmZidWNrZXRQAAAAAAAAAAAAAAAAAAAAAWlvcGVyYXRpb25paGlzdG9ncmFt", + "key_id": "example_id", + "payload": "cVrsURY6QvrT1fsdPxWOXb3bjlzB6ih3APjLpydjyRoAJ6FTyZf9AUj1l5VlDweb3fssxCaMkz5jr5oHPlehnBVjrKLiSvtX3mTJ3CJD9DEp3uiiKV5Fsjv4G4RJcdwZERnJN/8s1OWJ6syEqKnJ" + } ], + "attribution_destination": "https://conversion.test", + "shared_info": "{\"debug_mode\":\"enabled\",\"privacy_budget_key\":\"aOEbtVxG8dYzAR2K/xuW/OppNaQikp5RdjAXshOQ9w8=\",\"report_id\":\"21abd97f-73e8-4b88-9389-a9fee6abda5e\",\"reporting_origin\":\"https://report.test\",\"scheduled_report_time\":\"1234486400\",\"version\":\"\"}", + "source_debug_key": "123", + "source_registration_time": "1234483200", + "source_site": "https://impression.test", + "trigger_debug_key": "456" +} diff --git a/javatests/testData/aggregatable_report_goldens/version_/report_1_cleartext_payloads.json b/javatests/testData/aggregatable_report_goldens/version_/report_1_cleartext_payloads.json new file mode 100644 index 00000000..bc85911f --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/version_/report_1_cleartext_payloads.json @@ -0,0 +1,3 @@ +[ + "omRkYXRhgaJldmFsdWVEAAAAAmZidWNrZXRQAAAAAAAAAAAAAAAAAAAAAWlvcGVyYXRpb25paGlzdG9ncmFt" +] \ No newline at end of file diff --git a/javatests/testData/aggregatable_report_goldens/version_/report_2.json b/javatests/testData/aggregatable_report_goldens/version_/report_2.json new file mode 100644 index 00000000..114ec202 --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/version_/report_2.json @@ -0,0 +1,10 @@ +{ + "aggregation_service_payloads": [ { + "key_id": "example_id", + "payload": "eXr+bRdXPiUmHsh/Dn0MKTALoJI6/LTzcZ4cUZiXmjFcVaojbQT+fzj+Wmfrg9qFecZbddTMMT4hkBXow+fxLyuLGgOfN++2c7oEijuqPLLBUQYHHhsxfdx9i/DzP6G7OVb7XvsvuB9q5MGDGesg" + } ], + "attribution_destination": "https://conversion.test", + "shared_info": "{\"privacy_budget_key\":\"aOEbtVxG8dYzAR2K/xuW/OppNaQikp5RdjAXshOQ9w8=\",\"report_id\":\"21abd97f-73e8-4b88-9389-a9fee6abda5e\",\"reporting_origin\":\"https://report.test\",\"scheduled_report_time\":\"1234486400\",\"version\":\"\"}", + "source_registration_time": "1234483200", + "source_site": "https://impression.test" +} \ No newline at end of file diff --git a/javatests/testData/aggregatable_report_goldens/version_/report_2_cleartext_payloads.json b/javatests/testData/aggregatable_report_goldens/version_/report_2_cleartext_payloads.json new file mode 100644 index 00000000..bc85911f --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/version_/report_2_cleartext_payloads.json @@ -0,0 +1,3 @@ +[ + "omRkYXRhgaJldmFsdWVEAAAAAmZidWNrZXRQAAAAAAAAAAAAAAAAAAAAAWlvcGVyYXRpb25paGlzdG9ncmFt" +] \ No newline at end of file diff --git a/javatests/testData/aggregatable_report_goldens/version_/report_3.json b/javatests/testData/aggregatable_report_goldens/version_/report_3.json new file mode 100644 index 00000000..75b7bfbc --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/version_/report_3.json @@ -0,0 +1,13 @@ +{ + "aggregation_service_payloads": [ { + "debug_cleartext_payload": "omRkYXRhgqJldmFsdWVEAAAAAmZidWNrZXRQAAAAAAAAAAAAAAAAAAAAAaJldmFsdWVEAAAABGZidWNrZXRQAAAAAAAAAAAAAAAAAAAAA2lvcGVyYXRpb25paGlzdG9ncmFt", + "key_id": "example_id", + "payload": "zxdMIx1NKDOndRaLGkyzx7f28Mxw7xicACB/9cbYCibLDv7JDxmx+fScuuVOUziz874o4/hmOHU546wtzo9wSXM2QhPY9fmH4jTAwBOz7azvC6pqX8pJrjqxl6AjOG0JRSa3mBz3m4SOVPl9EuGSpb5imA9s4a1cqBtrgS3QxLjnd3LCihK/Mgc3m5PVhk9X+Iz5" + } ], + "attribution_destination": "https://conversion.test", + "shared_info": "{\"debug_mode\":\"enabled\",\"privacy_budget_key\":\"aOEbtVxG8dYzAR2K/xuW/OppNaQikp5RdjAXshOQ9w8=\",\"report_id\":\"21abd97f-73e8-4b88-9389-a9fee6abda5e\",\"reporting_origin\":\"https://report.test\",\"scheduled_report_time\":\"1234486500\",\"version\":\"\"}", + "source_debug_key": "123", + "source_registration_time": "1234483200", + "source_site": "https://impression.test", + "trigger_debug_key": "456" +} diff --git a/javatests/testData/aggregatable_report_goldens/version_/report_3_cleartext_payloads.json b/javatests/testData/aggregatable_report_goldens/version_/report_3_cleartext_payloads.json new file mode 100644 index 00000000..c3578b04 --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/version_/report_3_cleartext_payloads.json @@ -0,0 +1,3 @@ +[ + "omRkYXRhgqJldmFsdWVEAAAAAmZidWNrZXRQAAAAAAAAAAAAAAAAAAAAAaJldmFsdWVEAAAABGZidWNrZXRQAAAAAAAAAAAAAAAAAAAAA2lvcGVyYXRpb25paGlzdG9ncmFt" +] \ No newline at end of file diff --git a/javatests/testData/aggregatable_report_goldens/version_/report_4.json b/javatests/testData/aggregatable_report_goldens/version_/report_4.json new file mode 100644 index 00000000..45884bd0 --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/version_/report_4.json @@ -0,0 +1,10 @@ +{ + "aggregation_service_payloads": [ { + "key_id": "example_id", + "payload": "0xJTQXpWySmhg30YZCaa7WoEoDskHoyOxa4P3g3DJAUaEjBX1WWFinKZ53P9GlX6y/nDufrtgPPXDhgQd6n8ujkpItzahUQCgRxuACEbNctbY94+rYRuRqXuOByi+OXOtN5RpCBnHXhxTGLv4+hVF0pLLWybTS6LiX15VykhK0KiQzjMqeV2fv0da/52YdC7FApQ" + } ], + "attribution_destination": "https://conversion.test", + "shared_info": "{\"privacy_budget_key\":\"aOEbtVxG8dYzAR2K/xuW/OppNaQikp5RdjAXshOQ9w8=\",\"report_id\":\"21abd97f-73e8-4b88-9389-a9fee6abda5e\",\"reporting_origin\":\"https://report.test\",\"scheduled_report_time\":\"1234486500\",\"version\":\"\"}", + "source_registration_time": "1234483200", + "source_site": "https://impression.test" +} diff --git a/javatests/testData/aggregatable_report_goldens/version_/report_4_cleartext_payloads.json b/javatests/testData/aggregatable_report_goldens/version_/report_4_cleartext_payloads.json new file mode 100644 index 00000000..c3578b04 --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/version_/report_4_cleartext_payloads.json @@ -0,0 +1,3 @@ +[ + "omRkYXRhgqJldmFsdWVEAAAAAmZidWNrZXRQAAAAAAAAAAAAAAAAAAAAAaJldmFsdWVEAAAABGZidWNrZXRQAAAAAAAAAAAAAAAAAAAAA2lvcGVyYXRpb25paGlzdG9ncmFt" +] \ No newline at end of file diff --git a/javatests/testData/aggregatable_report_goldens/version_/report_5.json b/javatests/testData/aggregatable_report_goldens/version_/report_5.json new file mode 100644 index 00000000..8914905c --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/version_/report_5.json @@ -0,0 +1,13 @@ +{ + "aggregation_service_payloads": [ { + "debug_cleartext_payload": "omRkYXRhgaJldmFsdWVEAAAD6GZidWNrZXRQ/////////////////////2lvcGVyYXRpb25paGlzdG9ncmFt", + "key_id": "example_id", + "payload": "C87AumVqWwJd8+S82/hGj9CFrIJGTVg7R6ioUeFa+UhnxI9yUB7UQJcFg5LN0LTdi1dGTHtZM85c6ObeVWMkSUDWIDaR3lDjpL3l+utAfseAb/mSPBA5CAw6snudGctUPTF+0uQALSW1QnuPLgM+" + } ], + "attribution_destination": "https://conversion.test", + "shared_info": "{\"debug_mode\":\"enabled\",\"privacy_budget_key\":\"aOEbtVxG8dYzAR2K/xuW/OppNaQikp5RdjAXshOQ9w8=\",\"report_id\":\"21abd97f-73e8-4b88-9389-a9fee6abda5e\",\"reporting_origin\":\"https://report.test\",\"scheduled_report_time\":\"1234486600\",\"version\":\"\"}", + "source_debug_key": "123", + "source_registration_time": "1234483200", + "source_site": "https://impression.test", + "trigger_debug_key": "456" +} diff --git a/javatests/testData/aggregatable_report_goldens/version_/report_5_cleartext_payloads.json b/javatests/testData/aggregatable_report_goldens/version_/report_5_cleartext_payloads.json new file mode 100644 index 00000000..26be6ded --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/version_/report_5_cleartext_payloads.json @@ -0,0 +1,3 @@ +[ + "omRkYXRhgaJldmFsdWVEAAAD6GZidWNrZXRQ/////////////////////2lvcGVyYXRpb25paGlzdG9ncmFt" +] \ No newline at end of file diff --git a/javatests/testData/aggregatable_report_goldens/version_/report_6.json b/javatests/testData/aggregatable_report_goldens/version_/report_6.json new file mode 100644 index 00000000..b6463037 --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/version_/report_6.json @@ -0,0 +1,10 @@ +{ + "aggregation_service_payloads": [ { + "key_id": "example_id", + "payload": "IczoOtZk7tC/cBhFlc7tYZj0zaZqfB5RvQ2uWPeufFMf4XW7qyOhw7GfrP2+HtycNkgtY4qWTxJZeg7LsQiDcltXIvoUGvBe0mb9Ygu//uVhn1a/NTlD9nvDlgoxzpSpQvEaOKoqIElZ/0LTvRsZ" + } ], + "attribution_destination": "https://conversion.test", + "shared_info": "{\"privacy_budget_key\":\"aOEbtVxG8dYzAR2K/xuW/OppNaQikp5RdjAXshOQ9w8=\",\"report_id\":\"21abd97f-73e8-4b88-9389-a9fee6abda5e\",\"reporting_origin\":\"https://report.test\",\"scheduled_report_time\":\"1234486600\",\"version\":\"\"}", + "source_registration_time": "1234483200", + "source_site": "https://impression.test" +} diff --git a/javatests/testData/aggregatable_report_goldens/version_/report_6_cleartext_payloads.json b/javatests/testData/aggregatable_report_goldens/version_/report_6_cleartext_payloads.json new file mode 100644 index 00000000..26be6ded --- /dev/null +++ b/javatests/testData/aggregatable_report_goldens/version_/report_6_cleartext_payloads.json @@ -0,0 +1,3 @@ +[ + "omRkYXRhgaJldmFsdWVEAAAD6GZidWNrZXRQ/////////////////////2lvcGVyYXRpb25paGlzdG9ncmFt" +] \ No newline at end of file diff --git a/licenses/MIT/SLF4J API/LICENSE.txt b/licenses/MIT/SLF4J API/LICENSE.txt index 1a3d0532..f687729a 100644 --- a/licenses/MIT/SLF4J API/LICENSE.txt +++ b/licenses/MIT/SLF4J API/LICENSE.txt @@ -19,6 +19,3 @@ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - - diff --git a/licenses/MIT/SLF4J Log4J/LICENSE.txt b/licenses/MIT/SLF4J Log4J/LICENSE.txt index 508a2728..5a11c0c2 100644 --- a/licenses/MIT/SLF4J Log4J/LICENSE.txt +++ b/licenses/MIT/SLF4J Log4J/LICENSE.txt @@ -19,6 +19,3 @@ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - - diff --git a/licenses/MIT/SLF4J Simple Binding/LICENSE.txt b/licenses/MIT/SLF4J Simple Binding/LICENSE.txt index 508a2728..5a11c0c2 100644 --- a/licenses/MIT/SLF4J Simple Binding/LICENSE.txt +++ b/licenses/MIT/SLF4J Simple Binding/LICENSE.txt @@ -19,6 +19,3 @@ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - - diff --git a/licenses/apache-2/AWS Lambda Java Core/LICENSE.txt b/licenses/apache-2/AWS Lambda Java Core/LICENSE.txt index 8f71f43f..8dada3ed 100644 --- a/licenses/apache-2/AWS Lambda Java Core/LICENSE.txt +++ b/licenses/apache-2/AWS Lambda Java Core/LICENSE.txt @@ -199,4 +199,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/licenses/apache-2/AWS Lambda Java Events SDK Transformer/LICENSE.txt b/licenses/apache-2/AWS Lambda Java Events SDK Transformer/LICENSE.txt index 8f71f43f..8dada3ed 100644 --- a/licenses/apache-2/AWS Lambda Java Events SDK Transformer/LICENSE.txt +++ b/licenses/apache-2/AWS Lambda Java Events SDK Transformer/LICENSE.txt @@ -199,4 +199,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/licenses/apache-2/AWS Lambda Java Events/LICENSE.txt b/licenses/apache-2/AWS Lambda Java Events/LICENSE.txt index 8f71f43f..8dada3ed 100644 --- a/licenses/apache-2/AWS Lambda Java Events/LICENSE.txt +++ b/licenses/apache-2/AWS Lambda Java Events/LICENSE.txt @@ -199,4 +199,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/licenses/apache-2/Apache Avro/LICENSE.txt b/licenses/apache-2/Apache Avro/LICENSE.txt index 7e159a69..24cb1425 100644 --- a/licenses/apache-2/Apache Avro/LICENSE.txt +++ b/licenses/apache-2/Apache Avro/LICENSE.txt @@ -258,7 +258,7 @@ Copyright (c) 2006-2008 Alexander Chemeris | EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -| OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +| OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR | OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. @@ -267,7 +267,7 @@ Copyright (c) 2006-2008 Alexander Chemeris License for st.c and st.h used in the C implementation: | This is a public domain general purpose hash table package written by -| Peter Moore @ UCB. +| Peter Moore @ UCB. ---------------------------------------------------------------------- License for Dirent API for Microsoft Visual Studio used in the C implementation: @@ -325,14 +325,14 @@ File: nunit.framework.dll | | The origin of this software must not be misrepresented; you must not claim that | you wrote the original software. If you use this software in a product, an -| acknowledgment (see the following) in the product documentation is required. +| acknowledgment (see the following) in the product documentation is required. | | Portions Copyright © 2002-2012 Charlie Poole or Copyright © 2002-2004 James W. | Newkirk, Michael C. Two, Alexei A. Vorontsov or Copyright © 2000-2002 Philip A. -| Craig +| Craig | | Altered source versions must be plainly marked as such, and must not be -| misrepresented as being the original software. +| misrepresented as being the original software. | | This notice may not be removed or altered from any source distribution. | License Note @@ -455,10 +455,10 @@ Copyright (c) 2008 Jason Frame (jason@onehackoranother.com) | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | copies of the Software, and to permit persons to whom the Software is | furnished to do so, subject to the following conditions: -| +| | The above copyright notice and this permission notice shall be included in | all copies or substantial portions of the Software. -| +| | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE @@ -475,18 +475,18 @@ All rights reserved. | Redistribution and use in source and binary forms, with or without modification, | are permitted provided that the following conditions are met: -| +| | * Redistributions of source code must retain the above copyright notice, | this list of conditions and the following disclaimer. -| +| | * Redistributions in binary form must reproduce the above copyright notice, | this list of conditions and the following disclaimer in the documentation | and/or other materials provided with the distribution. -| +| | * Neither the name of Stanford University nor the names of its contributors | may be used to endorse or promote products derived from this software | without specific prior written permission. -| +| | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE diff --git a/licenses/apache-2/Apache Commons Math/LICENSE.txt b/licenses/apache-2/Apache Commons Math/LICENSE.txt index d97b49ab..876a2dd7 100644 --- a/licenses/apache-2/Apache Commons Math/LICENSE.txt +++ b/licenses/apache-2/Apache Commons Math/LICENSE.txt @@ -220,10 +220,10 @@ Apache License 2.0: has been imported from the Orekit space flight dynamics library. =============================================================================== - -APACHE COMMONS MATH DERIVATIVE WORKS: + +APACHE COMMONS MATH DERIVATIVE WORKS: The Apache commons-math library includes a number of subcomponents whose implementation is derived from original sources written @@ -233,7 +233,7 @@ are reproduced below. =============================================================================== For the lmder, lmpar and qrsolv Fortran routine from minpack and translated in the LevenbergMarquardtOptimizer class in package -org.apache.commons.math3.optimization.general +org.apache.commons.math3.optimization.general Original source copyright and license statement: Minpack Copyright Notice (1999) University of Chicago. All rights reserved @@ -296,36 +296,36 @@ in package org.apache.commons.math3.ode.nonstiff: Copyright (c) 2004, Ernst Hairer -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: -- Redistributions of source code must retain the above copyright +- Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -- Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS -IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. =============================================================================== Copyright and license statement for the original Mersenne twister C -routines translated in MersenneTwister class in package +routines translated in MersenneTwister class in package org.apache.commons.math3.random: Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura, - All rights reserved. + All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -338,8 +338,8 @@ org.apache.commons.math3.random: notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. The names of its contributors may not be used to endorse or promote - products derived from this software without specific prior written + 3. The names of its contributors may not be used to endorse or promote + products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS @@ -443,7 +443,7 @@ modification, are permitted provided that the following conditions are met: be used to endorse or promote products derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY @@ -454,4 +454,3 @@ ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. =============================================================================== - diff --git a/licenses/apache-2/Apache HTTP Core/LICENSE.txt b/licenses/apache-2/Apache HTTP Core/LICENSE.txt index e454a525..f433b1a5 100644 --- a/licenses/apache-2/Apache HTTP Core/LICENSE.txt +++ b/licenses/apache-2/Apache HTTP Core/LICENSE.txt @@ -175,4 +175,3 @@ of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS - diff --git a/licenses/apache-2/Awaitility/LICENSE.txt b/licenses/apache-2/Awaitility/LICENSE.txt index b5711489..ebcf1487 100644 --- a/licenses/apache-2/Awaitility/LICENSE.txt +++ b/licenses/apache-2/Awaitility/LICENSE.txt @@ -2,17 +2,17 @@ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ - + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - + 1. Definitions. - + "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. - + "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. - + "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, @@ -20,24 +20,24 @@ direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - + "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. - + "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. - + "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. - + "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). - + "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications @@ -45,7 +45,7 @@ of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. - + "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally @@ -59,18 +59,18 @@ Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." - + "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. - + 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. - + 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable @@ -86,24 +86,24 @@ or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. - + 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: - + (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and - + (b) You must cause any modified files to carry prominent notices stating that You changed the files; and - + (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and - + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained @@ -120,14 +120,14 @@ or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. - + You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. - + 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of @@ -135,12 +135,12 @@ Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. - + 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. - + 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, @@ -150,7 +150,7 @@ PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. - + 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly @@ -162,7 +162,7 @@ work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. - + 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, @@ -173,11 +173,11 @@ defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - + END OF TERMS AND CONDITIONS - + APPENDIX: How to apply the Apache License to your work. - + To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include @@ -186,15 +186,15 @@ file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. - + Copyright [2018] [Johan Haleby] - + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/licenses/apache-2/Findbugs/LICENSE.txt b/licenses/apache-2/Findbugs/LICENSE.txt index 29fae787..84247609 100644 --- a/licenses/apache-2/Findbugs/LICENSE.txt +++ b/licenses/apache-2/Findbugs/LICENSE.txt @@ -2,7 +2,7 @@ The JSR-305 reference implementation (lib/jsr305.jar) is distributed under the terms of the New BSD license: http://www.opensource.org/licenses/bsd-license.php - + See the JSR-305 home page for more information: http://code.google.com/p/jsr-305/ diff --git a/licenses/apache-2/JCommander/LICENSE.txt b/licenses/apache-2/JCommander/LICENSE.txt index d0c18cff..477eb7b7 100644 --- a/licenses/apache-2/JCommander/LICENSE.txt +++ b/licenses/apache-2/JCommander/LICENSE.txt @@ -200,4 +200,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/protocol/avro/debug_results.avsc b/protocol/avro/debug_results.avsc index 71bccffb..f70ad4a8 100644 --- a/protocol/avro/debug_results.avsc +++ b/protocol/avro/debug_results.avsc @@ -29,4 +29,4 @@ } } ] -} \ No newline at end of file +} diff --git a/protocol/avro/output_domain.avsc b/protocol/avro/output_domain.avsc index 65559342..85f19a52 100644 --- a/protocol/avro/output_domain.avsc +++ b/protocol/avro/output_domain.avsc @@ -8,4 +8,4 @@ "doc": "A single bucket that appears in the aggregation service output. 128-bit integer encoded as a 16-byte big-endian bytestring." } ] -} \ No newline at end of file +} diff --git a/protocol/avro/reports.avsc b/protocol/avro/reports.avsc index 14f9d443..4bf62b58 100644 --- a/protocol/avro/reports.avsc +++ b/protocol/avro/reports.avsc @@ -15,4 +15,4 @@ "type": "string" } ] -} \ No newline at end of file +} diff --git a/protocol/avro/results.avsc b/protocol/avro/results.avsc index ee5bd3dc..3568eb34 100644 --- a/protocol/avro/results.avsc +++ b/protocol/avro/results.avsc @@ -13,4 +13,4 @@ "doc": "Metric associated with the bucket" } ] -} \ No newline at end of file +} diff --git a/simulation/proto/aggregate_service_key.proto b/simulation/proto/aggregate_service_key.proto index c9daf8b8..b7c4549a 100644 --- a/simulation/proto/aggregate_service_key.proto +++ b/simulation/proto/aggregate_service_key.proto @@ -35,4 +35,4 @@ message AsymmetricKeyPair { // Always set. optional AsymmetricKey public_key = 2; -} \ No newline at end of file +} diff --git a/simulation/proto/simulation_config.proto b/simulation/proto/simulation_config.proto index 32031b1c..2c806369 100644 --- a/simulation/proto/simulation_config.proto +++ b/simulation/proto/simulation_config.proto @@ -105,4 +105,4 @@ message EncryptionKeyConfig { // Limit on number of encryption keys to encrypt a single aggregatable report with. optional int32 num_encryption_keys = 1; optional string key_vending_service_uri = 2; -} \ No newline at end of file +} diff --git a/terraform/aws/BUILD b/terraform/aws/BUILD index fdf685a3..dcff859e 100644 --- a/terraform/aws/BUILD +++ b/terraform/aws/BUILD @@ -1,3 +1,5 @@ +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") + # Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. load("//build_defs:release.bzl", "s3_jar_release") -load("@bazel_skylib//rules:common_settings.bzl", "string_flag") string_flag( name = "bucket_flag", diff --git a/terraform/aws/download_prebuilt_dependencies.sh b/terraform/aws/download_prebuilt_dependencies.sh index 2848f947..ca33d6bf 100644 --- a/terraform/aws/download_prebuilt_dependencies.sh +++ b/terraform/aws/download_prebuilt_dependencies.sh @@ -28,7 +28,7 @@ mkdir -p jars # URL where prebuilt jar artifacts are published and publicly accessible S3_URL="https://aggregation-service-published-artifacts.s3.amazonaws.com" -# Prebuilt jars to download - _{VERSION} will be postfix added +# Prebuilt jars to download - _{VERSION} will be postfix added jars=( "AwsChangeHandlerLambda" "AwsApiGatewayFrontend" diff --git a/terraform/aws/fetch_terraform.sh b/terraform/aws/fetch_terraform.sh index de51c431..e86a7712 100644 --- a/terraform/aws/fetch_terraform.sh +++ b/terraform/aws/fetch_terraform.sh @@ -47,6 +47,10 @@ cat <> environments/shared/release_params.auto.tfvars ami_name = "aggregation-service-enclave_$VERSION" ami_owners = ["971056657085"] +# temp fix for staying within default VPC EIP quota +vpc_availability_zones = ["a","b","c","d","e"] + +# paths to services lambda jar change_handler_lambda = "../../jars/AwsChangeHandlerLambda_${VERSION}.jar" frontend_lambda = "../../jars/AwsApiGatewayFrontend_${VERSION}.jar" sqs_write_failure_cleanup_lambda = "../../jars/AwsFrontendCleanupLambda_${VERSION}.jar" diff --git a/testing/build_defs/avro.bzl b/testing/build_defs/avro.bzl index 73d8ae45..cd257da9 100644 --- a/testing/build_defs/avro.bzl +++ b/testing/build_defs/avro.bzl @@ -25,6 +25,7 @@ def _generate_avro_impl(ctx): args.add("--distribution_file_path", ctx.file.human_readable_reports) args.add("--asymmetric_key_file_path", ctx.file.key) args.add("--generated_report_scheduled_time", ctx.attr.generated_report_scheduled_time) + args.add("--domain_overlap", ctx.attr.domain_overlap) if ctx.attr.generate_output_domain: args.add("--generate_output_domain") @@ -61,6 +62,17 @@ ReportsInfo = provider( generate_avro = rule( implementation = _generate_avro_impl, attrs = { + "domain_overlap": attr.string( + doc = "Type of overlap domain keys should have with report keys.", + default = "FULL", + ), + "generate_output_domain": attr.bool( + doc = "If true, indicates that output domain of results should be generated.", + ), + "generated_report_scheduled_time": attr.string( + doc = "Scheduled time for the generated reports", + default = "1970-01-01T00:00:00Z", + ), "human_readable_reports": attr.label( doc = "Text file containing human-readable reports.", allow_single_file = True, @@ -71,24 +83,17 @@ generate_avro = rule( allow_single_file = True, mandatory = True, ), - "generated_report_scheduled_time": attr.string( - doc = "Scheduled time for the generated reports", - default = "1970-01-01T00:00:00Z", - ), # TODO: remove the attr below, the tool should figure out on its own how # many reports are there. "num_reports": attr.int( doc = "Number of reports that should be generated.", mandatory = True, ), - "generate_output_domain": attr.bool( - doc = "If true, indicates that output domain of results should be generated.", - ), "output_domain_size": attr.int( doc = "Number of buckets that should be present in the output domain.", ), "_simulation": attr.label( - default = Label("//java/com/google/aggregate/simulation:SimluationRunner"), + default = Label("//java/com/google/aggregate/simulation:SimulationRunner"), executable = True, cfg = "target", ), @@ -125,10 +130,6 @@ def _shard_avro_impl(ctx): shard_avro = rule( implementation = _shard_avro_impl, attrs = { - "reports_path": attr.label( - doc = "Path to avro reports file that need to be sharded.", - mandatory = False, - ), "domain_path": attr.label( doc = "Path to domain reports file that need to be sharded.", mandatory = False, @@ -137,6 +138,10 @@ shard_avro = rule( doc = "Number of shards to be generated", mandatory = True, ), + "reports_path": attr.label( + doc = "Path to avro reports file that need to be sharded.", + mandatory = False, + ), "_shard_tool": attr.label( default = Label("//java/com/google/aggregate/tools/shard:AvroShard"), executable = True, diff --git a/tools/load_tests/README.md b/tools/load_tests/README.md index 25255f51..cc2caa89 100644 --- a/tools/load_tests/README.md +++ b/tools/load_tests/README.md @@ -1,24 +1,35 @@ # Aggregation Service Load Tests -Perform load testing on Aggregation Service. This uses the AWS Step Function. -Lambdas are fanned out and each lambda makes some number of requests to the -service. The result is finally compiled and returned. +Perform load testing on Aggregation Service. This uses the AWS Step Function. Lambdas are fanned out +and each lambda makes some number of requests to the service. The result is finally compiled and +returned. Here are user parameter options: 1. `base_url`: **REQUIRED**. Base URL of the Aggregation Service. -2. `numWorkers`: **OPTIONAL**. Number of lambdas to be invoked in parallel. - Default: 1. +2. `numWorkers`: **OPTIONAL**. Number of lambdas to be invoked in parallel. Default: 1. 3. `numRequests`: **OPTIONAL**. Number of requests per lambda. Default: 1. -4. `timeBetweenRequests`: **OPTIONAL**. Sleep time in between requests. - Default: 10 seconds. +4. `timeBetweenRequests`: **OPTIONAL**. Sleep time in between requests. Default: 10 seconds. +5. `access_key`: **REQUIRED**. AWS access key. +6. `secret_key`: **REQUIRED**. AWS secret access key. +7. `host`: **REQUIRED**. Aggregation server host. +8. `region`: **REQUIRED**. Aggregation service deployed region. +9. `service`: **REQUIRED**. Aggregation service service. +10. `input_data_bucket_name`: **REQUIRED**. +11. `input_data_blob_prefix`: **REQUIRED**. +12. `output_data_bucket_name`: **REQUIRED**. +13. `output_data_blob_prefix`: **REQUIRED**. +14. `attribution_report_to`: **REQUIRED**. +15. `output_domain_bucket_name`: **REQUIRED**. +16. `output_domain_blob_prefix`: **REQUIRED**. +17. `debug_run`: **REQUIRED**. ## Set up ### Install SAM Follow instructions here: -https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html + ### Configure AWS profile @@ -29,13 +40,13 @@ Make sure to have aws credentials configured. 1. You can allocate more resources by updating the `template.yaml` file. 2. You can modify the state machine by updating the `statemachine/AggregationServiceLoadTests.asl.json`. -3. You can update files in `lambda/functions` to update the lambda functions to - be deployed. +3. You can update files in `lambda/functions` to update the lambda functions to be deployed. ### Test -Unit tests are in `functions/*_test.py`. Once you've made changes, please run -tests to ensure your changes work. Tests are run inside docker with bazel. +Unit tests are in `functions/*_test.py`. Once you've made changes, please run tests to ensure your +changes work. Tests are run inside docker with bazel. **NOTE** These tests are not run in CI so you +should run them to ensure everything works. ```sh docker build -t load-test-image /test_image/ @@ -89,17 +100,17 @@ where `event.json` is something like An `event.json` for the `CheckJobsStatus` function would be ```json -[{ - "job_request_ids":[ - "aabbcc4002" - ], - "base_url":"https://pvnx5wx9sg.execute-api.us-east-1.amazonaws.com", - "host":"pvnx5wx9sg.execute-api.us-east-1.amazonaws.com", - "region":"us-east-1", - "access_key":"key", - "secret_key":"secret", - "service":"execute-api" -}] +[ + { + "job_request_ids": ["aabbcc4002"], + "base_url": "https://pvnx5wx9sg.execute-api.us-east-1.amazonaws.com", + "host": "pvnx5wx9sg.execute-api.us-east-1.amazonaws.com", + "region": "us-east-1", + "access_key": "key", + "secret_key": "secret", + "service": "execute-api" + } +] ``` ### Deploy diff --git a/tools/load_tests/functions/BUILD b/tools/load_tests/functions/BUILD index 7150f5ee..5276eb9f 100644 --- a/tools/load_tests/functions/BUILD +++ b/tools/load_tests/functions/BUILD @@ -13,6 +13,7 @@ # limitations under the License. load("@pip_deps//:requirements.bzl", "requirement") +load("@rules_python//python:defs.bzl", "py_library", "py_test") py_library( name = "aggregation_service_load_tests", diff --git a/tools/load_tests/functions/check_jobs_status.py b/tools/load_tests/functions/check_jobs_status.py index e1f7f7b4..ab9105b8 100644 --- a/tools/load_tests/functions/check_jobs_status.py +++ b/tools/load_tests/functions/check_jobs_status.py @@ -36,188 +36,222 @@ def validate_event(event): - "Validates that required keys are present in the event." - keys = [ - "access_key", "base_url", "host", "job_request_ids", "region", - "secret_key", "service" - ] - for key in keys: - if key not in event: - return False - return True + "Validates that required keys are present in the event." + keys = [ + "access_key", + "base_url", + "host", + "job_request_ids", + "region", + "secret_key", + "service", + ] + for key in keys: + if key not in event: + return False + return True def sign(key, msg): - return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest() + return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest() def getSignatureKey(key, dateStamp, regionName, serviceName): - kDate = sign(("AWS4" + key).encode("utf-8"), dateStamp) - kRegion = sign(kDate, regionName) - kService = sign(kRegion, serviceName) - kSigning = sign(kService, "aws4_request") - return kSigning - - -def create_headers(access_key, secret_key, host, region, service, - job_request_id): - t = datetime.utcnow() - amz_date = t.strftime("%Y%m%dT%H%M%SZ") - date_stamp = t.strftime("%Y%m%d") - canonical_uri = ENDPOINT - canonical_querystring = "job_request_id={job_request_id}".format( - job_request_id=job_request_id) - canonical_headers = "host:" + host + "\n" + "x-amz-date:" + amz_date + "\n" - signed_headers = "host;x-amz-date" - payload_hash = hashlib.sha256(("").encode("utf-8")).hexdigest() - canonical_request = ( - METHOD + "\n" + canonical_uri + "\n" + canonical_querystring + "\n" + - canonical_headers + "\n" + signed_headers + "\n" + payload_hash) - algorithm = "AWS4-HMAC-SHA256" - credential_scope = ( - date_stamp + "/" + region + "/" + service + "/" + "aws4_request") - string_to_sign = ( - algorithm + "\n" + amz_date + "\n" + credential_scope + "\n" + - hashlib.sha256(canonical_request.encode("utf-8")).hexdigest()) - signing_key = getSignatureKey(secret_key, date_stamp, region, service) - signature = hmac.new(signing_key, (string_to_sign).encode("utf-8"), - hashlib.sha256).hexdigest() - authorization_header = ( - algorithm + " " + "Credential=" + access_key + "/" + credential_scope + - ", " + "SignedHeaders=" + signed_headers + ", " + "Signature=" + - signature) - return { - "X-Amz-Date": amz_date, - "Authorization": authorization_header, - } + kDate = sign(("AWS4" + key).encode("utf-8"), dateStamp) + kRegion = sign(kDate, regionName) + kService = sign(kRegion, serviceName) + kSigning = sign(kService, "aws4_request") + return kSigning + + +def create_headers(access_key, secret_key, host, region, service, job_request_id): + t = datetime.utcnow() + amz_date = t.strftime("%Y%m%dT%H%M%SZ") + date_stamp = t.strftime("%Y%m%d") + canonical_uri = ENDPOINT + canonical_querystring = "job_request_id={job_request_id}".format( + job_request_id=job_request_id + ) + canonical_headers = "host:" + host + "\n" + "x-amz-date:" + amz_date + "\n" + signed_headers = "host;x-amz-date" + payload_hash = hashlib.sha256(("").encode("utf-8")).hexdigest() + canonical_request = ( + METHOD + + "\n" + + canonical_uri + + "\n" + + canonical_querystring + + "\n" + + canonical_headers + + "\n" + + signed_headers + + "\n" + + payload_hash + ) + algorithm = "AWS4-HMAC-SHA256" + credential_scope = date_stamp + "/" + region + "/" + service + "/" + "aws4_request" + string_to_sign = ( + algorithm + + "\n" + + amz_date + + "\n" + + credential_scope + + "\n" + + hashlib.sha256(canonical_request.encode("utf-8")).hexdigest() + ) + signing_key = getSignatureKey(secret_key, date_stamp, region, service) + signature = hmac.new( + signing_key, (string_to_sign).encode("utf-8"), hashlib.sha256 + ).hexdigest() + authorization_header = ( + algorithm + + " " + + "Credential=" + + access_key + + "/" + + credential_scope + + ", " + + "SignedHeaders=" + + signed_headers + + ", " + + "Signature=" + + signature + ) + return { + "X-Amz-Date": amz_date, + "Authorization": authorization_header, + } def extract_from_response(data): - "Extracts relevant data from the response." - datetime_format = "%Y-%m-%dT%H:%M:%S.%fZ" - received = datetime.strptime(data["request_received_at"], datetime_format) - finished = datetime.strptime(data["request_updated_at"], datetime_format) - job_status = data["job_status"] - return_code = data.get("result_info", {}).get("return_code", "") - error_summary = data.get("result_info", {}).get("error_summary", - {"error_counts": []}) - error_count = 0 - for error in error_summary["error_counts"]: - error_count += error["count"] - return { - "job_status": job_status, - "return_code": return_code, - "error_summary": error_summary, - "error_count": error_count, - "received": received, - "finished": finished, - } + "Extracts relevant data from the response." + datetime_format = "%Y-%m-%dT%H:%M:%S.%fZ" + received = datetime.strptime(data["request_received_at"], datetime_format) + finished = datetime.strptime(data["request_updated_at"], datetime_format) + job_status = data["job_status"] + return_code = data.get("result_info", {}).get("return_code", "") + error_summary = data.get("result_info", {}).get( + "error_summary", {"error_counts": []} + ) + error_count = 0 + for error in error_summary["error_counts"]: + error_count += error["count"] + return { + "job_status": job_status, + "return_code": return_code, + "error_summary": error_summary, + "error_count": error_count, + "received": received, + "finished": finished, + } def make_get_request(job_request_id, event): - "Makes a GET request for a job_request_id." - headers = create_headers(event["access_key"], event["secret_key"], - event["host"], event["region"], event["service"], - job_request_id) - url = "{base_url}{endpoint}?job_request_id={job_request_id}".format( - base_url=event["base_url"], - endpoint=ENDPOINT, - job_request_id=job_request_id) - response = http.request( - METHOD, - url, - headers=headers, - ) - return json.loads(response.data.decode("utf-8")), response.status + "Makes a GET request for a job_request_id." + headers = create_headers( + event["access_key"], + event["secret_key"], + event["host"], + event["region"], + event["service"], + job_request_id, + ) + url = "{base_url}{endpoint}?job_request_id={job_request_id}".format( + base_url=event["base_url"], endpoint=ENDPOINT, job_request_id=job_request_id + ) + response = http.request( + METHOD, + url, + headers=headers, + ) + return json.loads(response.data.decode("utf-8")), response.status def check_job_status(job_request_id, event, received_at, finished_at): - "Checks the status of a single job." - start_time = time.time() - status = False - return_code = "MAX_RETRY_TIME_EXCEEDED" - error_count = 0 - while (time.time() - start_time < MAX_RETRY_TIME): - resp, resp_status = make_get_request(job_request_id, event) - - # Error response code. - if resp_status >= 400: - print(resp) - return_code = "ERROR" - break - - data = extract_from_response(resp) - return_code = data["return_code"] - error_count = data["error_count"] - - # Job is still running. - if data["job_status"] != FINISHED: - print(resp) - time.sleep(1) - continue - - # Job is successful. - if data["return_code"] in SUCCESS_CODES and data["error_count"] == 0: - received_at.append(data["received"]) - finished_at.append(data["finished"]) - status = True - - break - - return { - "status": status, - "return_code": return_code, - "error_count": error_count - } + "Checks the status of a single job." + start_time = time.time() + status = False + return_code = "MAX_RETRY_TIME_EXCEEDED" + error_count = 0 + while time.time() - start_time < MAX_RETRY_TIME: + resp, resp_status = make_get_request(job_request_id, event) + + # Error response code. + if resp_status >= 400: + print(resp) + return_code = "ERROR" + break + + data = extract_from_response(resp) + return_code = data["return_code"] + error_count = data["error_count"] + + # Job is still running. + if data["job_status"] != FINISHED: + print(resp) + time.sleep(1) + continue + + # Job is successful. + if data["return_code"] in SUCCESS_CODES and data["error_count"] == 0: + received_at.append(data["received"]) + finished_at.append(data["finished"]) + status = True + + break + + return {"status": status, "return_code": return_code, "error_count": error_count} def check_jobs_status(event, failed_job_ids, received_at, finished_at): - "Checks the job_status for the list of jobs." - job_request_ids = event["job_request_ids"] - for job_request_id in job_request_ids: - overall_status = ( - check_job_status(job_request_id, event, received_at, finished_at)) - if not overall_status["status"]: - failed_job_ids.append({ - "job_request_id": job_request_id, - "return_code": overall_status["return_code"], - "error_count": overall_status["error_count"], - }) + "Checks the job_status for the list of jobs." + job_request_ids = event["job_request_ids"] + for job_request_id in job_request_ids: + overall_status = check_job_status( + job_request_id, event, received_at, finished_at + ) + if not overall_status["status"]: + failed_job_ids.append( + { + "job_request_id": job_request_id, + "return_code": overall_status["return_code"], + "error_count": overall_status["error_count"], + } + ) def calc_avg_completion_time(received_at, finished_at): - "Calculates the averge time it takes for a job to complete." - no_of_jobs = len(received_at) - _sum = 0 - for idx in range(no_of_jobs): - _sum += (finished_at[idx] - received_at[idx]).seconds - return _sum / no_of_jobs + "Calculates the averge time it takes for a job to complete." + no_of_jobs = len(received_at) + _sum = 0 + for idx in range(no_of_jobs): + _sum += (finished_at[idx] - received_at[idx]).seconds + return _sum / no_of_jobs def lambda_handler(events, context): - failed_job_ids = [] - received_at = [] - finished_at = [] - total = 0 - errot_count = 0 - average_time = None - total_time = None - for event in events: - if not validate_event(event): - raise Exception( - "Please provide access_key, secret_key, host and base_url") - - total += len(event["job_request_ids"]) - check_jobs_status(event, failed_job_ids, received_at, finished_at) - failed = len(failed_job_ids) - if finished_at and received_at: - total_time = (max(finished_at) - min(received_at)).seconds - average_time = calc_avg_completion_time(received_at, finished_at) - return { - "total_jobs": total, - "success_jobs": total - failed, - "failed_jobs": failed, - "failed_job_details": failed_job_ids, - "average_job_completion_time": average_time, - "total_time": total_time, - } + failed_job_ids = [] + received_at = [] + finished_at = [] + total = 0 + errot_count = 0 + average_time = None + total_time = None + for event in events: + if not validate_event(event): + raise Exception("Please provide access_key, secret_key, host and base_url") + + total += len(event["job_request_ids"]) + check_jobs_status(event, failed_job_ids, received_at, finished_at) + failed = len(failed_job_ids) + if finished_at and received_at: + total_time = (max(finished_at) - min(received_at)).seconds + average_time = calc_avg_completion_time(received_at, finished_at) + return { + "total_jobs": total, + "success_jobs": total - failed, + "failed_jobs": failed, + "failed_job_details": failed_job_ids, + "average_job_completion_time": average_time, + "total_time": total_time, + } diff --git a/tools/load_tests/functions/check_jobs_status_test.py b/tools/load_tests/functions/check_jobs_status_test.py index 8c70f7d7..8e715c43 100644 --- a/tools/load_tests/functions/check_jobs_status_test.py +++ b/tools/load_tests/functions/check_jobs_status_test.py @@ -22,291 +22,293 @@ class MockCheckJobStatusSuccessResponse: - status = 200 - data = json.dumps({ - "job_status": "FINISHED", - "request_received_at": "2022-09-16T05:36:37.904Z", - "request_updated_at": "2022-09-16T05:46:37.904Z", - "result_info": { - "return_code": "SUCCESS", - "error_summary": { - "error_counts": [{ - "count": 0 - }] - } - } - }).encode("utf-8") + status = 200 + data = json.dumps( + { + "job_status": "FINISHED", + "request_received_at": "2022-09-16T05:36:37.904Z", + "request_updated_at": "2022-09-16T05:46:37.904Z", + "result_info": { + "return_code": "SUCCESS", + "error_summary": {"error_counts": [{"count": 0}]}, + }, + } + ).encode("utf-8") class MockCheckJobWithSuccessReturnCodeAndErrorCountResponse: - status = 200 - data = json.dumps({ - "job_status": "FINISHED", - "request_received_at": "2022-09-16T05:36:37.904Z", - "request_updated_at": "2022-09-16T05:46:37.904Z", - "result_info": { - "return_code": "SUCCESS", - "error_summary": { - "error_counts": [{ - "count": 1000 - }] - } - } - }).encode("utf-8") + status = 200 + data = json.dumps( + { + "job_status": "FINISHED", + "request_received_at": "2022-09-16T05:36:37.904Z", + "request_updated_at": "2022-09-16T05:46:37.904Z", + "result_info": { + "return_code": "SUCCESS", + "error_summary": {"error_counts": [{"count": 1000}]}, + }, + } + ).encode("utf-8") class MockCheckJobWithErrorReturnCodeResponse: - status = 200 - data = json.dumps({ - "job_status": "FINISHED", - "request_received_at": "2022-09-16T05:36:37.904Z", - "request_updated_at": "2022-09-16T05:46:37.904Z", - "result_info": { - "return_code": "UNSPECIFIED_ERROR", - "error_summary": { - "error_counts": [{ - "count": 0 - }] - } - } - }).encode("utf-8") + status = 200 + data = json.dumps( + { + "job_status": "FINISHED", + "request_received_at": "2022-09-16T05:36:37.904Z", + "request_updated_at": "2022-09-16T05:46:37.904Z", + "result_info": { + "return_code": "UNSPECIFIED_ERROR", + "error_summary": {"error_counts": [{"count": 0}]}, + }, + } + ).encode("utf-8") class MockCheckJobWithErrorCountResponse: - status = 200 - data = json.dumps({ - "job_status": "FINISHED", - "request_received_at": "2022-09-16T05:36:37.904Z", - "request_updated_at": "2022-09-16T05:46:37.904Z", - "result_info": { - "return_code": "UNSPECIFIED_ERROR", - "error_summary": { - "error_counts": [{ - "count": 1000 - }] - } - } - }).encode("utf-8") + status = 200 + data = json.dumps( + { + "job_status": "FINISHED", + "request_received_at": "2022-09-16T05:36:37.904Z", + "request_updated_at": "2022-09-16T05:46:37.904Z", + "result_info": { + "return_code": "UNSPECIFIED_ERROR", + "error_summary": {"error_counts": [{"count": 1000}]}, + }, + } + ).encode("utf-8") class MockCheckJobStatusFailResponse: - status = 400 - data = json.dumps({"error": "Something went wrong"}).encode("utf-8") + status = 400 + data = json.dumps({"error": "Something went wrong"}).encode("utf-8") class CheckJobsStatusTests(TestCase): + def test_check_job_status_no_access_key_raises_exception(self): + with patch("urllib3.PoolManager.request") as resp: + resp.return_value = MockCheckJobStatusSuccessResponse() + event = [ + { + "job_request_ids": ["1", "2"], + "base_url": "foo.bar", + }, + { + "job_request_ids": ["3", "4", "5"], + "base_url": "foo.bar", + }, + ] + with self.assertRaises(Exception): + result = check_jobs_status_handler(event, "") - def test_check_job_status_no_access_key_raises_exception(self): - with patch("urllib3.PoolManager.request") as resp: - resp.return_value = MockCheckJobStatusSuccessResponse() - event = [ - { - "job_request_ids": ["1", "2"], - "base_url": "foo.bar", - }, - { - "job_request_ids": ["3", "4", "5"], - "base_url": "foo.bar", - }, - ] - with self.assertRaises(Exception): - result = check_jobs_status_handler(event, "") + def test_check_job_status_finished(self): + with patch("urllib3.PoolManager.request") as resp: + resp.return_value = MockCheckJobStatusSuccessResponse() + event = [ + { + "job_request_ids": ["1", "2"], + "base_url": "foo.bar", + "access_key": "access", + "secret_key": "secret", + "host": "host", + "region": "us-east-1", + "service": "execute-api", + }, + { + "job_request_ids": ["3", "4", "5"], + "base_url": "foo.bar", + "access_key": "access", + "secret_key": "secret", + "host": "host", + "region": "us-east-1", + "service": "execute-api", + }, + ] + result = check_jobs_status_handler(event, "") + self.assertEqual(result["total_jobs"], 5) + self.assertEqual(result["success_jobs"], 5) + self.assertEqual(result["failed_jobs"], 0) + self.assertEqual(result["average_job_completion_time"], 600) + self.assertEqual(result["total_time"], 600) - def test_check_job_status_finished(self): - with patch("urllib3.PoolManager.request") as resp: - resp.return_value = MockCheckJobStatusSuccessResponse() - event = [ - { - "job_request_ids": ["1", "2"], - "base_url": "foo.bar", - "access_key": "access", - "secret_key": "secret", - "host": "host", - "region": "us-east-1", - "service": "execute-api", - }, - { - "job_request_ids": ["3", "4", "5"], - "base_url": "foo.bar", - "access_key": "access", - "secret_key": "secret", - "host": "host", - "region": "us-east-1", - "service": "execute-api", - }, - ] - result = check_jobs_status_handler(event, "") - self.assertEqual(result["total_jobs"], 5) - self.assertEqual(result["success_jobs"], 5) - self.assertEqual(result["failed_jobs"], 0) - self.assertEqual(result["average_job_completion_time"], 600) - self.assertEqual(result["total_time"], 600) + def test_check_job_status_failed(self): + with patch("urllib3.PoolManager.request") as resp: + resp.return_value = MockCheckJobStatusFailResponse() + event = [ + { + "job_request_ids": ["1", "2"], + "base_url": "foo.bar", + "access_key": "access", + "secret_key": "secret", + "host": "host", + "region": "us-east-1", + "service": "execute-api", + }, + { + "job_request_ids": ["3"], + "base_url": "foo.bar", + "access_key": "access", + "secret_key": "secret", + "host": "host", + "region": "us-east-1", + "service": "execute-api", + }, + ] + result = check_jobs_status_handler(event, "") + self.assertEqual(result["total_jobs"], 3) + self.assertEqual(result["success_jobs"], 0) + self.assertEqual(result["failed_jobs"], 3) + self.assertListEqual( + result["failed_job_details"], + [ + {"job_request_id": "1", "error_count": 0, "return_code": "ERROR"}, + {"job_request_id": "2", "error_count": 0, "return_code": "ERROR"}, + { + "job_request_id": "3", + "error_count": 0, + "return_code": "ERROR", + }, + ], + ) - def test_check_job_status_failed(self): - with patch("urllib3.PoolManager.request") as resp: - resp.return_value = MockCheckJobStatusFailResponse() - event = [ - { - "job_request_ids": ["1", "2"], - "base_url": "foo.bar", - "access_key": "access", - "secret_key": "secret", - "host": "host", - "region": "us-east-1", - "service": "execute-api", - }, - { - "job_request_ids": ["3"], - "base_url": "foo.bar", - "access_key": "access", - "secret_key": "secret", - "host": "host", - "region": "us-east-1", - "service": "execute-api", - }, - ] - result = check_jobs_status_handler(event, "") - self.assertEqual(result["total_jobs"], 3) - self.assertEqual(result["success_jobs"], 0) - self.assertEqual(result["failed_jobs"], 3) - self.assertListEqual(result["failed_job_details"], [{ - "job_request_id": "1", - "error_count": 0, - "return_code": "ERROR" - }, { - "job_request_id": "2", - "error_count": 0, - "return_code": "ERROR" - }, { - "job_request_id": "3", - "error_count": 0, - "return_code": "ERROR", - }]) + def test_job_success_return_code_with_error_counts(self): + with patch("urllib3.PoolManager.request") as resp: + resp.return_value = MockCheckJobWithSuccessReturnCodeAndErrorCountResponse() + event = [ + { + "job_request_ids": ["1", "2"], + "base_url": "foo.bar", + "access_key": "access", + "secret_key": "secret", + "host": "host", + "region": "us-east-1", + "service": "execute-api", + }, + { + "job_request_ids": ["3"], + "base_url": "foo.bar", + "access_key": "access", + "secret_key": "secret", + "host": "host", + "region": "us-east-1", + "service": "execute-api", + }, + ] + result = check_jobs_status_handler(event, "") + self.assertEqual(result["total_jobs"], 3) + self.assertEqual(result["success_jobs"], 0) + self.assertEqual(result["failed_jobs"], 3) + self.assertListEqual( + result["failed_job_details"], + [ + {"job_request_id": "1", "error_count": 1000, "return_code": "SUCCESS"}, + {"job_request_id": "2", "error_count": 1000, "return_code": "SUCCESS"}, + { + "job_request_id": "3", + "error_count": 1000, + "return_code": "SUCCESS", + }, + ], + ) - def test_job_success_return_code_with_error_counts(self): - with patch("urllib3.PoolManager.request") as resp: - resp.return_value = MockCheckJobWithSuccessReturnCodeAndErrorCountResponse( - ) - event = [ - { - "job_request_ids": ["1", "2"], - "base_url": "foo.bar", - "access_key": "access", - "secret_key": "secret", - "host": "host", - "region": "us-east-1", - "service": "execute-api", - }, - { - "job_request_ids": ["3"], - "base_url": "foo.bar", - "access_key": "access", - "secret_key": "secret", - "host": "host", - "region": "us-east-1", - "service": "execute-api", - }, - ] - result = check_jobs_status_handler(event, "") - self.assertEqual(result["total_jobs"], 3) - self.assertEqual(result["success_jobs"], 0) - self.assertEqual(result["failed_jobs"], 3) - self.assertListEqual(result["failed_job_details"], [{ - "job_request_id": "1", - "error_count": 1000, - "return_code": "SUCCESS" - }, { - "job_request_id": "2", - "error_count": 1000, - "return_code": "SUCCESS" - }, { - "job_request_id": "3", - "error_count": 1000, - "return_code": "SUCCESS", - }]) + def test_job_error_return_code_with_error_counts(self): + with patch("urllib3.PoolManager.request") as resp: + resp.return_value = MockCheckJobWithErrorCountResponse() + event = [ + { + "job_request_ids": ["1", "2"], + "base_url": "foo.bar", + "access_key": "access", + "secret_key": "secret", + "host": "host", + "region": "us-east-1", + "service": "execute-api", + }, + { + "job_request_ids": ["3"], + "base_url": "foo.bar", + "access_key": "access", + "secret_key": "secret", + "host": "host", + "region": "us-east-1", + "service": "execute-api", + }, + ] + result = check_jobs_status_handler(event, "") + self.assertEqual(result["total_jobs"], 3) + self.assertEqual(result["success_jobs"], 0) + self.assertEqual(result["failed_jobs"], 3) + self.assertListEqual( + result["failed_job_details"], + [ + { + "job_request_id": "1", + "error_count": 1000, + "return_code": "UNSPECIFIED_ERROR", + }, + { + "job_request_id": "2", + "error_count": 1000, + "return_code": "UNSPECIFIED_ERROR", + }, + { + "job_request_id": "3", + "error_count": 1000, + "return_code": "UNSPECIFIED_ERROR", + }, + ], + ) - def test_job_error_return_code_with_error_counts(self): - with patch("urllib3.PoolManager.request") as resp: - resp.return_value = MockCheckJobWithErrorCountResponse() - event = [ - { - "job_request_ids": ["1", "2"], - "base_url": "foo.bar", - "access_key": "access", - "secret_key": "secret", - "host": "host", - "region": "us-east-1", - "service": "execute-api", - }, - { - "job_request_ids": ["3"], - "base_url": "foo.bar", - "access_key": "access", - "secret_key": "secret", - "host": "host", - "region": "us-east-1", - "service": "execute-api", - }, - ] - result = check_jobs_status_handler(event, "") - self.assertEqual(result["total_jobs"], 3) - self.assertEqual(result["success_jobs"], 0) - self.assertEqual(result["failed_jobs"], 3) - self.assertListEqual(result["failed_job_details"], [{ - "job_request_id": "1", - "error_count": 1000, - "return_code": "UNSPECIFIED_ERROR" - }, { - "job_request_id": "2", - "error_count": 1000, - "return_code": "UNSPECIFIED_ERROR" - }, { - "job_request_id": "3", - "error_count": 1000, - "return_code": "UNSPECIFIED_ERROR", - }]) - - def test_job_error_return_code_with_no_error_counts(self): - with patch("urllib3.PoolManager.request") as resp: - resp.return_value = MockCheckJobWithErrorReturnCodeResponse() - event = [ - { - "job_request_ids": ["1", "2"], - "base_url": "foo.bar", - "access_key": "access", - "secret_key": "secret", - "host": "host", - "region": "us-east-1", - "service": "execute-api", - }, - { - "job_request_ids": ["3"], - "base_url": "foo.bar", - "access_key": "access", - "secret_key": "secret", - "host": "host", - "region": "us-east-1", - "service": "execute-api", - }, - ] - result = check_jobs_status_handler(event, "") - self.assertEqual(result["total_jobs"], 3) - self.assertEqual(result["success_jobs"], 0) - self.assertEqual(result["failed_jobs"], 3) - self.assertListEqual(result["failed_job_details"], [{ - "job_request_id": "1", - "error_count": 0, - "return_code": "UNSPECIFIED_ERROR" - }, { - "job_request_id": "2", - "error_count": 0, - "return_code": "UNSPECIFIED_ERROR" - }, { - "job_request_id": "3", - "error_count": 0, - "return_code": "UNSPECIFIED_ERROR", - }]) + def test_job_error_return_code_with_no_error_counts(self): + with patch("urllib3.PoolManager.request") as resp: + resp.return_value = MockCheckJobWithErrorReturnCodeResponse() + event = [ + { + "job_request_ids": ["1", "2"], + "base_url": "foo.bar", + "access_key": "access", + "secret_key": "secret", + "host": "host", + "region": "us-east-1", + "service": "execute-api", + }, + { + "job_request_ids": ["3"], + "base_url": "foo.bar", + "access_key": "access", + "secret_key": "secret", + "host": "host", + "region": "us-east-1", + "service": "execute-api", + }, + ] + result = check_jobs_status_handler(event, "") + self.assertEqual(result["total_jobs"], 3) + self.assertEqual(result["success_jobs"], 0) + self.assertEqual(result["failed_jobs"], 3) + self.assertListEqual( + result["failed_job_details"], + [ + { + "job_request_id": "1", + "error_count": 0, + "return_code": "UNSPECIFIED_ERROR", + }, + { + "job_request_id": "2", + "error_count": 0, + "return_code": "UNSPECIFIED_ERROR", + }, + { + "job_request_id": "3", + "error_count": 0, + "return_code": "UNSPECIFIED_ERROR", + }, + ], + ) if __name__ == "__main__": - unittest_main() + unittest_main() diff --git a/tools/load_tests/functions/data_generation.py b/tools/load_tests/functions/data_generation.py index 4189a634..12baa85c 100644 --- a/tools/load_tests/functions/data_generation.py +++ b/tools/load_tests/functions/data_generation.py @@ -19,5 +19,5 @@ def lambda_handler(event, context): - num_of_workers = int(event.get("numWorkers", "1")) - return [event for _ in range(num_of_workers)] + num_of_workers = int(event.get("numWorkers", "1")) + return [event for _ in range(num_of_workers)] diff --git a/tools/load_tests/functions/data_generation_test.py b/tools/load_tests/functions/data_generation_test.py index 0ed7be57..11a1b22f 100644 --- a/tools/load_tests/functions/data_generation_test.py +++ b/tools/load_tests/functions/data_generation_test.py @@ -19,16 +19,15 @@ class DataGenerationTests(TestCase): + def test_data_generation_defaults(self): + self.assertEqual(len(data_generation_handler({}, "")), 1) - def test_data_generation_defaults(self): - self.assertEqual(len(data_generation_handler({}, "")), 1) - - def test_data_generation(self): - num_workers = 3 - self.assertEqual( - len(data_generation_handler({"numWorkers": num_workers}, "")), - num_workers) + def test_data_generation(self): + num_workers = 3 + self.assertEqual( + len(data_generation_handler({"numWorkers": num_workers}, "")), num_workers + ) if __name__ == "__main__": - unittest_main() + unittest_main() diff --git a/tools/load_tests/functions/trigger_jobs.py b/tools/load_tests/functions/trigger_jobs.py index 39a2be8e..f8d6f7b2 100644 --- a/tools/load_tests/functions/trigger_jobs.py +++ b/tools/load_tests/functions/trigger_jobs.py @@ -37,137 +37,181 @@ def sign(key, msg): - return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest() + return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest() def getSignatureKey(key, dateStamp, regionName, serviceName): - kDate = sign(("AWS4" + key).encode("utf-8"), dateStamp) - kRegion = sign(kDate, regionName) - kService = sign(kRegion, serviceName) - kSigning = sign(kService, "aws4_request") - return kSigning + kDate = sign(("AWS4" + key).encode("utf-8"), dateStamp) + kRegion = sign(kDate, regionName) + kService = sign(kRegion, serviceName) + kSigning = sign(kService, "aws4_request") + return kSigning def create_headers(payload, access_key, secret_key, host, region, service): - t = datetime.datetime.utcnow() - amz_date = t.strftime("%Y%m%dT%H%M%SZ") - date_stamp = t.strftime("%Y%m%d") - canonical_uri = ENDPOINT - canonical_querystring = "" - payload_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest() - canonical_headers = ("host:" + host + "\n" + "x-amz-content-sha256:" + - payload_hash + "\n" + "x-amz-date:" + amz_date + "\n") - signed_headers = "host;x-amz-content-sha256;x-amz-date" - canonical_request = ( - METHOD + "\n" + canonical_uri + "\n" + canonical_querystring + "\n" + - canonical_headers + "\n" + signed_headers + "\n" + payload_hash) - algorithm = "AWS4-HMAC-SHA256" - credential_scope = ( - date_stamp + "/" + region + "/" + service + "/" + "aws4_request") - string_to_sign = ( - algorithm + "\n" + amz_date + "\n" + credential_scope + "\n" + - hashlib.sha256(canonical_request.encode("utf-8")).hexdigest()) - signing_key = getSignatureKey(secret_key, date_stamp, region, service) - signature = hmac.new(signing_key, (string_to_sign).encode("utf-8"), - hashlib.sha256).hexdigest() - authorization_header = ( - algorithm + " " + "Credential=" + access_key + "/" + credential_scope + - ", " + "SignedHeaders=" + signed_headers + ", " + "Signature=" + - signature) - return { - "X-Amz-Date": amz_date, - "Authorization": authorization_header, - "X-Amz-Content-Sha256": payload_hash, - "Content-Type": CONTENT_TYPE, - } - - -def generate_payload(debug_run, attribution_report_to, input_data_blob_prefix, - input_data_bucket_name, output_data_blob_prefix, - output_data_bucket_name, output_domain_bucket_name, - output_domain_blob_prefix): - job_request_id = str(uuid.uuid4()) - output_prefix = ( - "output-data/{job_request_id}/{output_data_blob_prefix}".format( - job_request_id=job_request_id, - output_data_blob_prefix=output_data_blob_prefix)) - return { - "job_request_id": job_request_id, - "input_data_blob_prefix": input_data_blob_prefix, - "input_data_bucket_name": input_data_bucket_name, - "output_data_blob_prefix": output_prefix, - "output_data_bucket_name": output_data_bucket_name, - "postback_url": "fizz.com/api/buzz", - "job_parameters": { - "attribution_report_to": attribution_report_to, - "output_domain_blob_prefix": output_domain_blob_prefix, - "output_domain_bucket_name": output_domain_bucket_name, - "debug_run": debug_run, - } - } + t = datetime.datetime.utcnow() + amz_date = t.strftime("%Y%m%dT%H%M%SZ") + date_stamp = t.strftime("%Y%m%d") + canonical_uri = ENDPOINT + canonical_querystring = "" + payload_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest() + canonical_headers = ( + "host:" + + host + + "\n" + + "x-amz-content-sha256:" + + payload_hash + + "\n" + + "x-amz-date:" + + amz_date + + "\n" + ) + signed_headers = "host;x-amz-content-sha256;x-amz-date" + canonical_request = ( + METHOD + + "\n" + + canonical_uri + + "\n" + + canonical_querystring + + "\n" + + canonical_headers + + "\n" + + signed_headers + + "\n" + + payload_hash + ) + algorithm = "AWS4-HMAC-SHA256" + credential_scope = date_stamp + "/" + region + "/" + service + "/" + "aws4_request" + string_to_sign = ( + algorithm + + "\n" + + amz_date + + "\n" + + credential_scope + + "\n" + + hashlib.sha256(canonical_request.encode("utf-8")).hexdigest() + ) + signing_key = getSignatureKey(secret_key, date_stamp, region, service) + signature = hmac.new( + signing_key, (string_to_sign).encode("utf-8"), hashlib.sha256 + ).hexdigest() + authorization_header = ( + algorithm + + " " + + "Credential=" + + access_key + + "/" + + credential_scope + + ", " + + "SignedHeaders=" + + signed_headers + + ", " + + "Signature=" + + signature + ) + return { + "X-Amz-Date": amz_date, + "Authorization": authorization_header, + "X-Amz-Content-Sha256": payload_hash, + "Content-Type": CONTENT_TYPE, + } + + +def generate_payload( + debug_run, + attribution_report_to, + input_data_blob_prefix, + input_data_bucket_name, + output_data_blob_prefix, + output_data_bucket_name, + output_domain_bucket_name, + output_domain_blob_prefix, +): + job_request_id = str(uuid.uuid4()) + output_prefix = "output-data/{job_request_id}/{output_data_blob_prefix}".format( + job_request_id=job_request_id, output_data_blob_prefix=output_data_blob_prefix + ) + return { + "job_request_id": job_request_id, + "input_data_blob_prefix": input_data_blob_prefix, + "input_data_bucket_name": input_data_bucket_name, + "output_data_blob_prefix": output_prefix, + "output_data_bucket_name": output_data_bucket_name, + "postback_url": "fizz.com/api/buzz", + "job_parameters": { + "attribution_report_to": attribution_report_to, + "output_domain_blob_prefix": output_domain_blob_prefix, + "output_domain_bucket_name": output_domain_bucket_name, + "debug_run": debug_run, + }, + } def lambda_handler(event, context): - access_key = event.get("access_key") - base_url = event.get("base_url") - host = event.get("host") - num_requests = int(event.get("numRequests", "1")) - region = event.get("region", "us-east-1") - secret_key = event.get("secret_key") - service = event.get("service", "execute-api") - sleep_time = int(event.get("timeBetweenRequests", "10")) - - if not (access_key and secret_key and host and base_url): - raise Exception("Please provide access_key, secret_key, host and base_url") - debug_run = event.get("debug_run") - attribution_report_to = event.get("attribution_report_to") - input_data_blob_prefix = event.get("input_data_blob_prefix") - input_data_bucket_name = event.get("input_data_bucket_name") - output_data_blob_prefix = event.get("output_data_blob_prefix") - output_data_bucket_name = event.get("output_data_bucket_name") - output_domain_bucket_name = event.get("output_domain_bucket_name") - output_domain_blob_prefix = event.get("output_domain_blob_prefix") - - url = "{base_url}{endpoint}".format(base_url=base_url, endpoint=ENDPOINT) - job_request_ids = [] - http = urllib3.PoolManager() - success = 0 - failed = 0 - for _ in range(num_requests): - payload_dict = generate_payload( - debug_run, attribution_report_to, input_data_blob_prefix, - input_data_bucket_name, output_data_blob_prefix, - output_data_bucket_name, output_domain_bucket_name, - output_domain_blob_prefix) - - payload = json.dumps(payload_dict) - headers = create_headers(payload, access_key, secret_key, host, region, - service) - print(url) - print(headers) - print(payload) - response = http.request( - METHOD, - url, - headers=headers, - body=payload, - ) - if response.status >= 400: - data = json.loads(response.data.decode("utf-8")) - print(data) - failed += 1 - else: - success += 1 - job_request_ids.append(payload_dict["job_request_id"]) - time.sleep(sleep_time) - return { - "success": success, - "failed": failed, - "job_request_ids": job_request_ids, - "base_url": base_url, - "host": host, - "region": region, - "access_key": access_key, - "secret_key": secret_key, - "service": service, - } + access_key = event.get("access_key") + base_url = event.get("base_url") + host = event.get("host") + num_requests = int(event.get("numRequests", "1")) + region = event.get("region", "us-east-1") + secret_key = event.get("secret_key") + service = event.get("service", "execute-api") + sleep_time = int(event.get("timeBetweenRequests", "10")) + + if not (access_key and secret_key and host and base_url): + raise Exception("Please provide access_key, secret_key, host and base_url") + debug_run = event.get("debug_run") + attribution_report_to = event.get("attribution_report_to") + input_data_blob_prefix = event.get("input_data_blob_prefix") + input_data_bucket_name = event.get("input_data_bucket_name") + output_data_blob_prefix = event.get("output_data_blob_prefix") + output_data_bucket_name = event.get("output_data_bucket_name") + output_domain_bucket_name = event.get("output_domain_bucket_name") + output_domain_blob_prefix = event.get("output_domain_blob_prefix") + + url = "{base_url}{endpoint}".format(base_url=base_url, endpoint=ENDPOINT) + job_request_ids = [] + http = urllib3.PoolManager() + success = 0 + failed = 0 + for _ in range(num_requests): + payload_dict = generate_payload( + debug_run, + attribution_report_to, + input_data_blob_prefix, + input_data_bucket_name, + output_data_blob_prefix, + output_data_bucket_name, + output_domain_bucket_name, + output_domain_blob_prefix, + ) + + payload = json.dumps(payload_dict) + headers = create_headers(payload, access_key, secret_key, host, region, service) + print(url) + print(headers) + print(payload) + response = http.request( + METHOD, + url, + headers=headers, + body=payload, + ) + if response.status >= 400: + data = json.loads(response.data.decode("utf-8")) + print(data) + failed += 1 + else: + success += 1 + job_request_ids.append(payload_dict["job_request_id"]) + time.sleep(sleep_time) + return { + "success": success, + "failed": failed, + "job_request_ids": job_request_ids, + "base_url": base_url, + "host": host, + "region": region, + "access_key": access_key, + "secret_key": secret_key, + "service": service, + } diff --git a/tools/load_tests/functions/trigger_jobs_test.py b/tools/load_tests/functions/trigger_jobs_test.py index a58e5cd7..b6c5abc8 100644 --- a/tools/load_tests/functions/trigger_jobs_test.py +++ b/tools/load_tests/functions/trigger_jobs_test.py @@ -22,98 +22,97 @@ class MockTriggerJobSuccessResponse: - status = 202 + status = 202 class MockTriggerJobBadRequestResponse: - status = 400 - data = json.dumps({"message": "bad request"}).encode("utf-8") + status = 400 + data = json.dumps({"message": "bad request"}).encode("utf-8") class MockTriggerJobServerErrorResponse: - status = 500 - data = json.dumps({"message": "server error"}).encode("utf-8") + status = 500 + data = json.dumps({"message": "server error"}).encode("utf-8") class TriggerJobsTests(TestCase): - - def test_trigger_jobs_no_access_key_raises_exception(self): - with patch("urllib3.PoolManager.request") as resp: - resp.return_value = MockTriggerJobSuccessResponse() - event = { - "base_url": "foo.bar", - "numRequests": 3, - "timeBetweenRequests": "0", - } - with self.assertRaises(Exception): - result = trigger_jobs_handler(event, "") - - def test_trigger_jobs_successfully(self): - with patch("urllib3.PoolManager.request") as resp: - resp.return_value = MockTriggerJobSuccessResponse() - event = { - "base_url": "foo.bar", - "numRequests": 3, - "timeBetweenRequests": "0", - "access_key": "key", - "secret_key": "secret", - "host": "host", - "debug_run": "false", - "attribution_report_to": "a", - "input_data_blob_prefix": "a", - "input_data_bucket_name": "a", - "output_data_blob_prefix": "a", - "output_data_bucket_name": "a", - "output_domain_bucket_name": "a", - "output_domain_blob_prefix": "a", - } - result = trigger_jobs_handler(event, "") - self.assertEqual(result["success"], event["numRequests"]) - - def test_trigger_jobs_with_bad_request(self): - with patch("urllib3.PoolManager.request") as resp: - resp.return_value = MockTriggerJobBadRequestResponse() - event = { - "base_url": "foo.bar", - "numRequests": 2, - "timeBetweenRequests": "0", - "access_key": "key", - "secret_key": "secret", - "host": "host", - "debug_run": "false", - "attribution_report_to": "a", - "input_data_blob_prefix": "a", - "input_data_bucket_name": "a", - "output_data_blob_prefix": "a", - "output_data_bucket_name": "a", - "output_domain_bucket_name": "a", - "output_domain_blob_prefix": "a", - } - result = trigger_jobs_handler(event, "") - self.assertEqual(result["failed"], event["numRequests"]) - - def test_trigger_jobs_server_error(self): - with patch("urllib3.PoolManager.request") as resp: - resp.return_value = MockTriggerJobServerErrorResponse() - event = { - "base_url": "foo.bar", - "numRequests": 2, - "timeBetweenRequests": "0", - "access_key": "key", - "secret_key": "secret", - "host": "host", - "debug_run": "false", - "attribution_report_to": "a", - "input_data_blob_prefix": "a", - "input_data_bucket_name": "a", - "output_data_blob_prefix": "a", - "output_data_bucket_name": "a", - "output_domain_bucket_name": "a", - "output_domain_blob_prefix": "a", - } - result = trigger_jobs_handler(event, "") - self.assertEqual(result["failed"], event["numRequests"]) + def test_trigger_jobs_no_access_key_raises_exception(self): + with patch("urllib3.PoolManager.request") as resp: + resp.return_value = MockTriggerJobSuccessResponse() + event = { + "base_url": "foo.bar", + "numRequests": 3, + "timeBetweenRequests": "0", + } + with self.assertRaises(Exception): + result = trigger_jobs_handler(event, "") + + def test_trigger_jobs_successfully(self): + with patch("urllib3.PoolManager.request") as resp: + resp.return_value = MockTriggerJobSuccessResponse() + event = { + "base_url": "foo.bar", + "numRequests": 3, + "timeBetweenRequests": "0", + "access_key": "key", + "secret_key": "secret", + "host": "host", + "debug_run": "false", + "attribution_report_to": "a", + "input_data_blob_prefix": "a", + "input_data_bucket_name": "a", + "output_data_blob_prefix": "a", + "output_data_bucket_name": "a", + "output_domain_bucket_name": "a", + "output_domain_blob_prefix": "a", + } + result = trigger_jobs_handler(event, "") + self.assertEqual(result["success"], event["numRequests"]) + + def test_trigger_jobs_with_bad_request(self): + with patch("urllib3.PoolManager.request") as resp: + resp.return_value = MockTriggerJobBadRequestResponse() + event = { + "base_url": "foo.bar", + "numRequests": 2, + "timeBetweenRequests": "0", + "access_key": "key", + "secret_key": "secret", + "host": "host", + "debug_run": "false", + "attribution_report_to": "a", + "input_data_blob_prefix": "a", + "input_data_bucket_name": "a", + "output_data_blob_prefix": "a", + "output_data_bucket_name": "a", + "output_domain_bucket_name": "a", + "output_domain_blob_prefix": "a", + } + result = trigger_jobs_handler(event, "") + self.assertEqual(result["failed"], event["numRequests"]) + + def test_trigger_jobs_server_error(self): + with patch("urllib3.PoolManager.request") as resp: + resp.return_value = MockTriggerJobServerErrorResponse() + event = { + "base_url": "foo.bar", + "numRequests": 2, + "timeBetweenRequests": "0", + "access_key": "key", + "secret_key": "secret", + "host": "host", + "debug_run": "false", + "attribution_report_to": "a", + "input_data_blob_prefix": "a", + "input_data_bucket_name": "a", + "output_data_blob_prefix": "a", + "output_data_bucket_name": "a", + "output_domain_bucket_name": "a", + "output_domain_blob_prefix": "a", + } + result = trigger_jobs_handler(event, "") + self.assertEqual(result["failed"], event["numRequests"]) if __name__ == "__main__": - unittest_main() + unittest_main() diff --git a/worker/aws/BUILD b/worker/aws/BUILD index b9a13dfa..39a640bd 100644 --- a/worker/aws/BUILD +++ b/worker/aws/BUILD @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@com_google_adm_cloud_scp//operator/worker/aws/build_defs:deploy.bzl", "worker_aws_deployment") load("@bazel_skylib//rules:common_settings.bzl", "string_flag") load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") +load("@com_google_adm_cloud_scp//operator/worker/aws/build_defs:deploy.bzl", "worker_aws_deployment") package(default_visibility = ["//visibility:public"]) @@ -51,7 +51,7 @@ DEFAULT_ENCLAVE_ARGS = [ "--job_client", "AWS", "--result_logger", - "LOCAL_AVRO_TO_S3", + "LOCAL_TO_CLOUD", # coord-staging environment "--private_key_service_base_url", "https://il9t0ajpsa.execute-api.us-west-2.amazonaws.com/stage/v1alpha", @@ -88,7 +88,7 @@ PROD_ENCLAVE_ARGS = [ "--job_client", "AWS", "--result_logger", - "LOCAL_AVRO_TO_S3", + "LOCAL_TO_CLOUD", "--decryption_key_service", "AWS_ENCLAVE_CLI_DECRYPTION_KEY_SERVICE", "--trusted_party_region_override", diff --git a/worker/aws/testing/BUILD b/worker/aws/testing/BUILD index db0dd741..c9b9e6a7 100644 --- a/worker/aws/testing/BUILD +++ b/worker/aws/testing/BUILD @@ -24,7 +24,7 @@ DEFAULT_ENCLAVE_ARGS = [ "--job_client", "AWS", "--result_logger", - "LOCAL_AVRO_TO_S3", + "LOCAL_TO_CLOUD", # coord-staging environment "--private_key_service_base_url", "https://il9t0ajpsa.execute-api.us-west-2.amazonaws.com/stage/v1alpha", @@ -48,7 +48,7 @@ RELEASE_ENCLAVE_ARGS = [ "--job_client", "AWS", "--result_logger", - "LOCAL_AVRO_TO_S3", + "LOCAL_TO_CLOUD", "--decryption_key_service", "AWS_ENCLAVE_CLI_DECRYPTION_KEY_SERVICE", "--param_client", @@ -74,7 +74,6 @@ worker_aws_deployment( "0.1", "--noising_l1_sensitivity", "4", - "--domain_optional", # TODO(b/227210339) remove this and provide the output domain for these tests # Ignore privacy budgeting when using single coordinator "--privacy_budgeting", "UNLIMITED", @@ -97,7 +96,6 @@ worker_aws_deployment( "0.1", "--noising_l1_sensitivity", "4", - "--domain_optional", # TODO(b/227210339) remove this and provide the output domain for these tests # Enforce privacy budgeting "--privacy_budgeting", "HTTP", @@ -137,7 +135,6 @@ worker_aws_deployment( "0.1", "--noising_l1_sensitivity", "4", - "--domain_optional", # TODO(b/227210339) remove this and provide the output domain for these tests # Enforce privacy budgeting "--privacy_budgeting", "HTTP", @@ -243,7 +240,6 @@ worker_aws_deployment( "--timer_exporter", "AWS", "--benchmark", - "--domain_optional", # TODO(b/227210339) remove this and provide the output domain for these tests ], jar_file = "/WorkerRunner_deploy.jar", repo_name = TEE_REPO_NAME, diff --git a/worker/testing/data/library/expected_output_set_0/output.json b/worker/testing/data/library/expected_output_set_0/output.json index bced0210..583d1cf0 100644 --- a/worker/testing/data/library/expected_output_set_0/output.json +++ b/worker/testing/data/library/expected_output_set_0/output.json @@ -16,4 +16,4 @@ }, { "bucket" : "key2", "value" : 13000000 -} ] \ No newline at end of file +} ] diff --git a/worker/testing/data/library/expected_output_set_2/output.json b/worker/testing/data/library/expected_output_set_2/output.json index 03ddbe54..b936be28 100644 --- a/worker/testing/data/library/expected_output_set_2/output.json +++ b/worker/testing/data/library/expected_output_set_2/output.json @@ -16,4 +16,4 @@ }, { "bucket" : "key12", "value" : 0 -} ] \ No newline at end of file +} ] diff --git a/worker/testing/data/library/expected_output_set_3/output.json b/worker/testing/data/library/expected_output_set_3/output.json index 3206248f..63368911 100644 --- a/worker/testing/data/library/expected_output_set_3/output.json +++ b/worker/testing/data/library/expected_output_set_3/output.json @@ -16,4 +16,4 @@ }, { "bucket" : "key12", "value" : 0 -} ] \ No newline at end of file +} ] diff --git a/worker/testing/data/library/input_set_0/flags.txt b/worker/testing/data/library/input_set_0/flags.txt index f652c417..61a163fc 100644 --- a/worker/testing/data/library/input_set_0/flags.txt +++ b/worker/testing/data/library/input_set_0/flags.txt @@ -1,3 +1,3 @@ epsilon:64 no_noise:true -skip_domain:true \ No newline at end of file +skip_domain:true diff --git a/worker/testing/data/library/input_set_0/text_batch_data.txt b/worker/testing/data/library/input_set_0/text_batch_data.txt index c6d79b55..983520f6 100644 --- a/worker/testing/data/library/input_set_0/text_batch_data.txt +++ b/worker/testing/data/library/input_set_0/text_batch_data.txt @@ -7,4 +7,4 @@ key1:7000000,key2:1000000 key1:1000000,key3:1000000 key2:9000000,key4:1000000 key5:1000000,key6:1000000 -key1:1000000,key6:3000000 \ No newline at end of file +key1:1000000,key6:3000000 diff --git a/worker/testing/data/library/input_set_0_old/flags.txt b/worker/testing/data/library/input_set_0_old/flags.txt index f652c417..61a163fc 100644 --- a/worker/testing/data/library/input_set_0_old/flags.txt +++ b/worker/testing/data/library/input_set_0_old/flags.txt @@ -1,3 +1,3 @@ epsilon:64 no_noise:true -skip_domain:true \ No newline at end of file +skip_domain:true diff --git a/worker/testing/data/library/input_set_0_old/text_batch_data.txt b/worker/testing/data/library/input_set_0_old/text_batch_data.txt index c6d79b55..983520f6 100644 --- a/worker/testing/data/library/input_set_0_old/text_batch_data.txt +++ b/worker/testing/data/library/input_set_0_old/text_batch_data.txt @@ -7,4 +7,4 @@ key1:7000000,key2:1000000 key1:1000000,key3:1000000 key2:9000000,key4:1000000 key5:1000000,key6:1000000 -key1:1000000,key6:3000000 \ No newline at end of file +key1:1000000,key6:3000000 diff --git a/worker/testing/data/library/input_set_1/flags.txt b/worker/testing/data/library/input_set_1/flags.txt index ed99faa8..8d732e91 100644 --- a/worker/testing/data/library/input_set_1/flags.txt +++ b/worker/testing/data/library/input_set_1/flags.txt @@ -2,4 +2,4 @@ epsilon:64 no_noise:false domain_required:true no_overlap_keys:true -total_keys_in_domain:10000 \ No newline at end of file +total_keys_in_domain:10000 diff --git a/worker/testing/data/library/input_set_1/text_batch_data.txt b/worker/testing/data/library/input_set_1/text_batch_data.txt index c6d79b55..983520f6 100644 --- a/worker/testing/data/library/input_set_1/text_batch_data.txt +++ b/worker/testing/data/library/input_set_1/text_batch_data.txt @@ -7,4 +7,4 @@ key1:7000000,key2:1000000 key1:1000000,key3:1000000 key2:9000000,key4:1000000 key5:1000000,key6:1000000 -key1:1000000,key6:3000000 \ No newline at end of file +key1:1000000,key6:3000000 diff --git a/worker/testing/data/library/input_set_1_old/flags.txt b/worker/testing/data/library/input_set_1_old/flags.txt index ed99faa8..8d732e91 100644 --- a/worker/testing/data/library/input_set_1_old/flags.txt +++ b/worker/testing/data/library/input_set_1_old/flags.txt @@ -2,4 +2,4 @@ epsilon:64 no_noise:false domain_required:true no_overlap_keys:true -total_keys_in_domain:10000 \ No newline at end of file +total_keys_in_domain:10000 diff --git a/worker/testing/data/library/input_set_1_old/text_batch_data.txt b/worker/testing/data/library/input_set_1_old/text_batch_data.txt index c6d79b55..983520f6 100644 --- a/worker/testing/data/library/input_set_1_old/text_batch_data.txt +++ b/worker/testing/data/library/input_set_1_old/text_batch_data.txt @@ -7,4 +7,4 @@ key1:7000000,key2:1000000 key1:1000000,key3:1000000 key2:9000000,key4:1000000 key5:1000000,key6:1000000 -key1:1000000,key6:3000000 \ No newline at end of file +key1:1000000,key6:3000000 diff --git a/worker/testing/data/library/input_set_2/domain.txt b/worker/testing/data/library/input_set_2/domain.txt index 3ac4a7ad..5cfb15f3 100644 --- a/worker/testing/data/library/input_set_2/domain.txt +++ b/worker/testing/data/library/input_set_2/domain.txt @@ -3,4 +3,4 @@ key2 key3 key4 key11 -key12 \ No newline at end of file +key12 diff --git a/worker/testing/data/library/input_set_2/flags.txt b/worker/testing/data/library/input_set_2/flags.txt index 1e4bb911..9983be3a 100644 --- a/worker/testing/data/library/input_set_2/flags.txt +++ b/worker/testing/data/library/input_set_2/flags.txt @@ -1,4 +1,4 @@ epsilon:64 no_noise:true domain_required:true -extra_keys:true \ No newline at end of file +extra_keys:true diff --git a/worker/testing/data/library/input_set_2/text_batch_data.txt b/worker/testing/data/library/input_set_2/text_batch_data.txt index c6d79b55..983520f6 100644 --- a/worker/testing/data/library/input_set_2/text_batch_data.txt +++ b/worker/testing/data/library/input_set_2/text_batch_data.txt @@ -7,4 +7,4 @@ key1:7000000,key2:1000000 key1:1000000,key3:1000000 key2:9000000,key4:1000000 key5:1000000,key6:1000000 -key1:1000000,key6:3000000 \ No newline at end of file +key1:1000000,key6:3000000 diff --git a/worker/testing/data/library/input_set_2_old/domain.txt b/worker/testing/data/library/input_set_2_old/domain.txt index 3ac4a7ad..5cfb15f3 100644 --- a/worker/testing/data/library/input_set_2_old/domain.txt +++ b/worker/testing/data/library/input_set_2_old/domain.txt @@ -3,4 +3,4 @@ key2 key3 key4 key11 -key12 \ No newline at end of file +key12 diff --git a/worker/testing/data/library/input_set_2_old/flags.txt b/worker/testing/data/library/input_set_2_old/flags.txt index 1e4bb911..9983be3a 100644 --- a/worker/testing/data/library/input_set_2_old/flags.txt +++ b/worker/testing/data/library/input_set_2_old/flags.txt @@ -1,4 +1,4 @@ epsilon:64 no_noise:true domain_required:true -extra_keys:true \ No newline at end of file +extra_keys:true diff --git a/worker/testing/data/library/input_set_2_old/text_batch_data.txt b/worker/testing/data/library/input_set_2_old/text_batch_data.txt index c6d79b55..983520f6 100644 --- a/worker/testing/data/library/input_set_2_old/text_batch_data.txt +++ b/worker/testing/data/library/input_set_2_old/text_batch_data.txt @@ -7,4 +7,4 @@ key1:7000000,key2:1000000 key1:1000000,key3:1000000 key2:9000000,key4:1000000 key5:1000000,key6:1000000 -key1:1000000,key6:3000000 \ No newline at end of file +key1:1000000,key6:3000000 diff --git a/worker/testing/data/library/input_set_3/domain.txt b/worker/testing/data/library/input_set_3/domain.txt index 2bd14731..ee11c216 100644 --- a/worker/testing/data/library/input_set_3/domain.txt +++ b/worker/testing/data/library/input_set_3/domain.txt @@ -3,4 +3,4 @@ key14 key15 key16 key11 -key12 \ No newline at end of file +key12 diff --git a/worker/testing/data/library/input_set_3/flags.txt b/worker/testing/data/library/input_set_3/flags.txt index c73fb52d..8627e9a1 100644 --- a/worker/testing/data/library/input_set_3/flags.txt +++ b/worker/testing/data/library/input_set_3/flags.txt @@ -1,4 +1,4 @@ epsilon:64 no_noise:true domain_required:true -no_overlap_keys:true \ No newline at end of file +no_overlap_keys:true diff --git a/worker/testing/data/library/input_set_3/text_batch_data.txt b/worker/testing/data/library/input_set_3/text_batch_data.txt index c6d79b55..983520f6 100644 --- a/worker/testing/data/library/input_set_3/text_batch_data.txt +++ b/worker/testing/data/library/input_set_3/text_batch_data.txt @@ -7,4 +7,4 @@ key1:7000000,key2:1000000 key1:1000000,key3:1000000 key2:9000000,key4:1000000 key5:1000000,key6:1000000 -key1:1000000,key6:3000000 \ No newline at end of file +key1:1000000,key6:3000000 diff --git a/worker/testing/data/library/input_set_3_old/domain.txt b/worker/testing/data/library/input_set_3_old/domain.txt index 2bd14731..ee11c216 100644 --- a/worker/testing/data/library/input_set_3_old/domain.txt +++ b/worker/testing/data/library/input_set_3_old/domain.txt @@ -3,4 +3,4 @@ key14 key15 key16 key11 -key12 \ No newline at end of file +key12 diff --git a/worker/testing/data/library/input_set_3_old/flags.txt b/worker/testing/data/library/input_set_3_old/flags.txt index c73fb52d..8627e9a1 100644 --- a/worker/testing/data/library/input_set_3_old/flags.txt +++ b/worker/testing/data/library/input_set_3_old/flags.txt @@ -1,4 +1,4 @@ epsilon:64 no_noise:true domain_required:true -no_overlap_keys:true \ No newline at end of file +no_overlap_keys:true diff --git a/worker/testing/data/library/input_set_3_old/text_batch_data.txt b/worker/testing/data/library/input_set_3_old/text_batch_data.txt index c6d79b55..983520f6 100644 --- a/worker/testing/data/library/input_set_3_old/text_batch_data.txt +++ b/worker/testing/data/library/input_set_3_old/text_batch_data.txt @@ -7,4 +7,4 @@ key1:7000000,key2:1000000 key1:1000000,key3:1000000 key2:9000000,key4:1000000 key5:1000000,key6:1000000 -key1:1000000,key6:3000000 \ No newline at end of file +key1:1000000,key6:3000000 diff --git a/worker/testing/data/sqs_aggregation_request.txt b/worker/testing/data/sqs_aggregation_request.txt index a08a69de..2021c583 100644 --- a/worker/testing/data/sqs_aggregation_request.txt +++ b/worker/testing/data/sqs_aggregation_request.txt @@ -1 +1 @@ -CgASDS9yZXBvcnRzLmF2cm8aACIA \ No newline at end of file +CgASDS9yZXBvcnRzLmF2cm8aACIA