Update README and remove credstash where still in use.

README.md

@@ -33,43 +33,25 @@ CI and CD are performed autonomously with Taskcluster and the [Orion Decision](h
 
 #### Usage
 
-You can build, test and push locally, which is great for testing locally. In order to do
-that run the command below and adjust the path to the service you want to interact and the
-repository `DOCKER_ORG` to which you intent to push. `DOCKER_ORG` is used as tag name for the image.
-
-> Note that you might want to edit the `service.yaml` of the image too, if you intent to make use of
-> custom `build_args`, parent images and manifest destinations.
-
-```bash
-#!/usr/bin/env bash
-export DOCKER_ORG=<DOCKER_USERNAME>
-export TRAVIS_PULL_REQUEST=false
-export TRAVIS_BRANCH=master
-export TRAVIS_EVENT_TYPE=cron
-./monorepo.py -ci travis -build -test -deliver -path core/linux
-./monorepo.py -ci travis -build -test -deliver -path base/linux/fuzzos
-```
-
-```
-make help
-```
+You can build, test and push locally, which is great for testing locally. The commands below are general,
+and each service may have more specific instructions defined in the README.md of the service.
 
-#### Testing
+    TAG=dev
+    docker build -t mozillasecurity/service:$TAG ../.. -f Dockerfile
 
-Before a build task is initiated in Taskcluster, each shell script and Dockerfile undergo a linting and testing process which may or may not abort each succeeding task. To ensure your Dockerfile passes, you are encouraged to install the [`pre-commit`](https://pre-commit.com/) hook (`pre-commit install`) prior to commit, and to run any tests defined in the service folder before pushing your commit.
+... or to test the latest build:
+
+    TAG=latest
 
-#### Known Issues
+Running the fuzzer locally:
 
-#### error creating overlay mount to /var/lib/docker/overlay2/<...>/merged: device or resource busy
+    eval $(TASKCLUSTER_ROOT_URL=https://community-tc.services.mozilla.com taskcluster signin)
+    LOGS="logs-$(date +%Y%m%d%H%M%S)"
+    mkdir -p "$LOGS"
+    docker run --rm -e TASKCLUSTER_ROOT_URL -e TASKCLUSTER_CLIENT_ID -e TASKCLUSTER_ACCESS_TOKEN -it -v "$(pwd)/$LOGS":/logs mozillasecurity/service:$TAG 2>&1 | tee "$LOGS/live.log"
 
-Workaround: https://github.com/docker/for-linux/issues/711
+... add any environment variables required by the fuzzer using `-e VAR=value`. Some fuzzer images alter kernel sysctls and will require `docker run --privileged`.
 
-```
-$ sudo systemctl stop docker
-$ sudo nano /etc/docker/daemon.json
-{
-  "max-concurrent-uploads": 1
-}
-$ sudo systemctl start docker
-$ docker push [...]
-```
+#### Testing
+
+Before a build task is initiated in Taskcluster, each shell script and Dockerfile undergo a linting and testing process which may or may not abort each succeeding task. To ensure your Dockerfile passes, you are encouraged to install the [`pre-commit`](https://pre-commit.com/) hook (`pre-commit install`) prior to commit, and to run any tests defined in the service folder before pushing your commit.

services/funfuzz/funfuzz.sh

@@ -10,9 +10,9 @@ set -o pipefail
 # shellcheck source=recipes/linux/common.sh
 source ~/.local/bin/common.sh
 
-if [[ ! -e ~/.fuzzmanagerconf ]] && [[ -z "$NO_CREDSTASH" ]]
+if [[ ! -e ~/.fuzzmanagerconf ]] && [[ -z "$NO_SECRETS" ]]
 then
-  retry credstash get fuzzmanagerconf > ~/.fuzzmanagerconf
+  get-tc-secret fuzzmanagerconf ~/.fuzzmanagerconf
   setup-fuzzmanager-hostname
   chmod 0600 ~/.fuzzmanagerconf
 fi

services/funfuzz/launch.sh

@@ -12,13 +12,10 @@ source ~worker/.local/bin/common.sh
 
 if [[ "$(id -u)" = "0" ]]
 then
-  if [[ -z "$NO_CREDSTASH" ]]
+  if [[ -z "$NO_SECRETS" ]]
   then
-    # In some environments, we require credentials for talking to credstash
-    su worker -c ". ~/.local/bin/common.sh && setup-aws-credentials"
-    mkdir -p /etc/google/auth /etc/td-agent-bit
-    su worker -c ". ~/.local/bin/common.sh && retry credstash get google-logging-creds.json" > /etc/google/auth/application_default_credentials.json
-    chmod 0600 /etc/google/auth/application_default_credentials.json
+    get-tc-secret google-logging-creds /etc/google/auth/application_default_credentials.json raw
+    mkdir -p /etc/td-agent-bit
     cat > /etc/td-agent-bit/td-agent-bit.conf << EOF
 [SERVICE]
     Daemon       On

services/funfuzz/setup.sh

@@ -14,8 +14,6 @@ source "${0%/*}/common.sh"
 
 cd "${0%/*}"
 ./js32_deps.sh  # does the initial sys-update
-./berglas.sh
-./credstash.sh
 ./fluentbit.sh
 ./fuzzfetch.sh
 SRCDIR=/tmp/fuzzing-tc ./fuzzing_tc.sh

services/grizzly-android/recipes/setup.sh

@@ -33,7 +33,7 @@ apt-install-auto zstd
 )
 
 pip3 install -r /tmp/recipes/requirements.txt
-python3 /tmp/recipes/emulator.py install
+#python3 /tmp/recipes/emulator.py install
 
 ~worker/.local/bin/cleanup.sh
 

services/grizzly/README.md

@@ -1,69 +1,22 @@
-#### Example: Faulty
-```bash
-CID=$(docker run \
--e MOZ_IPC_MESSAGE_LOG=1 \
--e FAULTY_PROBABILITY=40000 \
--e FAULTY_LARGE_VALUES=1 \
--e FAULTY_ENABLE_LOGGING=1 \
--e FAULTY_PICKLE=1 \
--e FAULTY_PARENT=1 \
--e CORPMAN=ipc \
--e TARGET="-a --fuzzing" \
--e CACHE=4 \
--e INSTANCES=3 \
--e TOOLNAME=grizzly-ipc-faulty \
--e FAULTY_PARENT=1 \
--e TIMEOUT=45 \
--e INPUT=grammars/html-fuzz.gmr \
--e PREFS=prefs/prefs-default-e10s.js \
--e RELAUNCH=100 \
--dit mozillasecurity/grizzly:latest /bin/bash)
-docker exec -it $CID /bin/bash
-```
-
 #### Example: Domino
 ```bash
-CID=$(docker run \
+eval $(TASKCLUSTER_ROOT_URL=https://community-tc.services.mozilla.com taskcluster signin)
+docker run \
 -e DOMINO_ROOT=domino \
--e CORPMAN=dominode \
+-e ADAPTER=dominode \
 -e TARGET=asan \
 -e IGNORE="log-limit memory timeout" \
--e BEARSPRAY=1 \
--e CACHE=5 \
+-e COLLECT=5 \
 -e INSTANCES_PER_CORE=0 \
 -e TOOLNAME=grizzly-domino \
 -e TIMEOUT=90 \
 -e INPUT=domino/package.json \
--e PREFS=prefs/prefs-default-e10s.js \
+-e PREFS=default \
 -e RELAUNCH=250 \
 -e MEM_LIMIT=7000 \
--v $HOME/.aws:/home/worker/.aws \
--dit mozillasecurity/grizzly:latest)
+-e TASKCLUSTER_ROOT_URL \
+-e TASKCLUSTER_CLIENT_ID \
+-e TASKCLUSTER_ACCESS_TOKEN \
+--rm -it mozillasecurity/grizzly
 docker exec -uworker -it $CID /bin/bash
 ```
-
-#### Run
-```bash
-docker run -dit -e ENV taskclusterprivate/grizzly:latest /bin/bash
-```
-
-#### Enter the container (1)
-```bash
-docker exec -it <CONTAINER_ID> /bin/bash
-```
-
-#### Enter the container (2)
-###### **Note**: Needs ^P^Q to exit without destroying the container.
-```bash
-docker attach <CONTAINER_ID>
-```
-
-#### Stop
-```bash
-docker stop <CONTAINER_ID>
-```
-
-#### References
-
-* https://blog.docker.com/2014/06/why-you-dont-need-to-run-sshd-in-docker/
-* https://stackoverflow.com/questions/28212380/why-docker-container-exits-immediately

services/libfuzzer/README.md

@@ -1,17 +1,41 @@
 ## Local Development
 
-The production Dockerfile uses credstash to fetch credentials for our private FuzzManager instance. If you do not have the credentials of your FuzzManager instance stored in a KMS or similar database and/or you want to develop on this image then the following approach is recommended.
+The production Dockerfile uses Taskcluster secrets to fetch credentials for our private FuzzManager instance. If you do not have the credentials of your FuzzManager instance stored in a KMS or similar database and/or you want to develop on this image then the following examples are recommended.
+
+Building locally:
+
+    TAG=dev
+    docker build -t mozillasecurity/libfuzzer:$TAG ../.. -f Dockerfile
+
+... or to test the latest build:
+
+    TAG=latest
+
+Testing locally:
+
+    eval $(TASKCLUSTER_ROOT_URL=https://community-tc.services.mozilla.com taskcluster signin)
+    LOGS="logs-$(date +%Y%m%d%H%M%S)"
+    mkdir -p "$LOGS"
+    docker run --rm -it \
+        -e TASKCLUSTER_ROOT_URL \
+        -e TASKCLUSTER_CLIENT_ID \
+        -e TASKCLUSTER_ACCESS_TOKEN \
+        -v "$(pwd)/$LOGS":/logs \
+        mozillasecurity/libfuzzer:$TAG 2>&1 | tee "$LOGS/live.log"
+
+... add any environment variables required by the fuzzer using `-e VAR=value`
 
 ### Example: LibFuzzer Coverage Run
 
 ```bash
-REVISION=$(curl -sL https://build.fuzzing.mozilla.org/builds/coverage-revision.txt)
-fuzzfetch --build "$REVISION" --fuzzing --coverage -a --gtest -n firefox
+REVISION="$(curl --compressed -sSL https://community-tc.services.mozilla.com/api/index/v1/task/project.fuzzing.coverage-revision.latest/artifacts/public/coverage-revision.txt)"
+fuzzfetch --build "$REVISION" --asan --fuzzing --coverage --gtest -n firefox
 
 docker run \
-    -h `hostname` \
+    -h `uname -n` \
     -v ~/.fuzzmanagerconf:/home/worker/.fuzzmanagerconf \
-    -v ~/firefox/:/home/worker/firefox \
+    -v $PWD/firefox/:/home/worker/firefox \
+    -e NO_SECRETS=1 \
     -e COVERAGE=1 \
     -e COVRUNTIME=600 \
     -e LIBFUZZER_ARGS=-max_total_time=180 \
@@ -26,12 +50,13 @@ It is recommended to reserve at least 4GB of memory for containers running cover
 ### Example: LibFuzzer Run
 
 ```bash
-fuzzfetch --fuzzing --coverage -a --gtest -n firefox
+fuzzfetch --fuzzing --asan --gtest -n firefox
 
 docker run \
-    -h `hostname` \
+    -h `uname -n` \
     -v ~/.fuzzmanagerconf:/home/worker/.fuzzmanagerconf \
-    -v ~/firefox/:/home/worker/firefox \
+    -v $PWD/firefox/:/home/worker/firefox \
+    -e NO_SECRETS=1 \
     -e TOKENS=dicts/sdp.dict \
     -e FUZZER=SdpParser \
     -e CORPORA=samples/sdp/ \
@@ -46,11 +71,12 @@ If you want to use local corpora you can mount the folder containing the corpora
 
 ```bash
 docker run \
- -h `hostname` \
- -v ~/.fuzzmanagerconf:/home/worker/.fuzzmanagerconf \
- -v LOCAL_FOLDER:/home/worker/corpora/ \
- -e FUZZER=Dav1dDecode \
- --rm -it mozillasecurity/libfuzzer
+    -h `uname -n` \
+    -v ~/.fuzzmanagerconf:/home/worker/.fuzzmanagerconf \
+    -v LOCAL_FOLDER:/home/worker/corpora/ \
+    -e NO_SECRETS=1 \
+    -e FUZZER=Dav1dDecode \
+    --rm -it mozillasecurity/libfuzzer
 ```
 
 In case you run the container on EC2 or a similar service, you can use `-e SHIP=<ProviderName>` and omit the `-h` parameter, which will determine the correct hostname of the container host for sending it to FuzzManager.

services/libfuzzer/launch.sh

@@ -12,13 +12,10 @@ source ~worker/.local/bin/common.sh
 
 if [[ "$(id -u)" = "0" ]]
 then
-  if [[ -z "$NO_CREDSTASH" ]]
+  if [[ -z "$NO_SECRETS" ]]
   then
-    # In some environments, we require credentials for talking to credstash
-    su worker -c ". ~/.local/bin/common.sh && setup-aws-credentials"
-    mkdir -p /etc/google/auth /etc/td-agent-bit
-    su worker -c ". ~/.local/bin/common.sh && retry credstash get google-logging-creds.json" > /etc/google/auth/application_default_credentials.json
-    chmod 0600 /etc/google/auth/application_default_credentials.json
+    get-tc-secret google-logging-creds /etc/google/auth/application_default_credentials.json raw
+    mkdir -p /etc/td-agent-bit
     cat > /etc/td-agent-bit/td-agent-bit.conf << EOF
 [SERVICE]
     Daemon       On

services/libfuzzer/libfuzzer.sh

@@ -20,11 +20,16 @@ then
   exit 1
 fi
 
+if [[ -z "$NO_SECRETS" ]]
+then
+  # setup AWS credentials to use S3
+  setup-aws-credentials
+fi
+
 mkdir -p ~/.ssh
-if [[ ! -e ~/.ssh/id_rsa.fuzzing-shells-private ]] && [[ -z "$NO_CREDSTASH" ]]
+if [[ ! -e ~/.ssh/id_rsa.fuzzing-shells-private ]] && [[ -z "$NO_SECRETS" ]]
 then
-  retry credstash get deploy-fuzzing-shells-private.pem > ~/.ssh/id_rsa.fuzzing-shells-private
-  chmod 0600 ~/.ssh/id_*
+  get-tc-secret deploy-fuzzing-shells-private ~/.ssh/id_rsa.fuzzing-shells-private
 fi
 cat >> ~/.ssh/config << EOF
 Host fuzzing-shells-private github.com
@@ -38,9 +43,9 @@ then
   then
     git-clone https://github.com/google/oss-fuzz.git
   fi
-  if [[ ! -f "$HOME/.boto" ]] && [[ -z "$NO_CREDSTASH" ]]
+  if [[ ! -f "$HOME/.boto" ]] && [[ -z "$NO_SECRETS" ]]
   then
-    retry credstash get ossfuzz.gutils >> ~/.boto
+    get-tc-secret ossfuzz-gutils >> ~/.boto
   fi
 fi
 
@@ -52,11 +57,11 @@ then
   JS=1
 fi
 
-# Get FuzzManager configuration from credstash.
+# Get FuzzManager configuration
 # We require FuzzManager credentials in order to submit our results.
-if [[ ! -e ~/.fuzzmanagerconf ]] && [[ -z "$NO_CREDSTASH" ]]
+if [[ ! -e ~/.fuzzmanagerconf ]] && [[ -z "$NO_SECRETS" ]]
 then
-  retry credstash get fuzzmanagerconf > .fuzzmanagerconf
+  get-tc-secret fuzzmanagerconf .fuzzmanagerconf
   # Update FuzzManager config for this instance.
   mkdir -p signatures
   cat >> .fuzzmanagerconf << EOF

services/libfuzzer/setup-target.sh

@@ -36,13 +36,11 @@ then
     cp "$HOME/js/dist/bin/js.fuzzmanagerconf" "$HOME/js/dist/bin/fuzz-tests.fuzzmanagerconf"
     TARGET_BIN="js/dist/bin/fuzz-tests"
   fi
-  chmod -R 0755 "$HOME/js"
 else
   TARGET_BIN="firefox/firefox"
   if [[ ! -d "$HOME/firefox" ]]
   then
     retry fuzzfetch -n firefox --gtest "${FETCH_ARGS[@]}"
   fi
-  chmod -R 0755 "$HOME/firefox"
 fi
 echo "$TARGET_BIN"

services/libfuzzer/setup.sh

@@ -19,8 +19,6 @@ sys-update
 cd "${0%/*}"
 ./htop.sh
 ./fuzzfetch.sh
-./credstash.sh
-./berglas.sh
 EDIT=1 ./fuzzmanager.sh
 ./grcov.sh
 ./gsutil.sh