diff --git a/.github/workflows/make-test.yml b/.github/workflows/make-test.yml new file mode 100644 index 0000000..a2c1107 --- /dev/null +++ b/.github/workflows/make-test.yml @@ -0,0 +1,14 @@ +on: + push +name: make test +jobs: + test: + name: make test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + with: + fetch-depth: '2' + - name: make test + run: make test + shell: bash \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e974752 --- /dev/null +++ b/Makefile @@ -0,0 +1,53 @@ +.PHONY: build +build: ## Builds all the dockerfiles in the repository. + @$(CURDIR)/build-all.sh + +.PHONY: latest-versions +latest-versions: ## Checks all the latest versions of the Dockerfile contents. + @$(CURDIR)/latest-versions.sh + +check_defined = \ + $(strip $(foreach 1,$1, \ + $(call __check_defined,$1,$(strip $(value 2))))) +__check_defined = \ + $(if $(value $1),, \ + $(error Undefined $1$(if $2, ($2))$(if $(value @), \ + required by target `$@'))) + +.PHONY: run +run: ## Run a Dockerfile from the command at the top of the file (ex. DIR=telnet). + @:$(call check_defined, DIR, directory of the Dockefile) + @$(CURDIR)/run.sh "$(DIR)" + +REGISTRY := r.j3ss.co +.PHONY: image +image: ## Build a Dockerfile (ex. DIR=telnet). + @:$(call check_defined, DIR, directory of the Dockefile) + docker build --rm --force-rm -t $(REGISTRY)/$(subst /,:,$(patsubst %/,%,$(DIR))) ./$(DIR) + +.PHONY: test +test: dockerfiles shellcheck ## Runs the tests on the repository. + +.PHONY: dockerfiles +dockerfiles: ## Tests the changes to the Dockerfiles build. + @$(CURDIR)/test.sh + +# if this session isn't interactive, then we don't want to allocate a +# TTY, which would fail, but if it is interactive, we do want to attach +# so that the user can send e.g. ^C through. +INTERACTIVE := $(shell [ -t 0 ] && echo 1 || echo 0) +ifeq ($(INTERACTIVE), 1) + DOCKER_FLAGS += -t +endif + +.PHONY: shellcheck +shellcheck: ## Runs the shellcheck tests on the scripts. + docker run --rm -i $(DOCKER_FLAGS) \ + --name df-shellcheck \ + -v $(CURDIR):/usr/src:ro \ + --workdir /usr/src \ + r.j3ss.co/shellcheck ./shellcheck.sh + +.PHONY: help +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/README.md b/README.md index 0cb2216..79270bd 100755 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A collection of dockerfiles I have made for different applications. Of note there are lots of other great collections of dockerfiles such as: -- Jess's: [jfrazele/dockerfiles](https://github.com/jfrazelle/dockerfiles) +- Jess's: [jfrazele/dockerfiles](https://github.com/jfrazelle/dockerfiles) Which is where the build and test script for this repo come from! - Sven's: [SvenDowideit/dockerfiles](https://github.com/svendowideit/dockerfiles) - Kev++'s: [vimagick/dockerfiles](https://github.com/vimagick/dockerfiles) diff --git a/build-all.sh b/build-all.sh new file mode 100755 index 0000000..ae12f0b --- /dev/null +++ b/build-all.sh @@ -0,0 +1,92 @@ +#!/bin/bash +set -e +set -o pipefail + +SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" +REPO_URL="${REPO_URL:-r.j3ss.co}" +JOBS=${JOBS:-2} + +ERRORS="$(pwd)/errors" + +build_and_push(){ + base=$1 + suite=$2 + build_dir=$3 + + echo "Building ${REPO_URL}/${base}:${suite} for context ${build_dir}" + docker build --rm --force-rm -t "${REPO_URL}/${base}:${suite}" "${build_dir}" || return 1 + + # on successful build, push the image + echo " --- " + echo "Successfully built ${base}:${suite} with context ${build_dir}" + echo " --- " + + # try push a few times because notary server sometimes returns 401 for + # absolutely no reason + n=0 + until [ $n -ge 5 ]; do + docker push --disable-content-trust=false "${REPO_URL}/${base}:${suite}" && break + echo "Try #$n failed... sleeping for 15 seconds" + n=$((n+1)) + sleep 15 + done + + # also push the tag latest for "stable" (chrome), "tools" (wireguard) or "3.5" tags for zookeeper + if [[ "$suite" == "stable" ]] || [[ "$suite" == "3.6" ]] || [[ "$suite" == "tools" ]]; then + docker tag "${REPO_URL}/${base}:${suite}" "${REPO_URL}/${base}:latest" + docker push --disable-content-trust=false "${REPO_URL}/${base}:latest" + fi +} + +dofile() { + f=$1 + image=${f%Dockerfile} + base=${image%%\/*} + build_dir=$(dirname "$f") + suite=${build_dir##*\/} + + if [[ -z "$suite" ]] || [[ "$suite" == "$base" ]]; then + suite=latest + fi + + { + $SCRIPT build_and_push "${base}" "${suite}" "${build_dir}" + } || { + # add to errors + echo "${base}:${suite}" >> "$ERRORS" +} +echo +echo +} + +main(){ + # get the dockerfiles + IFS=$'\n' + mapfile -t files < <(find -L . -iname '*Dockerfile' | sed 's|./||' | sort) + unset IFS + + # build all dockerfiles + echo "Running in parallel with ${JOBS} jobs." + parallel --tag --verbose --ungroup -j"${JOBS}" "$SCRIPT" dofile "{1}" ::: "${files[@]}" + + if [[ ! -f "$ERRORS" ]]; then + echo "No errors, hooray!" + else + echo "[ERROR] Some images did not build correctly, see below." >&2 + echo "These images failed: $(cat "$ERRORS")" >&2 + exit 1 + fi +} + +run(){ + args=$* + f=$1 + + if [[ "$f" == "" ]]; then + main "$args" + else + $args + fi +} + +run "$@" diff --git a/latest-versions.sh b/latest-versions.sh new file mode 100755 index 0000000..067c998 --- /dev/null +++ b/latest-versions.sh @@ -0,0 +1,189 @@ +#!/bin/bash +# This script gets the latest GitHub releases for the specified projects. + +if [[ -z "$GITHUB_TOKEN" ]]; then + echo "Set the GITHUB_TOKEN env variable." + exit 1 +fi + +URI=https://api.github.com +API_VERSION=v3 +API_HEADER="Accept: application/vnd.github.${API_VERSION}+json" +AUTH_HEADER="Authorization: token ${GITHUB_TOKEN}" + +get_latest() { + local repo=$1 + + local resp + resp=$(curl -sSL -H "${AUTH_HEADER}" -H "${API_HEADER}" "${URI}/repos/${repo}/releases") + if [[ "$repo" != "Radarr/Radarr" ]]; then + resp=$(echo "$resp" | jq --raw-output '[.[] | select(.prerelease == false)]') + fi + local tag + tag=$(echo "$resp" | jq -e --raw-output .[0].tag_name) + local name + name=$(echo "$resp" | jq -e --raw-output .[0].name) + + if [[ "$tag" == "null" ]]; then + # get the latest tag + resp=$(curl -sSL -H "${AUTH_HEADER}" -H "${API_HEADER}" "${URI}/repos/${repo}/tags") + tag=$(echo "$resp" | jq -e --raw-output .[0].name) + tag=${tag#release-} + fi + + if [[ "$name" == "null" ]] || [[ "$name" == "" ]]; then + name="-" + fi + + local dir=${repo#*/} + + if [[ "$dir" == "CouchPotatoServer" ]]; then + dir="couchpotato" + elif [[ "$dir" == "cri-o" ]]; then + dir="crio" + elif [[ "$dir" == "byte-unixbench" ]]; then + dir="unixbench" + elif [[ "$dir" == "Tautulli" ]]; then + dir="plexpy" + elif [[ "$dir" == "zookeeper" ]]; then + dir="zookeeper/3.6" + elif [[ "$dir" == "oauth2_proxy" ]]; then + dir="oauth2-proxy" + fi + + # Change to upper case for grep + local udir + udir=$(echo $dir | awk '{print toupper($0)}') + # Replace dashes (-) with underscores (_) + udir=${udir//-/_} + udir=${udir%/*} + + if [[ "$dir" == "wireguard-tools" ]]; then + dir="wireguard/install" + udir="WIREGUARD_TOOLS" + elif [[ "$dir" == "wireguard-linux-compat" ]]; then + dir="wireguard/install" + udir="WIREGUARD" + fi + + local current + if [[ ! -d "$dir" ]]; then + # If the directory does not exist, then grep all for it + current=$(grep -m 1 "${udir}_VERSION" -- **/Dockerfile | head -n 1 | awk '{print $(NF)}') + else + current=$(grep -m 1 "${udir}_VERSION" "${dir}/Dockerfile" | awk '{print $(NF)}') + fi + + + compare "$name" "$dir" "$tag" "$current" "https://github.com/${repo}/releases" +} + +get_latest_unifi() { + local latest current + latest=$(curl -sSL http://www.ubnt.com/downloads/unifi/debian/dists/cloudkey-stable/ubiquiti/binary-armhf/Packages \ + | awk 'BEGIN {FS="\n"; RS="";} /^Package: unifi/' \ + | awk '/^Version:/ {print $2}' \ + | cut -d- -f1) + + current=$(grep -m 1 UNIFI_VERSION unifi/Dockerfile | tr '"' ' ' | awk '{print $(NF)}') + + compare unifi unifi "$latest" "$current" https://www.ubnt.com/download/unifi +} + +compare() { + local name="$1" dir="$2" tag="$3" current="$4" releases="$5" + ignore_dirs=( "mc" "zookeeper/3.6" ) + + if [[ "$tag" =~ $current ]] || [[ "$name" =~ $current ]] || [[ "$current" =~ $tag ]] || [[ "$current" == "master" ]]; then + echo -e "\\e[36m${dir}:\\e[39m current ${current} | ${tag} | ${name}" + else + # add to the bad versions + if [[ ! " ${ignore_dirs[*]} " =~ ${dir} ]]; then + bad_versions+=( "${dir}" ) + fi + echo -e "\\e[31m${dir}:\\e[39m current ${current} | ${tag} | ${name} | ${releases}" + fi +} + +projects=( + iovisor/bcc + iovisor/bpftrace + browsh-org/browsh + certbot/certbot + cloudflare/cfssl + quay/clair + hashicorp/consul + coredns/coredns + CouchPotato/CouchPotatoServer + curl/curl + kolide/fleet + GoogleCloudPlatform/cloud-sdk-docker + google/gitiles + google/guetzli + irssi/irssi + cryptodotis/irssi-otr + keepassxreboot/keepassxc + robertdavidgraham/masscan + MidnightCommander/mc + zyedidia/micro + mitmproxy/mitmproxy + hashicorp/nomad + nzbget/nzbget + pusher/oauth2_proxy + facebook/osquery + hashicorp/packer + Tautulli/Tautulli + perkeep/perkeep + pomerium/pomerium + powershell/powershell + Radarr/Radarr + cesanta/docker_auth + ricochet-im/ricochet + reverse-shell/routersploit + rstudio/rstudio + tarsnap/tarsnap + nginx/nginx + simplresty/ngx_devel_kit + openresty/luajit2 + openresty/lua-cjson + openresty/lua-nginx-module + leev/ngx_http_geoip2_module + maxmind/libmaxminddb + openresty/lua-resty-core + openresty/lua-resty-lrucache + hashicorp/terraform + kdlucas/byte-unixbench + mitchellh/vagrant + hashicorp/vault + containrrr/watchtower + wireguard/wireguard-tools + wireguard/wireguard-linux-compat + znc/znc + apache/zookeeper + tianon/gosu +) + +other_projects=( + unifi +) + +bad_versions=() + +main() { + # shellcheck disable=SC2068 + for p in ${projects[@]}; do + get_latest "$p" + done + # shellcheck disable=SC2068 + for p in ${other_projects[@]}; do + get_latest_"$p" + done + + if [[ ${#bad_versions[@]} -ne 0 ]]; then + echo + echo "These Dockerfiles are not up to date: ${bad_versions[*]}" >&2 + exit 1 + fi +} + +main diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..5aa93ec --- /dev/null +++ b/run.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# +# This script allows you to launch several images +# from this repository once they're built. +# +# Make sure you add the `docker run` command +# in the header of the Dockerfile so the script +# can find it and execute it. +# +# Use pulseaudio/Dockerfile and skype/Dockerfile as examples. +set -e +set -o pipefail + +if [[ $# -eq 0 ]]; then + echo "Usage: $0 [--test] image1 image2 ..." + exit 1 +fi + +if [[ "$1" = "--test" ]]; then + TEST=1 + shift +fi + +for name in "$@"; do + if [[ ! -d "$name" ]]; then + echo "Unable to find container configuration with name: $name" + exit 1 + fi + + script=$(sed -n '/docker run/,/^#$/p' "$name/Dockerfile" | head -n -1 | sed "s/#//" | sed "s#\\\\##" | tr '\n' ' ' | sed "s/\$@//" | sed 's/""//') + echo "Running: $script" + + if [ $TEST ]; then + echo "$script" + else + eval "$script" + fi + + shift +done diff --git a/shellcheck.sh b/shellcheck.sh new file mode 100755 index 0000000..45ea112 --- /dev/null +++ b/shellcheck.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -e +set -o pipefail + +ERRORS=() + +# find all executables and run `shellcheck` +for f in $(find . -type f -not -iwholename '*.git*' -not -name "Dockerfile" | sort -u); do + if file "$f" | grep --quiet shell; then + { + shellcheck "$f" && echo "[OK]: sucessfully linted $f" + } || { + # add to errors + ERRORS+=("$f") + } + fi +done + +if [ ${#ERRORS[@]} -eq 0 ]; then + echo "No errors, hooray" +else + echo "These files failed shellcheck: ${ERRORS[*]}" + exit 1 +fi diff --git a/test.sh b/test.sh index c61e15d..9bf09dd 100755 --- a/test.sh +++ b/test.sh @@ -1,5 +1,6 @@ #!/bin/bash set -e +set -o pipefail # this is kind of an expensive check, so let's not do this twice if we # are running more than one validate bundlescript @@ -23,19 +24,21 @@ validate_diff() { # get the dockerfiles changed IFS=$'\n' +# shellcheck disable=SC2207 files=( $(validate_diff --name-only -- '*Dockerfile') ) unset IFS # build the changed dockerfiles -for f in "${files[@]}"; do +# shellcheck disable=SC2068 +for f in ${files[@]}; do if ! [[ -e "$f" ]]; then continue fi - image=${f%Dockerfile} - base=${image%%\/*} - suite=${image##*\/} - build_dir=$(dirname $f) + build_dir=$(dirname "$f") + base="${build_dir%%\/*}" + suite="${build_dir##$base}" + suite="${suite##\/}" if [[ -z "$suite" ]]; then suite=latest @@ -43,7 +46,7 @@ for f in "${files[@]}"; do ( set -x - sudo docker build -t ${base}:${suite} ${build_dir} + docker build -t "${base}:${suite}" "${build_dir}" ) echo " --- "