Add Nightly CI (#659)

Signed-off-by: quicksilver <[email protected]>

.github/workflows/nightly_ci.yml

@@ -0,0 +1,59 @@
+name: Nightly CI
+on:
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    #         ┌───────────── minute (0 - 59)
+    #         │ ┌───────────── hour (0 - 23)
+    #         │ │ ┌───────────── day of the month (1 - 31)
+    #         │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
+    #         │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
+    #         │ │ │ │ │
+    #         │ │ │ │ │
+    #         │ │ │ │ │
+    - cron:  '* 10 * * *'
+
+jobs:
+  nightly:
+    name: Run Nightly CI
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7]
+    env:
+      IMAGE_REPO: "milvusdb"
+      TAG_PREFIX: "master-"
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Get the latest of Milvus dev image tag
+      shell: bash
+      id: extracter
+      working-directory: ci/scripts
+      run: echo "::set-output name=tag::$(./docker_image_find_tag.sh -n ${IMAGE_REPO}/milvus-dev -t ${TAG_PREFIX}latest -f ${TAG_PREFIX} -F -L -q)"
+
+    - name: Install Milvus
+      working-directory: ci/docker/milvus
+      run: |
+        IMAGE_TAG=${{ steps.extracter.outputs.tag }}  docker-compose up -d
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Clone Milvus Repo
+      uses: actions/checkout@v2
+      with:
+        repository: milvus-io/milvus
+        path: 'milvus'
+
+    - name: Install PyMilvus
+      run: |
+        python3 -m pip install --no-cache-dir -r milvus/tests/python_client/requirements.txt
+        python3 -m pip uninstall -y pymilvus
+        python3 setup.py install
+
+    - name: Smoke Test
+      working-directory: milvus/tests/python_client
+      run: |
+        pytest -n 2 --tags L0

ci/docker/milvus/docker-compose.yml

@@ -0,0 +1,43 @@
+version: '3.5'
+
+services:
+  etcd:
+    container_name: milvus-etcd
+    image: quay.io/coreos/etcd:v3.5.0
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
+    command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
+
+  minio:
+    container_name: milvus-minio
+    image: minio/minio:RELEASE.2020-12-03T00-03-10Z
+    environment:
+      MINIO_ACCESS_KEY: minioadmin
+      MINIO_SECRET_KEY: minioadmin
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data
+    command: minio server /minio_data
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  standalone:
+    container_name: milvus-standalone
+    image: ${IMAGE_REPO}/milvus-dev:${IMAGE_TAG}
+    command: ["milvus", "run", "standalone"]
+    environment:
+      ETCD_ENDPOINTS: etcd:2379
+      MINIO_ADDRESS: minio:9000
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus
+    ports:
+      - "19530:19530"
+    depends_on:
+      - "etcd"
+      - "minio"
+
+networks:
+  default:
+    name: milvus

ci/scripts/docker_image_find_tag.sh

@@ -0,0 +1,230 @@
+#!/usr/bin/env bash
+
+# Reference: https://github.com/ryandaniels/docker-script-find-latest-image-tag/blob/master/docker_image_find_tag.sh
+
+set -ef -o pipefail
+
+REGISTRY=${REGISTRY:-"https://index.docker.io/v2"}
+REGISTRY_AUTH=${REGISTRY_AUTH:-"https://auth.docker.io"}
+REGISTRY_SERVICE=${REGISTRY_SERVICE:-"registry.docker.io"}   
+# IMAGE_NAME=library/traefik
+IMAGE_NAME=${IMAGE_NAME:-""}
+IMAGE_TAG=${IMAGE_TAG-""}
+IMAGE_ID_SHORT=${IMAGE_ID_SHORT:-""}
+# IMAGE_ID_SHORT="96c63a7d3e50"
+IMAGE_ID_TARGET=""
+IMAGE_ID_LONG=${IMAGE_ID_LONG:-""}
+# IMAGE_ID_LONG="sha256:96c63a7d3e502fcbbd8937a2523368c22d0edd1788b8389af095f64038318834"
+DOCKER_BIN=docker
+# TAGS_FILTER="1.7"
+TAGS_FILTER=${TAGS_FILTER:-""}
+FAST_MATCH=0
+VERBOSE=0
+QUIET=0
+TAGS_LIMIT=100
+TAG_UNLIMIT=0
+ignore_404=0
+
+show_help () {
+  echo "Usage:"
+  echo "$0 [-n image name] [-i image-id]"
+  echo "Example: $0 -n traefik -i 96c63a7d3e50 -f 1.7"
+  echo "  -n [text]: Image name (Required). '-n traefik' would reference the traefik image"
+  echo "  -D: Use Docker binary for Image ID check (Default) (Optional)"
+  echo "  -P: Use Podman binary for Image ID check (Optional)"
+  echo "  -r [text]: Registry URL to use. Example: -r https://index.docker.io/v2 (Default) (Optional)"
+  echo "  -a [text]: Registry AUTH to use. Example: -a https://auth.docker.io (Default) (Optional)"  
+  echo "  -l [number]: Tag limit. Defaults to 100. (Optional)"  
+  echo "  -L: Tag unlimit. (Optional)"
+  echo "  -f [text]: Filter tag to contain this value (Optional)"
+  echo "  -F: Fast match tag. (Optional)"
+  echo "  -q: Quiet output (Optional)"
+  echo "  -v: Verbose output (Optional)"
+}
+
+# From: https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash
+# A POSIX variable
+OPTIND=1         # Reset in case getopts has been used previously in the shell.
+
+while getopts "h?n:t:DPr:a:l:Lf:Fqv" opt; do
+    case "$opt" in
+    h|\?)
+        show_help
+        exit 0
+        ;;
+    n)  IMAGE_NAME="$OPTARG"
+        ;;
+    t)  IMAGE_TAG="$OPTARG"
+        ;;
+    D)  DOCKER_BIN=docker
+        ;;
+    P)  DOCKER_BIN=podman
+        ;;
+    r)  REGISTRY="$OPTARG"
+        ;;
+    a)  REGISTRY_AUTH="$OPTARG"
+        ;;
+    l)  TAGS_LIMIT="$OPTARG"
+        ;;
+    L)  TAG_UNLIMIT=1
+        ;;
+    f)  TAGS_FILTER="$OPTARG"
+        ;;
+    F)  FAST_MATCH=1
+        ;;
+    q)  QUIET=1
+        ;;
+    v)  VERBOSE=1
+        ;;
+    esac
+done
+
+shift $((OPTIND-1))
+
+[ "${1:-}" = "--" ] && shift
+
+if [ -z "$IMAGE_NAME" ]; then
+  echo "Requires Image Name"
+  exit 1;
+else
+  if [[ "$VERBOSE" -eq 1 ]]; then
+    echo "Using IMAGE_NAME: $IMAGE_NAME"
+  fi
+  # add library/ if no /. (Which is _ aka official image like hub.docker.com/_/traefik)
+  # Official images are in "library/"
+  if [[ "$IMAGE_NAME" != *"/"* ]]; then
+      IMAGE_NAME="library/$IMAGE_NAME"
+  fi
+fi
+
+if [[ -z "$IMAGE_TAG" ]]; then
+  echo "Requires Image Tag"
+  exit 1;
+fi
+
+if [[ "$VERBOSE" -eq 1 ]]; then
+  echo "Using REGISTRY: $REGISTRY"
+fi
+
+if ! [[ $TAGS_LIMIT =~ ^[0-9]+$ ]] ; then
+  echo "Tag limit (-l) must be an integer > 0"
+  exit 1;
+fi
+
+function log() {
+  if [[ "${QUIET}" -ne 1 || "$VERBOSE" -eq 1  ]]; then
+    echo "${1}"
+  fi
+}
+
+# https://unix.stackexchange.com/questions/459367/using-shell-variables-for-command-options/459369#459369
+# https://unix.stackexchange.com/questions/444946/how-can-we-run-a-command-stored-in-a-variable/444949#444949
+# https://askubuntu.com/questions/674333/how-to-pass-an-array-as-function-argument/995110#995110
+# Maybe this? https://stackoverflow.com/questions/45948172/executing-a-curl-request-through-bash-script/45948289#45948289
+# http://mywiki.wooledge.org/BashFAQ/050#I_only_want_to_pass_options_if_the_runtime_data_needs_them
+function do_curl_get () {
+  local URL="$1"
+  shift
+  local array=("$@")
+  HTTP_RESPONSE="$(curl -sSL --write-out "HTTPSTATUS:%{http_code}" \
+    -H "Content-Type: application/json;charset=UTF-8" \
+    "${array[@]}" \
+    -X GET "$URL")"
+  HTTP_BODY=$(echo "$HTTP_RESPONSE" | sed -E 's/HTTPSTATUS\:[0-9]{3}$//')
+  HTTP_STATUS=$(echo "$HTTP_RESPONSE" | tr -d '\n' | sed -E 's/.*HTTPSTATUS:([0-9]{3})$/\1/')
+  # Check that the http status is 200
+  if [[ "$HTTP_STATUS" -ne 200 ]]; then
+    if [[ "$ignore_404" -eq 0 ]]; then
+      if [[ "$VERBOSE" -eq 0 ]]; then
+        echo -e "\\nError $HTTP_STATUS from: $URL\\n"
+      else
+        echo -e "\\nError $HTTP_STATUS from: $URL\\nHTTP_BODY: $HTTP_BODY\\n"
+      fi
+      exit 1
+    fi
+  fi
+}
+
+# Get AUTH token
+CURL_AUTH=()
+CURL_URL="$REGISTRY_AUTH/token?service=${REGISTRY_SERVICE##*(//)}&scope=repository:$IMAGE_NAME:pull"
+do_curl_get "$CURL_URL" "${CURL_AUTH[@]}"
+AUTH=$(echo "$HTTP_BODY" | jq --raw-output .token)
+
+# Get Source Image ID
+CURL_AUTH=( -H "Authorization: Bearer $AUTH" -H "Accept:application/vnd.docker.distribution.manifest.v2+json" )
+CURL_URL="$REGISTRY/$IMAGE_NAME/manifests/$IMAGE_TAG"
+do_curl_get "$CURL_URL" "${CURL_AUTH[@]}"
+IMAGE_ID_SOURCE="$(echo "$HTTP_BODY" |jq -r .config.digest)"
+
+# Get Tags
+CURL_AUTH=( -H "Authorization: Bearer $AUTH" )
+CURL_URL="$REGISTRY/$IMAGE_NAME/tags/list"
+do_curl_get "$CURL_URL" "${CURL_AUTH[@]}"
+TAGS_CURL=$(echo "$HTTP_BODY")
+TAGS_COUNT=$(echo "$TAGS_CURL"|jq -r '.tags[]'|grep -vi windows|wc -l)
+TAGS_temp=$(echo "$TAGS_CURL"|jq --arg TAGS_FILTER "$TAGS_FILTER" -r '.tags[]|select(.|contains($TAGS_FILTER))'|grep -vi windows|sort -r --version-sort)
+
+if [[ "$TAG_UNLIMIT" -eq 1 ]]; then
+  TAGS_LIMIT="$TAGS_COUNT"
+fi
+
+TAGS=$(echo "$TAGS_temp"|sed -n 1,"$TAGS_LIMIT"p)
+log "Found Total Tags: $TAGS_COUNT"
+# Check if tags are not being filtered
+if [ -z "$TAGS_FILTER" ]; then
+  log "Limiting Tags to: $TAGS_LIMIT"
+  # Check if limit reached and display warning
+  if [[ "$TAGS_COUNT" -gt "$TAGS_LIMIT" ]]; then
+    log "Limit reached, consider increasing limit (-l [number]) or adding a filter (-f [text])"
+  fi
+# If tags are filtered, show how many filtered tags were found
+else
+  TAGS_FILTER_COUNT=$(echo "$TAGS_temp"|wc -l)
+  log "Found Tags (after filtering): $TAGS_FILTER_COUNT"
+  log "Limiting Tags to: $TAGS_LIMIT"
+  # Check if limit reached and display warning
+  if [[ "$TAGS_FILTER_COUNT" -ge "$TAGS_LIMIT" ]]; then
+    log "Limit reached, consider increasing limit (-l [number]) or use more specific filter (-f [text])"
+  fi
+fi
+if [[ "$VERBOSE" -eq 1 ]]; then
+  # Output all tags found
+  echo -e "\nFound Tags:\n$TAGS"
+fi
+
+# Loop through tags and look for sha Id match
+# Some "manifests/tag" endpoints do not exist (http404 error)? Seems to be windows images. Ignore any 404 error
+ignore_404=1
+counter=0
+log "Checking for image match ..."
+
+if [[ "$FAST_MATCH" -eq 1 ]]; then
+    log "Fast match tag."
+fi
+for i in $TAGS; do
+  if [[ "$VERBOSE" -eq 1 ]]; then
+  # Output still working text every 50 tags if -v
+    if [[ "$counter" =~ ^($(echo {50..1000..50}|sed 's/ /|/g'))$ ]]; then
+      log "Still working, currently on tag number: $counter"
+    fi
+    counter=$((counter+1))
+  fi
+
+  if [[ "$IMAGE_TAG" != "$i" ]]; then
+    # IMAGE_ID_TARGET="$(curl -sSLH "Authorization: Bearer $AUTH" -H "Accept:application/vnd.docker.distribution.manifest.v2+json" -X GET "$REGISTRY/$IMAGE_NAME/manifests/$i"|jq -r .config.digest)"
+    CURL_AUTH=( -H "Authorization: Bearer $AUTH" -H "Accept:application/vnd.docker.distribution.manifest.v2+json" )
+    CURL_URL="$REGISTRY/$IMAGE_NAME/manifests/$i"
+    do_curl_get "$CURL_URL" "${CURL_AUTH[@]}"
+    IMAGE_ID_TARGET="$(echo "$HTTP_BODY" |jq -r .config.digest)"
+
+    if [[ "$IMAGE_ID_TARGET" == "$IMAGE_ID_SOURCE" ]]; then
+        log "Found match. tag:"
+        echo "$i"
+
+        if [[ "$FAST_MATCH" -eq 1 ]]; then
+          exit 0
+        fi
+    fi
+  fi
+done;