[Dev] Remove version linter flakiness

buildkite/scripts/debian/install.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 
-
 if [[ $# -gt 2 ]] || [[ $# -lt 1 ]]; then
     echo "Usage: $0 '<debians>' '[use-sudo]'"
     exit 1

buildkite/scripts/dump-mina-type-shapes.sh

@@ -20,3 +20,5 @@ export TYPE_SHAPE_FILE=${MINA_COMMIT_SHA1}-type_shape.txt
 
 echo "--- Create type shapes git note for commit: ${MINA_COMMIT_SHA1}"
 mina internal dump-type-shapes > ${TYPE_SHAPE_FILE}
+
+source buildkite/scripts/gsutil-upload.sh ${TYPE_SHAPE_FILE} gs://mina-type-shapes

buildkite/scripts/gsutil-upload.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+KEY_FILE=/var/secrets/google/key.json
+
+if [ ! -f $KEY_FILE ]; then
+    echo "Cannot use gsutil for upload as key file cannot be found in $KEY_FILE"
+fi
+
+gcloud auth activate-service-account --key-file=$KEY_FILE
+
+gsutil cp $1 $2

buildkite/scripts/version-linter-patch-missing-type-shapes.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -eox pipefail
+
+if [[ $# -ne 1 ]]; then
+    echo "Usage: $0 <release-branch>"
+    exit 1
+fi
+
+git config --global --add safe.directory /workdir
+
+source buildkite/scripts/handle-fork.sh
+source buildkite/scripts/export-git-env-vars.sh
+
+release_branch=${REMOTE}/$1
+
+RELEASE_BRANCH_COMMIT=$(git log -n 1 --format="%h" --abbrev=7 $release_branch)
+
+function revert_checkout() {
+    git checkout $BUILDKITE_COMMIT
+    git submodule sync
+    git submodule update --init --recursive
+}
+
+function checkout_and_dump() {
+    local __commit=$1
+    git checkout $__commit
+    git submodule sync
+    git submodule update --init --recursive
+    eval $(opam config env)
+    TYPE_SHAPE_FILE=${__commit:0:7}-type_shape.txt
+    dune exec src/app/cli/src/mina.exe internal dump-type-shapes > /tmp/${TYPE_SHAPE_FILE}
+    revert_checkout
+    source buildkite/scripts/gsutil-upload.sh /tmp/${TYPE_SHAPE_FILE} gs://mina-type-shapes
+}
+
+if ! $(gsutil ls gs://mina-type-shapes/$RELEASE_BRANCH_COMMIT 2>/dev/null); then
+    checkout_and_dump $RELEASE_BRANCH_COMMIT
+fi
+
+if [[ -n "${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-}" ]]; then 
+    BUILDKITE_PULL_REQUEST_BASE_BRANCH_COMMIT=$(git log -n 1 --format="%h" --abbrev=7 ${REMOTE}/${BUILDKITE_PULL_REQUEST_BASE_BRANCH} )
+    if ! $(gsutil ls gs://mina-type-shapes/$BUILDKITE_PULL_REQUEST_BASE_BRANCH_COMMIT 2>/dev/null); then
+        checkout_and_dump $BUILDKITE_PULL_REQUEST_BASE_BRANCH_COMMIT
+    fi
+fi

buildkite/scripts/version-linter.sh

@@ -1,14 +1,12 @@
 #!/bin/bash
 
-set -eo pipefail
+set -eox pipefail
 
 if [[ $# -ne 1 ]]; then
     echo "Usage: $0 <release-branch>"
     exit 1
 fi
 
-TESTNET_NAME="${TESTNET_NAME:-devnet}"
-
 # Don't prompt for answers during apt-get install
 export DEBIAN_FRONTEND=noninteractive
 
@@ -23,17 +21,22 @@ source buildkite/scripts/export-git-env-vars.sh
 
 pip3 install sexpdata==1.0.0
 
-base_branch=${REMOTE}/${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-compatible}
 pr_branch=origin/${BUILDKITE_BRANCH}
 release_branch=origin/$1
 
+if [[ -n "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" ]]; then
+    base_branch=${REMOTE}/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+else 
+    # If we ran pipeline from buildkite directly some envs are not set like BUILDKITE_PULL_REQUEST_BASE_BRANCH
+    # In this case we are using release branch
+    base_branch=${release_branch}
+fi
+
 echo "--- Run Python version linter with branches: ${pr_branch} ${base_branch} ${release_branch}"
 ./scripts/version-linter.py ${pr_branch} ${base_branch} ${release_branch}
 
 echo "--- Install Mina"
-source buildkite/scripts/export-git-env-vars.sh
-
-source buildkite/scripts/debian/install.sh "mina-${TESTNET_NAME}" 1
+source buildkite/scripts/debian/install.sh "mina-devnet" 1
 
 echo "--- Audit type shapes"
 mina internal audit-type-shapes

buildkite/src/Jobs/Test/VersionLint.dhall

@@ -1,5 +1,3 @@
-let Cmd = ../../Lib/Cmds.dhall
-
 let S = ../../Lib/SelectFiles.dhall
 
 let B = ../../External/Buildkite.dhall
@@ -16,21 +14,24 @@ let Command = ../../Command/Base.dhall
 
 let RunInToolchain = ../../Command/RunInToolchain.dhall
 
-let Docker = ../../Command/Docker/Type.dhall
-
-let Size = ../../Command/Size.dhall
-
-let DebianVersions = ../../Constants/DebianVersions.dhall
+let Dockers = ../../Constants/DockerVersions.dhall
 
 let Network = ../../Constants/Network.dhall
 
 let Profiles = ../../Constants/Profiles.dhall
 
+let Artifacts = ../../Constants/Artifacts.dhall
+
+let Docker = ../../Command/Docker/Type.dhall
+
+let Size = ../../Command/Size.dhall
+
 let dependsOn =
-      DebianVersions.dependsOn
-        DebianVersions.DebVersion.Bullseye
+      Dockers.dependsOn
+        Dockers.Type.Bullseye
         Network.Type.Devnet
         Profiles.Type.Standard
+        Artifacts.Type.Daemon
 
 let buildTestCmd
     : Text -> Size -> List Command.TaggedKey.Type -> B/SoftFail -> Command.Type
@@ -44,9 +45,9 @@ let buildTestCmd
                   RunInToolchain.runInToolchain
                     ([] : List Text)
                     "buildkite/scripts/dump-mina-type-shapes.sh"
-                # [ Cmd.run
-                      "gsutil cp \$(git log -n 1 --format=%h --abbrev=7)-type_shape.txt \$MINA_TYPE_SHAPE gs://mina-type-shapes"
-                  ]
+                # RunInToolchain.runInToolchain
+                    ([] : List Text)
+                    "buildkite/scripts/version-linter-patch-missing-type-shapes.sh ${release_branch}"
                 # RunInToolchain.runInToolchain
                     ([] : List Text)
                     "buildkite/scripts/version-linter.sh ${release_branch}"

buildkite/src/Lib/Cmds.dhall

@@ -71,11 +71,11 @@ let module =
                         = if docker.useBash then "/bin/bash" else "/bin/sh"
 
                     in  { line =
-                            "docker run -it --rm --entrypoint ${entrypoint} --init --volume ${sharedDir}:/shared --volume ${outerDir}:/workdir --workdir /workdir${envVars}${      if docker.privileged
+                            "docker run -it --rm --entrypoint ${entrypoint} --init --volume /var/secrets:/var/secrets --volume ${sharedDir}:/shared --volume ${outerDir}:/workdir --workdir /workdir${envVars}${      if docker.privileged
 
-                                                                                                                                                                             then  " --privileged"
+                                                                                                                                                                                                                then  " --privileged"
 
-                                                                                                                                                                             else  ""} ${docker.image} -c '${inner.line}'"
+                                                                                                                                                                                                                else  ""} ${docker.image} -c '${inner.line}'"
                         , readable =
                             Optional/map
                               Text
@@ -142,7 +142,7 @@ let tests =
       let dockerExample =
               assert
             :     { line =
-                      "docker run -it --rm --entrypoint /bin/bash --init --volume /var/buildkite/shared:/shared --volume \\\$BUILDKITE_BUILD_CHECKOUT_PATH:/workdir --workdir /workdir --env ENV1 --env ENV2 --env TEST foo/bar:tag -c 'echo hello'"
+                      "docker run -it --rm --entrypoint /bin/bash --init --volume /var/secrets:/var/secrets --volume /var/buildkite/shared:/shared --volume \\\$BUILDKITE_BUILD_CHECKOUT_PATH:/workdir --workdir /workdir --env ENV1 --env ENV2 --env TEST foo/bar:tag -c 'echo hello'"
                   , readable = Some "Docker@foo/bar:tag ( echo hello )"
                   }
               ===  M.inDocker
@@ -154,7 +154,7 @@ let tests =
 
       let cacheExample =
               assert
-            :     "./buildkite/scripts/cache-through.sh data.tar \"docker run -it --rm --entrypoint /bin/bash --init --volume /var/buildkite/shared:/shared --volume \\\$BUILDKITE_BUILD_CHECKOUT_PATH:/workdir --workdir /workdir --env ENV1 --env ENV2 --env TEST foo/bar:tag -c 'echo hello > /tmp/data/foo.txt && tar cvf data.tar /tmp/data'\""
+            :     "./buildkite/scripts/cache-through.sh data.tar \"docker run -it --rm --entrypoint /bin/bash --init --volume /var/secrets:/var/secrets --volume /var/buildkite/shared:/shared --volume \\\$BUILDKITE_BUILD_CHECKOUT_PATH:/workdir --workdir /workdir --env ENV1 --env ENV2 --env TEST foo/bar:tag -c 'echo hello > /tmp/data/foo.txt && tar cvf data.tar /tmp/data'\""
               ===  M.format
                      ( M.cacheThrough
                          M.Docker::{

scripts/version-linter.py

@@ -19,8 +19,6 @@
 """
 
 import subprocess
-import os
-import io
 import sys
 import re
 import sexpdata
@@ -32,31 +30,38 @@ def set_error():
   global exit_code
   exit_code=1
 
-def branch_commit(branch):
+def latest_branch_commit(branch):
+  '''
+    Retrieves latest commit on branch
+  '''
   print ('Retrieving', branch, 'head commit...')
   result=subprocess.run(['git','log','-n','1','--format="%h"','--abbrev=7',f'{branch}'],
                         capture_output=True)
   output=result.stdout.decode('ascii')
   print ('command stdout:', output)
   print ('command stderr:', result.stderr.decode('ascii'))
-  return output.replace('"','').replace('\n','')
-
-def download_type_shapes(role,branch,sha1) :
+  return output.replace('"','').strip()
+
+def url_to_type_shape_file(file):
+  '''
+    Return url to mina type shape file
+  '''
+  return f'https://storage.googleapis.com/mina-type-shapes/{file}'
+
+def url_exists(url):
+  '''
+    Checks if url exists (by sending head and validating that status code is ok)
+  '''
+  return requests.head(url).status_code == 200
+
+def download_type_shape(role,branch,sha1) :
   file=type_shape_file(sha1)
-  print ('Downloading type shape file',file,'for',role,'branch',branch,'at commit',sha1)
-  url = f'https://storage.googleapis.com/mina-type-shapes/{file}'
-  r = requests.head(url, allow_redirects=True)
-  if r.status_code != 200:
-    print ("cannot fetch file reference from non-existing path: ${url}")
-    print ("looks like you need to generate it. Please use below steps")
-    print (f"git checkout ${sha1}")
-    print ("nix develop mina")
-    print (f"dune exec src/app/cli/src/mina.exe internal dump-type-shape > ${sha1}-type_shape.txt")
-    print ("gsutil cp gs://mina-type-shapes ${sha1}-type_shape.txt ")
+  url=url_to_type_shape_file(file)
+  if not url_exists(url):
+    raise Exception(f"reference file for '{sha1}' commit does not exists. Url does not exists {url} ")
 
-    sys.exit(1)
-
-  result=subprocess.run(['wget','--no-clobber',url])
+  print ('Downloading type shape file',file,'for',role,'branch',branch,'at commit',sha1)
+  subprocess.run(['wget','--no-clobber',url], check=True)
 
 def type_shape_file(sha1) :
   # created by buildkite build-artifact script
@@ -250,18 +255,18 @@ def assert_commit(commit, desc):
 
   subprocess.run(['git','fetch'],capture_output=False)
 
-  base_branch_commit=branch_commit(base_branch)
-  download_type_shapes('base',base_branch,base_branch_commit)
+  base_branch_commit = latest_branch_commit(base_branch)
+  download_type_shape('base',base_branch,base_branch_commit)
 
   print('')
 
-  release_branch_commit=branch_commit(release_branch)
-  download_type_shapes('release',release_branch,release_branch_commit)
+  release_branch_commit=latest_branch_commit(release_branch)
+  download_type_shape('release',release_branch,release_branch_commit)
 
   print('')
 
-  pr_branch_commit=branch_commit(pr_branch)
-  download_type_shapes('pr',pr_branch,pr_branch_commit)
+  pr_branch_commit=latest_branch_commit(pr_branch)
+  download_type_shape('pr',pr_branch,pr_branch_commit)
 
   print('')
 

src/lib/runtime_config/runtime_config.ml

@@ -995,6 +995,8 @@ module Proof_keys = struct
           Error
             "Runtime_config.Proof_keys.Level.of_json_layout: Expected the \
              field 'level' to contain a string"
+
+    let gen = Quickcheck.Generator.of_list [ Full; Check; No_check ]
   end
 
   module Transaction_capacity = struct