updates

Signed-off-by: aavarghese <[email protected]>

.github/workflows/tests_ibmcloud.yml

@@ -0,0 +1,37 @@
+name: CI Tests for IBM Cloud backend
+
+# cancel any prior runs for this workflow and this PR (or branch)
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  ci:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        SCRIPT:
+          - ./tests/bin/ci.sh -i 'test7f.*'
+          - ./tests/bin/go.sh
+        os: [ubuntu-latest]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Check Docker
+      run: docker version && podman version
+
+    - name: Run Test with args ${{ matrix.ARGS }}
+      env:
+        TERM: xterm-256color
+        IC_API_KEY: ${{ secrets.IC_API_KEY }}
+        RESOURCE_GROUP_ID: ${{ secrets.RESOURCE_GROUP_ID }}
+        SSH_KEY_PUB: ${{ secrets.SSH_KEY_PUB }}
+      run: bash -c "${{matrix.SCRIPT}} ${{matrix.ARGS }}"

pkg/be/ibmcloud/api.go

@@ -7,5 +7,5 @@ type Backend struct {
 	config       ibmConfig
 	vpcService   *vpcv1.VpcV1
 	sshKeyType   string
-	sshPublicKey string
+	sshPublicKey []string
 }

pkg/be/ibmcloud/authenticate.go

@@ -32,7 +32,7 @@ func Authenticator(apiKey string, config ibmConfig) (*vpcv1.VpcV1, error) {
 		Authenticator: auth,
 	})
 	if err != nil {
-		return nil, errors.New("Error creating VPC Service with apikey" + apiKey)
+		return nil, errors.New("Error creating VPC Service with apikey " + apiKey)
 	}
 	fmt.Printf("Accessing the VPC service via %s\n", method)
 

pkg/be/ibmcloud/create.go

@@ -277,7 +277,7 @@ func createResources(ctx context.Context, vpcService *vpcv1.VpcV1, name string,
 	t1e := time.Now()
 
 	t2s := t1e
-	keyID, err := createSSHKey(vpcService, name, resourceGroupID, keyType, publicKey)
+	keyID, err := createSSHKey(vpcService, name, resourceGroupID, keyType, strings.Join(publicKey, " "))
 	if err != nil {
 		return "", err
 	}

pkg/be/ibmcloud/discovery.go

@@ -11,6 +11,7 @@ import (
 	"github.com/IBM/vpc-go-sdk/vpcv1"
 	"golang.org/x/crypto/ssh"
 	"lunchpail.io/pkg/build"
+	"lunchpail.io/pkg/compilation"
 )
 
 type resourceGroup struct {
@@ -75,26 +76,26 @@ func getRandomizedZone(config ibmConfig, vpcService *vpcv1.VpcV1) (string, error
 
 // Retrieve public key from user's ssh dir, if exists
 // Looks for two ssh key types: “rsa” and “ed25519" (ibmcloud supported)
-func loadPublicKey(config ibmConfig, aopts build.Options) (string, string, error) {
+func loadPublicKey(config ibmConfig, aopts compilation.Options) (string, []string, error) {
 	homedir, err := os.UserHomeDir()
 	if err != nil {
-		return "", "", err
+		return "", []string{}, err
 	}
 
 	var bytes []byte
-	if aopts.PublicSSHKey != "" {
+	if len(aopts.PublicSSHKey) != 0 {
 		return aopts.SSHKeyType, aopts.PublicSSHKey, nil
 	} else if bytes, err = os.ReadFile(filepath.Join(homedir, ".ssh", "id_rsa.pub")); err == nil && bytes != nil {
 		pKeyComps := strings.Split(string(bytes), " ")
 		if len(pKeyComps) >= 2 && strings.Trim(pKeyComps[0], " ") == ssh.KeyAlgoRSA {
-			return "rsa", string(bytes), nil
+			return "rsa", []string{string(bytes)}, nil
 		}
 	} else if bytes, err = os.ReadFile(filepath.Join(homedir, ".ssh", "id_ed25519.pub")); err == nil && bytes != nil {
 		pKeyComps := strings.Split(string(bytes), " ")
 		if len(pKeyComps) >= 2 && strings.Trim(pKeyComps[0], " ") == ssh.KeyAlgoED25519 {
-			return "ed25519", string(bytes), nil
+			return "ed25519", []string{string(bytes)}, nil
 		}
 	}
 
-	return "", "", nil
+	return "", []string{}, nil
 }

pkg/compilation/options.go

@@ -0,0 +1,103 @@
+package compilation
+
+import (
+	_ "embed"
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	"lunchpail.io/pkg/be/target"
+)
+
+type TargetOptions struct {
+	Namespace string
+	target.Platform
+}
+
+type Options struct {
+	Target *TargetOptions
+
+	ImagePullSecret    string   `yaml:"imagePullSecret,omitempty"`
+	OverrideValues     []string `yaml:"overrideValues,omitempty"`
+	OverrideFileValues []string `yaml:"overrideFileValues,omitempty"`
+	Queue              string   `yaml:",omitempty"`
+	HasGpuSupport      bool     `yaml:"hasGpuSupport,omitempty"`
+	ApiKey             string   `yaml:"apiKey,omitempty"`
+	ResourceGroupID    string   `yaml:"resourceGroupID,omitempty"`
+	SSHKeyType         string   `yaml:"SSHKeyType,omitempty"`
+	PublicSSHKey       []string `yaml:"publicSSHKey,omitempty"`
+	Zone               string   `yaml:"zone,omitempty"`
+	Profile            string   `yaml:"profile,omitempty"`
+	ImageID            string   `yaml:"imageID,omitempty"`
+	CreateNamespace    bool     `yaml:"createNamespace,omitempty"`
+}
+
+//go:embed compilationOptions.json
+var valuesJson []byte
+
+func saveOptions(stagedir string, opts Options) error {
+	if serialized, err := json.Marshal(opts); err != nil {
+		return err
+	} else {
+		return os.WriteFile(filepath.Join(stagedir, "pkg/compilation/compilationOptions.json"), serialized, 0644)
+	}
+}
+
+func RestoreOptions() (Options, error) {
+	var compilationOptions Options
+
+	if err := json.Unmarshal(valuesJson, &compilationOptions); err != nil {
+		return compilationOptions, err
+	}
+
+	return compilationOptions, nil
+}
+
+// Overlay command line args with options from shrinkwrap (i.e. RestoreOptions)
+func RestoreOptionsWithCliOverlay(cliOpts Options) (Options, error) {
+	compiledOpts, err := RestoreOptions()
+	if err != nil {
+		return cliOpts, err
+	} else {
+		return cliOpts.overlay(compiledOpts), nil
+	}
+}
+
+func either(a string, b string) string {
+	if b == "" {
+		return a
+	}
+	return b
+}
+
+func eitherPlatform(a target.Platform, b target.Platform) target.Platform {
+	if b == "" {
+		return a
+	}
+	return b
+}
+
+func eitherB(a bool, b bool) bool {
+	return b || a
+}
+
+func (cliOpts Options) overlay(compiledOpts Options) Options {
+	cliOpts.Queue = either(compiledOpts.Queue, cliOpts.Queue)
+	cliOpts.ImagePullSecret = either(compiledOpts.ImagePullSecret, cliOpts.ImagePullSecret)
+	cliOpts.Target = &TargetOptions{
+		Platform:  eitherPlatform(compiledOpts.Target.Platform, cliOpts.Target.Platform),
+		Namespace: either(compiledOpts.Target.Namespace, cliOpts.Target.Namespace),
+	}
+
+	// TODO here... how do we determine that boolean values were unset?
+	cliOpts.HasGpuSupport = eitherB(compiledOpts.HasGpuSupport, cliOpts.HasGpuSupport)
+	cliOpts.CreateNamespace = eitherB(compiledOpts.CreateNamespace, cliOpts.CreateNamespace)
+
+	// careful: `--set x=3 --set x=4` results in x having
+	// value 4, so we need to place the compiled
+	// options first in the list
+	cliOpts.OverrideValues = append(compiledOpts.OverrideValues, cliOpts.OverrideValues...)
+	cliOpts.OverrideFileValues = append(compiledOpts.OverrideFileValues, cliOpts.OverrideFileValues...)
+
+	return cliOpts
+}

tests/bin/ci.sh

@@ -13,6 +13,11 @@ set -eo pipefail
 # e.g. see test7/init.sh
 export RUNNING_LUNCHPAIL_TESTS=1
 
+if [[ -n "$IC_API_KEY" ]]
+then
+    export TEST_IBMCLOUD=1
+fi
+
 SCRIPTDIR=$(cd $(dirname "$0") && pwd)
 TOP="$SCRIPTDIR"/../..
 

tests/bin/undeploy-tests.sh

@@ -11,6 +11,11 @@ set -o pipefail
 SCRIPTDIR=$(cd $(dirname "$0") && pwd)
 TOP="$SCRIPTDIR"/../../
 
+if [[ -n "$TEST_IBMCLOUD" ]]
+then 
+    IC_TARGET="--target=ibmcloud --api-key=$IC_API_KEY" #TODO: needs runname 
+fi
+
 appname="${2-$1}"
 
 # retry once after failure; this may help to cope with `etcdserver:

tests/bin/up.sh

@@ -12,6 +12,13 @@ if [[ -n "$taskqueue" ]]
 then QUEUE="--queue $taskqueue"
 fi
 
+if [[ -n "$TEST_IBMCLOUD" ]]
+then 
+    IC_TARGET="--target ibmcloud --api-key $IC_API_KEY"
+    IC_UP_ARGS="--resource-group-id $RESOURCE_GROUP_ID --zone us-south-1 --image-id r006-1169e41d-d654-45d5-bdd5-89e2dc6e8a68 --profile bx2-8x32"
+    #--public-ssh-key=${SSH_KEY_PUB} \  #TODO: include it in IC_UP_ARGS string
+fi
+
 echo "Calling up using target=${LUNCHPAIL_TARGET:-kubernetes}"
 if [[ -n "$inputapp" ]]
 then
@@ -29,6 +36,8 @@ else
     eval $testapp up \
          --verbose=${VERBOSE:-false} \
          $up_args \
+         $IC_TARGET \
+         $IC_UP_ARGS \
          $QUEUE \
          --create-cluster \
          --target=${LUNCHPAIL_TARGET:-kubernetes} \