Skip to content
Permalink

Comparing changes

This is a direct comparison between two commits made in this repository or its related repositories. View the default comparison for this range or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: IBM/lunchpail
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: f4a4ceb011568e8456c004ed89434835e853cf3d
Choose a base ref
..
head repository: IBM/lunchpail
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 0d4aa5f769eda1b4d87a2b378bcc82b047621a1e
Choose a head ref
Showing 483 changed files with 3,130 additions and 3,159 deletions.
7 changes: 3 additions & 4 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
@@ -33,9 +33,7 @@ jobs:
- ./tests/bin/ci.sh -i 'test7(b.*|c.*|d.*|e.*|g.*|h.*)'
- ./tests/bin/ci.sh -i 'test7f.*'
- ./tests/bin/ci.sh -i 'test8.*'
- ./tests/bin/ci.sh -i 'python-code.*'
- ./tests/bin/ci.sh -i 'python-language.*'
- ./tests/bin/ci.sh -i 'python-universal.*'
- /tmp/lunchpail bat demos/data-prep-kit --concurrency 1 --auto-clean --target=$LUNCHPAIL_TARGET # bat=Build and Test
- ./tests/bin/go.sh
- ./tests/bin/pipelines.sh
os: [ubuntu-latest]
@@ -44,7 +42,7 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
- name: Cache minio
uses: actions/cache@v3
uses: actions/cache@v4
id: cache-minio
with:
path: /home/runner/.cache/lunchpail/bin
@@ -62,6 +60,7 @@ jobs:
- name: Run Test with args ${{ matrix.ARGS }}
env:
TERM: xterm-256color
VERBOSE: false
LUNCHPAIL_TARGET: ${{ matrix.LUNCHPAIL_TARGET }}
LUNCHPAIL_BUILD_NOT_NEEDED: true # we did this in the "Build Lunchpail CLI" step above
run: bash -c "${{matrix.SCRIPT}} ${{matrix.ARGS }}"
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -12,5 +12,5 @@ __pycache__
.cache

# up copyout test leftovers
task.*.output.txt
task.*.txt
tmp.output.*
10 changes: 9 additions & 1 deletion cmd/main.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,15 @@
package main

import "lunchpail.io/cmd/subcommands"
import (
"go.uber.org/automaxprocs/maxprocs"

"lunchpail.io/cmd/subcommands"
)

func main() {
// Automatically set GOMAXPROCS to match Linux container CPU quota.
// https://github.com/uber-go/automaxprocs
maxprocs.Set()

subcommands.Execute()
}
6 changes: 6 additions & 0 deletions cmd/options/build.go
Original file line number Diff line number Diff line change
@@ -35,6 +35,12 @@ func AddBuildOptions(cmd *cobra.Command) (*build.Options, error) {
cmd.Flags().BoolVarP(&options.CreateNamespace, "create-namespace", "N", options.CreateNamespace, "Create a new namespace, if needed")
cmd.Flags().IntVarP(&options.Workers, "workers", "W", options.Workers, "Number of workers in the initial worker pool")

cmd.Flags().StringToStringVarP(&options.Env, "env", "e", options.Env, "Set environment variables")

cmd.Flags().IntVar(&options.Pack, "pack", options.Pack, "Run k concurrent tasks; if k=0 and machine has N cores, then k=N")
cmd.Flags().BoolVarP(&options.Gunzip, "gunzip", "z", options.Gunzip, "Gunzip inputs before passing them to the worker logic")
cmd.Flags().BoolVar(&options.AutoClean, "auto-clean", options.AutoClean, "Clean up any caches prior to exiting")

AddTargetOptionsTo(cmd, &options)
AddLogOptionsTo(cmd, &options)
return &options, nil
43 changes: 43 additions & 0 deletions cmd/subcommands/bat.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
//go:build full || manage

package subcommands

import (
"context"

"github.com/spf13/cobra"

"lunchpail.io/cmd/options"
"lunchpail.io/pkg/be"
"lunchpail.io/pkg/boot"
)

func init() {
var cmd = &cobra.Command{
Use: "bat",
Short: "Build and test",
Long: "Build and test",
Args: cobra.MatchAll(cobra.MinimumNArgs(1), cobra.OnlyValidArgs),
}

buildOpts, err := options.AddBuildOptions(cmd)
if err != nil {
panic(err)
}

concurrency := 4
cmd.Flags().IntVarP(&concurrency, "concurrency", "j", concurrency, "Maximum tests to run concurrently")

cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := context.Background()

backend, err := be.NewInitOk(ctx, true, *buildOpts)
if err != nil {
return err
}

return boot.BuildAndTester{Backend: backend, Concurrency: concurrency, Options: *buildOpts}.RunAll(ctx, args)
}

rootCmd.AddCommand(cmd)
}
16 changes: 11 additions & 5 deletions cmd/subcommands/build.go
Original file line number Diff line number Diff line change
@@ -10,11 +10,13 @@ import (

"lunchpail.io/cmd/options"
"lunchpail.io/pkg/fe/builder"
"lunchpail.io/pkg/fe/builder/overlay"
)

func newBuildCmd() *cobra.Command {
var outputFlag string
var branchFlag string
var sourceIsYaml bool
var allFlag bool

cmd := &cobra.Command{
@@ -31,10 +33,11 @@ func newBuildCmd() *cobra.Command {
}

cmd.Flags().StringVarP(&branchFlag, "branch", "b", branchFlag, "Git branch to pull from")
cmd.Flags().BoolVarP(&sourceIsYaml, "yaml", "y", sourceIsYaml, "The source directory contains high-level IR YAML")
cmd.Flags().BoolVarP(&allFlag, "all-platforms", "A", allFlag, "Generate binaries for all supported platform/arch combinations")

var eval string
cmd.Flags().StringVarP(&eval, "eval", "e", eval, "Run the given command line")
var command string
cmd.Flags().StringVarP(&command, "command", "c", command, "Run the given program given as a string")

buildOptions, err := options.AddBuildOptions(cmd)
if err != nil {
@@ -63,10 +66,13 @@ func newBuildCmd() *cobra.Command {

return builder.Build(context.Background(), sourcePath, builder.Options{
Name: outputFlag,
Branch: branchFlag,
Eval: eval,
AllPlatforms: allFlag,
BuildOptions: *buildOptions,
OverlayOptions: overlay.Options{
Branch: branchFlag,
Command: command,
SourceIsYaml: sourceIsYaml,
BuildOptions: *buildOptions,
},
})
}

1 change: 1 addition & 0 deletions cmd/subcommands/component.go
Original file line number Diff line number Diff line change
@@ -17,4 +17,5 @@ func init() {
cmd.AddCommand(component.Minio())
cmd.AddCommand(component.Worker())
cmd.AddCommand(component.WorkStealer())
cmd.AddCommand(component.RunLocally())
}
42 changes: 42 additions & 0 deletions cmd/subcommands/component/run-locally.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package component

import (
"context"

"github.com/spf13/cobra"
"lunchpail.io/cmd/options"
"lunchpail.io/pkg/build"
"lunchpail.io/pkg/runtime"
)

type RunLocallyOptions struct {
Component string
LLIR string
build.LogOptions
}

func AddRunLocallyOptions(cmd *cobra.Command) *RunLocallyOptions {
options := RunLocallyOptions{}
cmd.Flags().StringVarP(&options.Component, "component", "", "", "")
cmd.Flags().StringVar(&options.LLIR, "llir", "", "")
cmd.MarkFlagRequired("component")
cmd.MarkFlagRequired("llir")
return &options
}

func RunLocally() *cobra.Command {
cmd := &cobra.Command{
Use: "run-locally",
Short: "Commands for running a component locally",
Long: "Commands for running a component locally",
}

runOpts := AddRunLocallyOptions(cmd)
options.AddLogOptions(cmd)

cmd.RunE = func(cmd *cobra.Command, args []string) error {
return runtime.RunLocally(context.Background(), runOpts.Component, runOpts.LLIR, runOpts.LogOptions)
}

return cmd
}
6 changes: 5 additions & 1 deletion cmd/subcommands/component/worker/prestop.go
Original file line number Diff line number Diff line change
@@ -17,6 +17,10 @@ func PreStop() *cobra.Command {
Long: "Mark this worker as dead",
}

var step int
cmd.Flags().IntVar(&step, "step", step, "Which step are we part of")
cmd.MarkFlagRequired("step")

var poolName string
cmd.Flags().StringVar(&poolName, "pool", "", "Which worker pool are we part of")
cmd.MarkFlagRequired("pool")
@@ -35,7 +39,7 @@ func PreStop() *cobra.Command {

return worker.PreStop(context.Background(), worker.Options{
LogOptions: *logOpts,
RunContext: run.ForPool(poolName).ForWorker(workerName),
RunContext: run.ForStep(step).ForPool(poolName).ForWorker(workerName),
})
}

14 changes: 13 additions & 1 deletion cmd/subcommands/component/worker/run.go
Original file line number Diff line number Diff line change
@@ -19,6 +19,13 @@ func Run() *cobra.Command {
Args: cobra.MatchAll(cobra.OnlyValidArgs),
}

pack := 0
cmd.Flags().IntVar(&pack, "pack", pack, "Run k concurrent tasks; if k=0 and machine has N cores, then k=N")

var step int
cmd.Flags().IntVar(&step, "step", step, "Which step are we part of")
cmd.MarkFlagRequired("step")

var poolName string
cmd.Flags().StringVar(&poolName, "pool", "", "Which worker pool are we part of")
cmd.MarkFlagRequired("pool")
@@ -33,6 +40,9 @@ func Run() *cobra.Command {
var startupDelay int
cmd.Flags().IntVar(&startupDelay, "delay", 0, "Delay (in seconds) before engaging in any work")

var gunzip bool
cmd.Flags().BoolVarP(&gunzip, "gunzip", "z", gunzip, "Gunzip inputs before passing them to the worker logic")

ccOpts := options.AddCallingConventionOptions(cmd)
logOpts := options.AddLogOptions(cmd)

@@ -47,11 +57,13 @@ func Run() *cobra.Command {
}

return worker.Run(context.Background(), args, worker.Options{
Pack: pack,
Gunzip: gunzip,
CallingConvention: ccOpts.CallingConvention,
StartupDelay: startupDelay,
PollingInterval: pollingInterval,
LogOptions: *logOpts,
RunContext: run.ForPool(poolName).ForWorker(workerName),
RunContext: run.ForStep(step).ForPool(poolName).ForWorker(workerName),
})
}

13 changes: 6 additions & 7 deletions cmd/subcommands/down.go
Original file line number Diff line number Diff line change
@@ -16,7 +16,6 @@ import (
func newDownCmd() *cobra.Command {
var deleteNamespaceFlag bool
var deleteAllRunsFlag bool
var apiKey string
var deleteCloudResourcesFlag bool

var cmd = &cobra.Command{
@@ -26,16 +25,16 @@ func newDownCmd() *cobra.Command {
Long: "Undeploy a run",
}

cmd.Flags().BoolVarP(&deleteNamespaceFlag, "delete-namespace", "N", false, "Also delete namespace (only for empty namespaces)")
cmd.Flags().BoolVarP(&deleteAllRunsFlag, "all", "A", false, "Delete all runs in the given namespace")
cmd.Flags().StringVarP(&apiKey, "api-key", "a", "", "IBM Cloud api key")
cmd.Flags().BoolVarP(&deleteCloudResourcesFlag, "delete-cloud-resources", "D", false, "Delete all associated cloud resources and the virtual instance. If not enabled, the instance will only be stopped")

opts, err := options.RestoreBuildOptions()
if err != nil {
panic(err)
}

cmd.Flags().BoolVarP(&deleteNamespaceFlag, "delete-namespace", "N", false, "Also delete namespace (only for empty namespaces)")
cmd.Flags().BoolVarP(&deleteAllRunsFlag, "all", "A", false, "Delete all runs in the given namespace")
cmd.Flags().StringVarP(&opts.ApiKey, "api-key", "a", "", "IBM Cloud api key")
cmd.Flags().BoolVarP(&deleteCloudResourcesFlag, "delete-cloud-resources", "D", false, "Delete all associated cloud resources and the virtual instance. If not enabled, the instance will only be stopped")

options.AddTargetOptionsTo(cmd, &opts)
options.AddLogOptionsTo(cmd, &opts)

@@ -49,7 +48,7 @@ func newDownCmd() *cobra.Command {
return boot.DownList(ctx, args, backend, boot.DownOptions{
Namespace: opts.Target.Namespace, Verbose: opts.Log.Verbose, DeleteNamespace: deleteNamespaceFlag,
DeleteAll: deleteAllRunsFlag,
ApiKey: apiKey, DeleteCloudResources: deleteCloudResourcesFlag})
ApiKey: opts.ApiKey, DeleteCloudResources: deleteCloudResourcesFlag})
}

return cmd
9 changes: 8 additions & 1 deletion cmd/subcommands/queue/add/file.go
Original file line number Diff line number Diff line change
@@ -28,6 +28,9 @@ func File() *cobra.Command {
runOpts := options.AddRunOptions(cmd)
logOpts := options.AddLogOptions(cmd)

var next bool
cmd.Flags().BoolVar(&next, "next", false, "Inject tasks into the next step")

cmd.RunE = func(cmd *cobra.Command, args []string) error {
if !opts.Wait && ignoreWorkerErrors {
return fmt.Errorf("Invalid combination of options, not --wait and --ignore-worker-errors")
@@ -40,7 +43,11 @@ func File() *cobra.Command {
return err
}

exitcode, err := queue.Add(context.Background(), run, args[0], opts)
step := run.Step
if next {
step++
}
exitcode, err := queue.Add(context.Background(), run.ForStep(step), args[0], opts)

switch {
case err != nil:
5 changes: 4 additions & 1 deletion cmd/subcommands/queue/drain.go
Original file line number Diff line number Diff line change
@@ -28,6 +28,9 @@ func Drain() *cobra.Command {
runOpts := options.AddRunOptions(cmd)
options.AddTargetOptionsTo(cmd, &opts)

var step int
cmd.Flags().IntVar(&step, "step", step, "Which step are we part of")

cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := context.Background()
backend, err := be.New(ctx, opts)
@@ -40,7 +43,7 @@ func Drain() *cobra.Command {
return err
}

return queue.Drain(ctx, backend, run, *opts.Log)
return queue.Drain(ctx, backend, run.ForStep(step), opts.Queue, *opts.Log)
}

return cmd
5 changes: 4 additions & 1 deletion cmd/subcommands/queue/last.go
Original file line number Diff line number Diff line change
@@ -27,6 +27,9 @@ func Last() *cobra.Command {

options.AddTargetOptionsTo(cmd, &opts)

var step int
cmd.Flags().IntVar(&step, "step", step, "Which step are we part of")

cmd.RunE = func(cmd *cobra.Command, args []string) error {
marker := args[0]
extra := ""
@@ -40,7 +43,7 @@ func Last() *cobra.Command {
return err
}

val, err := qstat.Qlast(ctx, marker, extra, backend, qstat.QlastOptions{})
val, err := qstat.Qlast(ctx, marker, extra, backend, qstat.QlastOptions{Step: step})
if err != nil {
return err
}
7 changes: 5 additions & 2 deletions cmd/subcommands/queue/ls.go
Original file line number Diff line number Diff line change
@@ -31,6 +31,9 @@ func Ls() *cobra.Command {
runOpts := options.AddRunOptions(cmd)
options.AddTargetOptionsTo(cmd, &opts)

var step int
cmd.Flags().IntVar(&step, "step", step, "Which step are we part of")

cmd.RunE = func(cmd *cobra.Command, args []string) error {
path := ""
if len(args) == 1 {
@@ -45,7 +48,7 @@ func Ls() *cobra.Command {

run := runOpts.Run
if run == "" {
rrun, err := util.Singleton(ctx, backend)
rrun, err := util.Latest(ctx, backend)
if err != nil {
return err
}
@@ -57,7 +60,7 @@ func Ls() *cobra.Command {
return err
}

files, errors, err := queue.Ls(ctx, backend, runContext, path, *opts.Log)
files, errors, err := queue.Ls(ctx, backend, runContext.ForStep(step), path, opts.Queue, *opts.Log)
if err != nil {
return err
}
Loading