auto-clean

Signed-off-by: Nick Mitchell <[email protected]>

.github/workflows/tests.yml

@@ -33,7 +33,7 @@ jobs:
           - ./tests/bin/ci.sh -i 'test7(b.*|c.*|d.*|e.*|g.*|h.*)' 
           - ./tests/bin/ci.sh -i 'test7f.*'
           - ./tests/bin/ci.sh -i 'test8.*'
-          - /tmp/lunchpail bat demos/data-prep-kit # bat=Build and Test
+          - /tmp/lunchpail bat demos/data-prep-kit --concurrency 1 --auto-clean --target=$LUNCHPAIL_TARGET # bat=Build and Test
           - ./tests/bin/go.sh
           - ./tests/bin/pipelines.sh
         os: [ubuntu-latest]

cmd/options/build.go

@@ -39,6 +39,7 @@ func AddBuildOptions(cmd *cobra.Command) (*build.Options, error) {
 
 	cmd.Flags().IntVar(&options.Pack, "pack", options.Pack, "Run k concurrent tasks; if k=0 and machine has N cores, then k=N")
 	cmd.Flags().BoolVarP(&options.Gunzip, "gunzip", "z", options.Gunzip, "Gunzip inputs before passing them to the worker logic")
+	cmd.Flags().BoolVar(&options.AutoClean, "auto-clean", options.AutoClean, "Clean up any caches prior to exiting")
 
 	AddTargetOptionsTo(cmd, &options)
 	AddLogOptionsTo(cmd, &options)

cmd/subcommands/bat.go

@@ -25,13 +25,13 @@ func init() {
 		panic(err)
 	}
 
-	concurrency := 2
-	cmd.Flags().IntVarP(&concurrency, "concurrency", "", concurrency, "Maximum tests to run concurrently")
+	concurrency := 4
+	cmd.Flags().IntVar(&concurrency, "concurrency", concurrency, "Maximum tests to run concurrently")
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
 		ctx := context.Background()
 
-		backend, err := be.New(ctx, *buildOpts)
+		backend, err := be.NewInitOk(ctx, true, *buildOpts)
 		if err != nil {
 			return err
 		}

cmd/subcommands/tester.go

@@ -34,7 +34,7 @@ func init() {
 			ctx := context.Background()
 
 			buildOpts.Target.Platform = target.Local
-			backend, err := be.New(ctx, *buildOpts)
+			backend, err := be.NewInitOk(ctx, true, *buildOpts)
 			if err != nil {
 				return err
 			}

demos/data-prep-kit/code/header-cleanser/requirements.txt

@@ -1,5 +1,5 @@
 data-prep-toolkit==0.2.2.dev1
-scancode-toolkit ; platform_system != 'Darwin'
+scancode-toolkit-mini
 
 # we can probably update to 18+, but we will have to re-generate expected output as pyarrow 18 seems to have resulted in a binary format change
 pyarrow<17

pkg/boot/up.go

@@ -112,7 +112,7 @@ func upLLIR(ctx context.Context, backend be.Backend, ir llir.LLIR, opts UpOption
 		}
 	}()
 
-	if opts.Watch && !util.StdoutIsTty() {
+	if opts.Watch && opts.RedirectTo == "" && !util.StdoutIsTty() {
 		// if stdout is not a tty, then we can't support
 		// watch, no matter what the user asked for
 		fmt.Fprintf(os.Stderr, "Warning: disabling watch mode because stdout is not a tty\n")

pkg/build/options.go

@@ -46,6 +46,9 @@ type Options struct {
 
 	// Gunzip inputs before passing them to the worker logic
 	Gunzip bool `yaml:",omitempty"`
+
+	// Clean up any caches prior to exiting
+	AutoClean bool `yaml:"autoClean,omitempty"`
 }
 
 //go:embed buildOptions.json

pkg/fe/transformer/api/shell/lower.go

@@ -48,6 +48,11 @@ func LowerAsComponent(buildName string, ctx llir.Context, app hlir.Application,
 	app.Spec.Env["LUNCHPAIL_STEP"] = strconv.Itoa(ctx.Run.Step)
 	app.Spec.Env["LUNCHPAIL_QUEUE_BUCKET"] = ctx.Queue.Bucket
 
+	clean := ""
+	if opts.AutoClean {
+		clean = `trap "echo 'Cleaning up venv $(dirname $venvBin)'; rm -rf $(dirname $venvBin)" EXIT`
+	}
+
 	for _, needs := range app.Spec.Needs {
 		var req string
 
@@ -59,8 +64,10 @@ func LowerAsComponent(buildName string, ctx llir.Context, app hlir.Application,
 		}
 
 		component.Spec.Command = fmt.Sprintf(`set -e
-PATH=$($LUNCHPAIL_EXE needs %s %s %s --verbose=%v):$PATH
-%s`, needs.Name, needs.Version, req, opts.Log.Verbose, component.Spec.Command)
+venvBin="$($LUNCHPAIL_EXE needs %s %s %s --verbose=%v)"
+PATH="$venvBin":$PATH
+%s
+%s`, needs.Name, needs.Version, req, opts.Log.Verbose, clean, component.Spec.Command)
 	}
 
 	for _, dataset := range app.Spec.Datasets {