feat: Ability to import a set of jobs from a remote directory

See `examples/advaned/import-remote` for more information.
mumoshu · Apr 4, 2020 · ba1b8df · ba1b8df
1 parent 123bd89
commit ba1b8df
Show file tree

Hide file tree

Showing 5 changed files with 84 additions and 5 deletions.
diff --git a/examples/advanced/import-remote/import.variant b/examples/advanced/import-remote/import.variant
@@ -0,0 +1,9 @@
+job "foo" {
+  import = "git::ssh://[email protected]/mumoshu/variant2@examples/advanced/import/foo?ref=master"
+}
+
+job "test" {
+  run "foo bar" {
+    message = "message1"
+  }
+}
diff --git a/examples/advanced/import-remote/import_test.variant b/examples/advanced/import-remote/import_test.variant
@@ -0,0 +1,23 @@
+// test for `job "test"`
+test "test" {
+  case "case1" {
+    exitstatus = 0
+    err = ""
+    out = "message1"
+  }
+
+  run "test" {
+  }
+
+  assert "error" {
+    condition = run.err == case.err
+  }
+
+  assert "out" {
+    condition = (run.res.set && run.res.stdout == case.out) || !run.res.set
+  }
+
+  assert "exitstatus" {
+    condition = (run.res.set && run.res.exitstatus == case.exitstatus) || !run.res.set
+  }
+}
diff --git a/go.mod b/go.mod
@@ -12,7 +12,6 @@ require (
 	github.com/go-playground/universal-translator v0.17.0 // indirect
 	github.com/google/go-cmp v0.4.0
 	github.com/google/go-github/v27 v27.0.6 // indirect
-	github.com/hashicorp/go-getter v1.4.1 // indirect
 	github.com/hashicorp/go-multierror v1.0.0
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/hashicorp/hcl/v2 v2.3.0
@@ -33,15 +32,15 @@ require (
 	github.com/ulikunitz/xz v0.5.6 // indirect
 	github.com/urfave/cli v1.22.1 // indirect
 	github.com/variantdev/dag v0.0.0-20191028002400-bb0b3c785363
-	github.com/variantdev/mod v0.14.1
+	github.com/variantdev/mod v0.17.0
 	github.com/variantdev/vals v0.0.0-20191123083656-fc61ba6e8746
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/zclconf/go-cty v1.2.1
 	github.com/zclconf/go-cty-yaml v1.0.1
 	golang.org/x/crypto v0.0.0-20200214034016-1d94cc7ab1c6 // indirect
 	golang.org/x/exp v0.0.0-20200213203834-85f925bdd4d0 // indirect
-	golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2
+	golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2 // indirect
 	gopkg.in/go-playground/validator.v9 v9.31.0 // indirect
 	gopkg.in/yaml.v2 v2.2.8 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71

diff --git a/go.sum b/go.sum
@@ -14,9 +14,12 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T
 cloud.google.com/go v0.53.0 h1:MZQCQQaRwOrAcuKjiHWHrgKykt4fZyuwF2dtiG3fGW8=
 cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0 h1:sAbMqjY1PEQKZBWfbu6Y6bsupJ9c4QdHnzg/VvYTLcE=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0 h1:9/vpR43S4aJaROxqQHQ3nH9lfyKKV0dC3vOmnw8ebQQ=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0 h1:RPUcBvDeYgQFMfQu1eBMq6piD1SXmLH+vK3qjewZPus=
@@ -302,6 +305,7 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
@@ -837,6 +841,7 @@ github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f h1:BVwpUVJDADN2ufcGik7W992pyps0wZ888b/y9GXcLTU=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
@@ -991,6 +996,10 @@ github.com/variantdev/mod v0.14.0 h1:Zxf8XZEX7c7sHS8KXZIVGPdcp3cmUwUCRiQsdEncTvs
 github.com/variantdev/mod v0.14.0/go.mod h1:0a93gzewZUjp+DaS3SsfqgXvglNPPi97yoLi5rw2T48=
 github.com/variantdev/mod v0.14.1 h1:U1l/UboLnnq/VOI/m5AqpI4pSQe2JWcot1AyPl5qziE=
 github.com/variantdev/mod v0.14.1/go.mod h1:dYLHWwYJ9lCK6uHijn8dPJOcs31uWsm7PgRzyuZJwlI=
+github.com/variantdev/mod v0.16.0 h1:To3yKfbEG+p9mQepMcpEGkuAafJkZmzKt/HxsJt4fz8=
+github.com/variantdev/mod v0.16.0/go.mod h1:5UZ3felnbNUuTPul3SWYl4niPOh30gjDIqQCdJ+hT4A=
+github.com/variantdev/mod v0.17.0 h1:VML5qy3/PYrZaxGOkvS0sZYIOihwDcEqiD5JqxqpzAc=
+github.com/variantdev/mod v0.17.0/go.mod h1:5UZ3felnbNUuTPul3SWYl4niPOh30gjDIqQCdJ+hT4A=
 github.com/variantdev/vals v0.0.0-20191123083656-fc61ba6e8746 h1:uVcemWq1J1f3X49ZlrbQ/OkbqGvnpn6KuTfIvlP0hAY=
 github.com/variantdev/vals v0.0.0-20191123083656-fc61ba6e8746/go.mod h1:6Ma8+ZOzCxDY5cBSaoRQqE3E5IBaD9BYLfdUMmSsgHg=
 github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
@@ -1145,6 +1154,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1265,6 +1275,7 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
 google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.17.0 h1:0q95w+VuFtv4PAx4PZVQdBMmYbaCHbnfKaEiDIcVyag=
 google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.21.0 h1:zS+Q/CJJnVlXpXQVIz+lH0ZT2lBuT2ac7XD8Y/3w6hY=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

diff --git a/pkg/app/app.go b/pkg/app/app.go
@@ -3,8 +3,10 @@ package app
 import (
 	"flag"
 	"fmt"
+	"github.com/variantdev/mod/pkg/depresolver"
 	"io"
 	"io/ioutil"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -329,7 +331,35 @@ func NewFromSources(srcs map[string][]byte) (*App, error) {
 	return newApp(app, cc, "", false)
 }
 
-func newConfigFromDir(dir string) (map[string]*hcl2.File, *HCL2Config, error) {
+func newConfigFromDir(dirPathOrURL string) (map[string]*hcl2.File, *HCL2Config, error) {
+	var dir string
+
+	s := strings.Split(dirPathOrURL, "::")
+
+	if len(s) > 1 {
+		forcePrefix := s[0]
+
+		u, err := url.Parse(s[1])
+		if err != nil {
+			return nil, nil, err
+		}
+		remote, err := depresolver.New(depresolver.Home(".variant2/cache"))
+		if err != nil {
+			return nil, nil, err
+		}
+
+		us := forcePrefix + "::" + u.String()
+
+		cacheDir, err := remote.ResolveDir(us)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		dir = cacheDir
+	} else {
+		dir = dirPathOrURL
+	}
+
 	files, err := conf.FindVariantFiles(dir)
 	if err != nil {
 		return map[string]*hcl2.File{}, nil, fmt.Errorf("failed to get %s files: %v", conf.VariantFileExt, err)
@@ -389,7 +419,14 @@ func newApp(app *App, cc *HCL2Config, importBaseDir string, enableImports bool)
 				return nil, fmt.Errorf("[bug] Imports are disable in the embedded mode")
 			}
 
-			d := filepath.Join(importBaseDir, *j.Import)
+			var d string
+
+			if strings.Contains(*j.Import, ":") {
+				d = *j.Import
+			} else {
+				d = filepath.Join(importBaseDir, *j.Import)
+			}
+
 			a, err := New(d)
 
 			if err != nil {