Refactoring then add minimal update code, implements repos and flows …

…core and initial code for repotemplates.

app.go

@@ -93,12 +93,15 @@ type Forj struct {
     Infra_repo *string // Infra repository name flag value
     Orga_name  *string // Infra repository name flag value
 
-    Branch            string   // branch name
-    Workspace         string   // Workspace name
-    Workspace_path    string   // Workspace directory path.
-    ContribRepo_uri   *url.URL // URL to github raw files
-    contrib_repo_path string   // Contribution repository Path
-    flow              string   // Name of the flow implemented. defined at create time.
+    Workspace              string   // Workspace name
+    Workspace_path         string   // Workspace directory path.
+    ContribRepo_uri        *url.URL // URL to github raw files for plugin files.
+    RepotemplateRepo_uri   *url.URL // URL to github raw files for RepoTemplates.
+    FlowRepo_uri           *url.URL // URL to github raw files for Flows.
+    contrib_repo_path      string   // Contribution repository Path
+    flow_repo_path         string   // Contribution repository Path
+    repotemplate_repo_path string // Contribution repository Path
+    flow                   string // Name of the flow implemented. defined at create time.
     // TODO: enhance infra README.md with a template.
 
     infra_readme      string   // Initial infra repo README.md text.
@@ -122,23 +125,28 @@ func (a *Forj) init() {
     a.Infra_repo = a.infra_rep_f.String()
     a.app.Version("forjj V0.0.1 (POC)").Author("Christophe Larsonneur <[email protected]>")
 
-    u, _ := url.Parse("https://github.hpe.com/forj/forjj-contribs/raw")
+    u, _ := url.Parse("https://github.hpe.com/forj/forjj-contribs/raw/master")
 
     a.ContribRepo_uri = u
-    a.Branch = "master"
     a.drivers = make(map[string]*Driver)
     a.Actions = make(map[string]ActionOpts)
 
     no_opts := map[string]interface{}{}
+    contribs_repo := map[string]interface{}{"envar": "CONTRIBS_REPO"}
+    repotemplates_repo := map[string]interface{}{"envar": "REPOTEMPLATES_REPO"}
+    flows_repo := map[string]interface{}{"envar": "FLOWS_REPO"}
     required := map[string]interface{}{"required": true}
     ssh_dir_opts := map[string]interface{}{"default": fmt.Sprintf("%s/.ssh", os.Getenv("HOME"))}
     no_set_value_opts := map[string]interface{}{"set_value": false}
 
     a.SetCommand("create", create_action_help)
     a.SetCmdArg("create", "workspace", create_orga_help, required)
     a.SetCmdFlag("create", ssh_dir_flag_name, create_ssh_dir_help, ssh_dir_opts)
-    a.SetCmdFlag("create", "contrib-repo", create_contrib_help, no_opts)
+    a.SetCmdFlag("create", "contribs-repo", contribs_repo_help, contribs_repo)
+    a.SetCmdFlag("create", "flows-repo", flows_repo_help, flows_repo)
+    a.SetCmdFlag("create", "repotemplates-repo", repotemplates_repo_help, repotemplates_repo)
     a.SetCmdFlag("create", "infra-upstream", create_infra_upstream, no_opts)
+    a.SetCmdFlag("create", "docker-exe-path", docker_exe_path_help, no_opts)
 
     a.c_drivers_list_f = SetDriversListFlag(a.SetCmdFlag("create", "apps", driver_help, no_set_value_opts))
 
@@ -149,8 +157,11 @@ func (a *Forj) init() {
     a.SetCommand("update", update_action_help)
     a.SetCmdArg("update", "workspace", update_orga_help, required)
     a.SetCmdFlag("update", ssh_dir_flag_name, update_ssh_dir_help, ssh_dir_opts)
-    a.SetCmdFlag("update", "contrib-repo", update_contrib_help, no_opts)
-    a.SetCmdFlag("create", "infra_upstream", update_infra_upstream, no_opts)
+    a.SetCmdFlag("update", "contribs-repo", contribs_repo_help, contribs_repo)
+    a.SetCmdFlag("update", "flows-repo", flows_repo_help, flows_repo)
+    a.SetCmdFlag("update", "repotemplates-repo", repotemplates_repo_help, repotemplates_repo)
+    a.SetCmdFlag("update", "infra_upstream", update_infra_upstream, no_opts)
+    a.SetCmdFlag("update", "docker-exe-path", docker_exe_path_help, no_opts)
     // Additional options will be loaded from the selected driver itself.
 
     a.u_drivers_list_f = SetDriversListFlag(a.SetCmdFlag("update", "apps", driver_help, no_set_value_opts))
@@ -227,12 +238,11 @@ func (a *Forj) InitializeDriversFlag() {
     }
 }
 
+// Provide value for some forjj internal parameters. Used by InitializeDriversFlag to provide values to plugins as they requested it.
 func (a *Forj) GetInternalData(param string) (result string) {
     switch param {
     case "organization":
         result = a.w.Organization
-    case "branch":
-        result = a.Branch
     case "infra":
         result = a.w.Infra
     case "instance-name" :
@@ -368,22 +378,46 @@ func (a *Forj) LoadContext(args []string) {
 
     // Identifying appropriate Contribution Repository.
     // The value is not set in flagsv. But is in the parser context.
-    if contrib_repo, found := a.flagValue(context, opts.flags["contrib-repo"]); found {
-        if u, err := url.Parse(contrib_repo); err != nil {
+    opts.set_from_urlflag(a, context, "contribs-repo", a.ContribRepo_uri, &a.contrib_repo_path)
+    opts.set_from_urlflag(a, context, "flows-repo", a.FlowRepo_uri, &a.flow_repo_path)
+    opts.set_from_urlflag(a, context, "repotemplates-repo", a.RepotemplateRepo_uri, &a.repotemplate_repo_path)
+
+    // Getting list of drivers (--apps)
+    a.drivers_list.list = make(map[string]DriverDef)
+    a.drivers_list.GetDriversFromContext(context, a.c_drivers_list_f)
+    a.drivers_list.GetDriversFromContext(context, a.u_drivers_list_f)
+
+    // Retrieving additional extra parameters to store in the workspace.
+    opts.set_from_flag(a, context, "docker-exe-path", &a.w.DockerBinPath, nil)
+}
+
+type validateHdlr func(string) error
+
+// Set a string variable pointer with value found in cli context.
+func (o *ActionOpts)set_from_flag(a *Forj, context *kingpin.ParseContext, flag string, store *string, val_fcnt validateHdlr) error {
+    if d, found := a.flagValue(context, o.flags[flag]) ; found {
+        if val_fcnt != nil {
+            if err := val_fcnt(d) ; err != nil {
+                return err
+            }
+        }
+        *store = d
+    }
+    return nil
+}
+
+func (o *ActionOpts)set_from_urlflag(a *Forj, context *kingpin.ParseContext, flag string, theurl *url.URL, store *string) error {
+    if d, found := a.flagValue(context, o.flags[flag]) ; found {
+        if u, err := url.Parse(d); err != nil {
             println(err)
         } else {
-            a.ContribRepo_uri = u
+            *theurl = *u
             if u.Scheme == "" {
-                a.contrib_repo_path = contrib_repo
+                *store = d
             }
         }
     }
-
-    // Getting list of drivers (--app)
-    a.drivers_list.list = make(map[string]DriverDef)
-    a.drivers_list.GetDriversFromContext(context, a.c_drivers_list_f)
-    a.drivers_list.GetDriversFromContext(context, a.u_drivers_list_f)
-
+    return nil
 }
 
 func (*Forj) argValue(context *kingpin.ParseContext, f *kingpin.ArgClause) (value string, found bool) {
@@ -449,6 +483,10 @@ func (a *Forj) SetCmdFlag(cmd, name, help string, options map[string]interface{}
         arg.Hidden()
     }
 
+    if v, ok := options["envar"]; ok {
+        arg.Envar(to_string(v))
+    }
+
     if v, ok := options["set_value"]; ok && to_bool(v) {
         if to_bool(v) {
             a.Actions[cmd].flagsv[name] = arg.String()

create.go

@@ -24,28 +24,40 @@ func (a *Forj) Create() error {
     // save infra repository location in the workspace.
     defer a.w.Save(a)
 
+    defer a.driver_cleanup(a.w.Instance) // Ensure upstream instances will be shutted down when done.
+
     if err, aborted := a.ensure_infra_exists() ; err != nil {
         if !aborted {
             return fmt.Errorf("Failed to ensure infra exists. %s", err)
         }
         log.Printf("Warning. %s", err)
     }
 
+    if err := a.do_driver_maintain(a.w.Instance) ; err != nil { // This will create/configure the upstream service
+        return err
+    }
+
     // Now, we are in the infra repo root directory and at least, the 1st commit exist.
 
     // Loop on drivers requested like jenkins classified as ci type.
     for instance, _ := range a.drivers {
-        defer a.driver_cleanup(instance) // Ensure all instances will be shutted down when done.
-
         if instance == a.w.Instance {
             continue // Do not try to create infra-upstream twice.
         }
+
+        defer a.driver_cleanup(instance) // Ensure all instances will be shutted down when done.
+
         if err, aborted := a.do_driver_create(instance) ; err != nil {
             if !aborted {
                 return fmt.Errorf("Failed to create '%s' source files. %s", instance, err)
             }
             log.Printf("Warning. %s", err)
         }
+
+        // TODO: Except if --no-maintain is set, we could just create files and do maintain later.
+        if err := a.do_driver_maintain(instance) ; err != nil { // This will create/configure the upstream service
+            return err
+        }
     }
 
     log.Print("FORJJ - create ", a.w.Organization, " DONE")
@@ -107,6 +119,7 @@ func (a *Forj) ensure_infra_exists() (err error, aborted bool) {
             err = fmt.Errorf("%s\n%s", err, e)
         }
     }
+
     return
 }
 
@@ -281,9 +294,5 @@ func (a *Forj) do_driver_create(instance string) (err error, aborted bool) {
         }
     }
 
-    // TODO: Except if --no-maintain is set, we could just create files and do maintain later.
-    if err := a.do_driver_maintain(instance) ; err != nil { // This will create/configure the upstream service
-        return err, false
-    }
     return
 }

drivers_options.go

@@ -4,14 +4,11 @@ import (
     "fmt"
     "github.hpe.com/christophe-larsonneur/goforjj/trace"
     "gopkg.in/alecthomas/kingpin.v2"
-    "io/ioutil"
-    "net/http"
     "os"
-    "os/user"
     "regexp"
-    "strings"
     "text/template"
     "bytes"
+    "path"
 )
 
 // Load driver options to a Command requested.
@@ -41,77 +38,49 @@ func (d *Driver)Model() (m *DriverModel) {
 func (a *Forj) read_driver(instance_name string) (err error) {
     var (
         yaml_data   []byte
-        driver_name = a.drivers[instance_name].name
-        service_type = a.drivers[instance_name].driver_type
-        source      string
+        driver      *Driver
     )
+    if d, ok := a.drivers[instance_name] ; ok {
+        driver = d
+    }
 
-    if driver_name == "" {
+    if driver.name == "" {
         return
     }
 
-    if a.ContribRepo_uri.Scheme == "" {
-        // File to read locally
-        source = fmt.Sprintf("%s/%s/%s/%s.yaml", a.ContribRepo_uri.Path, service_type, driver_name, driver_name)
-        if source[:1] == "~" {
-            if user, err := user.Current(); err != nil {
-                kingpin.Fatalf("Unable to get your user. %s. Consider to replace ~ by $HOME\n", err)
-            } else {
-                source = string(regexp.MustCompile("^~").ReplaceAll([]byte(source), []byte(user.HomeDir)))
-            }
-        }
-        gotrace.Trace("Load plugin %s file definition at '%s'", service_type, source)
-        if d, err := ioutil.ReadFile(source); err != nil {
-            return fmt.Errorf("Unable to read '%s'. %s\n", source, err)
-        } else {
-            yaml_data = d
-        }
-
-    } else {
-        // File to read from an url. Usually, a raw from github.
-        source = fmt.Sprintf("%s/%s/%s/%s/%s.yaml", a.ContribRepo_uri, a.Branch, service_type, driver_name, driver_name)
-        gotrace.Trace("Load plugin %s file definition at '%s'", service_type, source)
+    ContribRepoUri := *a.ContribRepo_uri
+    ContribRepoUri.Path = path.Join(ContribRepoUri.Path, driver.driver_type, driver.name, driver.name + ".yaml")
 
-        var resp *http.Response
-        if resp, err = http.Get(source); err != nil {
-            return fmt.Errorf("Unable to read '%s'. %s\n", source, err)
-        }
-        defer resp.Body.Close()
-
-        var d []byte
-        if d, err = ioutil.ReadAll(resp.Body); err != nil {
-            return
-        }
-        if strings.Contains(http.DetectContentType(d), "text/plain") {
-            yaml_data = d
-        }
+    if yaml_data, err = read_document_from(a.ContribRepo_uri) ; err != nil {
+        return
     }
 
-    d := a.drivers[instance_name]
-    if err = d.plugin.PluginDefLoad(yaml_data); err != nil {
+    if err = driver.plugin.PluginDefLoad(yaml_data); err != nil {
         return
     }
+
     // Set defaults value for undefined parameters
     var ff string
-    if d.plugin.Yaml.CreatedFile == "" {
-        ff =  "." + d.instance_name + ".created"
-        d.forjj_flag_file = true // Forjj will test the creation success itself, as the driver did not created it automatically.
+    if driver.plugin.Yaml.CreatedFile == "" {
+        ff =  "." + driver.instance_name + ".created"
+        driver.forjj_flag_file = true // Forjj will test the creation success itself, as the driver did not created it automatically.
     } else {
-        ff = d.plugin.Yaml.CreatedFile
+        ff = driver.plugin.Yaml.CreatedFile
     }
 
     // Initialized defaults value from templates
     var doc bytes.Buffer
 
     if t, err := template.New("plugin").Parse(ff) ; err != nil {
-        return fmt.Errorf("Unable to interpret plugin yaml definition. '/created_flag_file' has an invalid template string '%s'. %s", d.plugin.Yaml.CreatedFile, err)
+        return fmt.Errorf("Unable to interpret plugin yaml definition. '/created_flag_file' has an invalid template string '%s'. %s", driver.plugin.Yaml.CreatedFile, err)
     } else {
-        t.Execute(&doc, d.Model())
+        t.Execute(&doc, driver.Model())
     }
-    d.flag_file = doc.String()
-    gotrace.Trace("Created flag file name Set to default for plugin instance '%s' to %s", d.instance_name, d.plugin.Yaml.CreatedFile)
+    driver.flag_file = doc.String()
+    gotrace.Trace("Created flag file name Set to default for plugin instance '%s' to %s", driver.instance_name, driver.plugin.Yaml.CreatedFile)
 
     return
+
 }
 
 // Initialize command drivers flags with plugin definition loaded from plugin yaml file.

help.go

@@ -20,11 +20,15 @@ But FORJJ is not DevOps if you do not help your teams to do DevOps.
   create_orga_help       = "organization workspace used to store repositories locally or in docker volume."
   create_ssh_dir_help    = "PATH to a git ssh keys directory. It will be mounted as local path '/home/devops/.ssh' in the container."
   driver_help            = "Define the application name and type. Formated as 'APP[,APP ...]' where APP is as '<Type>:<DriverName>[:<InstanceName>]'. It will search for <Type>/<DriverName>/<DriverName>.yaml from the forjj-contribs repo."
+
+  docker_exe_path_help   = "Path to a static docker binary used when a forjj plugin service container requires DooD (Docker out of Docker) capability."
+  contribs_repo_help     = "Set a local forjj-contribs directory like or a github like url for FORJJ plugins."
+  flows_repo_help        = "Set a local forjj-flows directory like or a github like url for FORJJ flows."
+  repotemplates_repo_help= "Set a local forjj-repotemplates directory like or a github like url for FORJJ Repository templates."
   create_ci_driver_help  = "Defines the Continous Integration system to implement"
   create_us_driver_help  = "Defines the GIT upstream to implement. To get the list of a driver parameter, set this flag in conjonction with --help."
   create_infra_url_help  = "Infra repository url to git clone. By default, the repository will be created."
   create_infra_path_help = "Original infra path containing source code. By default, if the repository is going to be created, it will be empty."
-  create_contrib_help    = "Set a local forjj-contribs directory like or a github like url."
   create_infra_upstream  = "Set infra repository upstream instance"
 
   update_infra_upstream  = "Set infra repository upstream instance"
@@ -35,7 +39,6 @@ But FORJJ is not DevOps if you do not help your teams to do DevOps.
   update_ssh_dir_help    = "PATH to a git ssh keys directory. It will be mounted as local path '/home/devops/.ssh' in the container."
   update_ci_driver_help  = "Defines the Continous Integration system to implement"
   update_us_driver_help  = "Defines the GIT upstream to implement. To get the list of a driver parameter, set this flag in conjonction with --help."
-  update_contrib_help    = create_contrib_help
 
   maintain_action_help   = "Used by your CI to update the infra from the 'infra' repository.\n"
   maintain_orga_help     = "organization workspace used to store repositories locally or in docker volume."

repos.go

@@ -0,0 +1,8 @@
+package main
+
+type RepoStruct struct {
+    Name string
+    Flow string
+    Templates RepoTemplatesStruct
+}
+

repotemplates.go

@@ -0,0 +1,5 @@
+package main
+
+// TODO: Develop the way to build a local Repository with initial commits or a way to apply some repo templates.
+
+type RepoTemplatesStruct []string