-
Notifications
You must be signed in to change notification settings - Fork 37
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
plugins: initial support of parallel execution (#58)
* add support for plugins, export some internal code, add plugin sample * fix docs in exported code * modify plugin runner to execute plugins independently in parallel * clean up usage doc whitespace * re-organize code, update release script, add better error message handling from plugins * update circle config * fix gitignore and revert circle config changes * pull plugin runner into own file, modify sample, add plugin test * revert test proto file change * add plugin samples and use in circle tests * update circle config to add debug detail * prefer explicit return from goroutine and send on done chan * override pipefail in plugin ci tests * update README with --plugins option in usage * revert uppercasing in json tag for Protolock type, add other json tags * use different plugin name for error sample * add json tags to Data and include example nodejs plugin * run ci test on nodejs plugin * update README with details and wiki link to plugin docs * remove console log demo and add note in nodejs plugin example
- Loading branch information
Showing
20 changed files
with
3,435 additions
and
107 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,5 @@ | ||
| protolock | ||
| pkg | ||
| ./protolock | ||
| pkg | ||
| node_modules | ||
| plugin-samples/plugin-sample-js/etc | ||
| plugin-samples/plugin-sample-js/plugin-sample-js |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,130 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "encoding/json" | ||
| "errors" | ||
| "fmt" | ||
| "os/exec" | ||
| "strings" | ||
| "sync" | ||
|
|
||
| "github.com/nilslice/protolock" | ||
| "github.com/nilslice/protolock/extend" | ||
| ) | ||
|
|
||
| func runPlugins(pluginList string, report *protolock.Report) (*protolock.Report, error) { | ||
| inputData := &bytes.Buffer{} | ||
|
|
||
| err := json.NewEncoder(inputData).Encode(&extend.Data{ | ||
| Current: report.Current, | ||
| Updated: report.Updated, | ||
| PluginWarnings: []protolock.Warning{}, | ||
| }) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| // collect plugin warnings and errors as they are returned from plugins | ||
| pluginWarningsChan := make(chan []protolock.Warning) | ||
| pluginsDone := make(chan struct{}) | ||
| pluginErrsChan := make(chan error) | ||
| var allPluginErrors []error | ||
| go func() { | ||
| for { | ||
| select { | ||
| case <-pluginsDone: | ||
| return | ||
|
|
||
| case err := <-pluginErrsChan: | ||
| if err != nil { | ||
| allPluginErrors = append(allPluginErrors, err) | ||
| } | ||
|
|
||
| case warnings := <-pluginWarningsChan: | ||
| for _, warning := range warnings { | ||
| report.Warnings = append(report.Warnings, warning) | ||
| } | ||
| } | ||
| } | ||
| }() | ||
|
|
||
| wg := &sync.WaitGroup{} | ||
| plugins := strings.Split(pluginList, ",") | ||
| for _, name := range plugins { | ||
| wg.Add(1) | ||
|
|
||
| // copy input data to be passed in to and processed by each plugin | ||
| pluginInputData := bytes.NewReader(inputData.Bytes()) | ||
|
|
||
| // run all provided plugins in parallel, each recieving the same current | ||
| // and updated Protolock structs from the `protolock status` call | ||
| go func(name string) { | ||
| defer wg.Done() | ||
| name = strings.TrimSpace(name) | ||
| path, err := exec.LookPath(name) | ||
| if err != nil { | ||
| if path == "" { | ||
| path = name | ||
| } | ||
| fmt.Println("[protolock] plugin exec error:", err) | ||
| return | ||
| } | ||
|
|
||
| // initialize the executable to be called from protolock using the | ||
| // absolute path and copy of the input data | ||
| plugin := &exec.Cmd{ | ||
| Path: path, | ||
| Stdin: pluginInputData, | ||
| } | ||
|
|
||
| // execute the plugin and capture the output | ||
| output, err := plugin.Output() | ||
| if err != nil { | ||
| pluginErrsChan <- wrapPluginErr(name, path, err) | ||
| return | ||
| } | ||
|
|
||
| pluginData := &extend.Data{} | ||
| err = json.Unmarshal(output, pluginData) | ||
| if err != nil { | ||
| fmt.Println("[protolock] plugin data decode error:", err) | ||
| return | ||
| } | ||
|
|
||
| // gather all warnings from each plugin, and send to warning chan | ||
| // collector as a slice to keep together | ||
| if pluginData.PluginWarnings != nil { | ||
| pluginWarningsChan <- pluginData.PluginWarnings | ||
| } | ||
|
|
||
| if pluginData.PluginErrorMessage != "" { | ||
| pluginErrsChan <- wrapPluginErr( | ||
| name, path, errors.New(pluginData.PluginErrorMessage), | ||
| ) | ||
| } | ||
| }(name) | ||
| } | ||
|
|
||
| wg.Wait() | ||
| pluginsDone <- struct{}{} | ||
|
|
||
| if allPluginErrors != nil { | ||
| var errorMsgs []string | ||
| for _, pluginError := range allPluginErrors { | ||
| errorMsgs = append(errorMsgs, pluginError.Error()) | ||
| } | ||
|
|
||
| return nil, fmt.Errorf( | ||
| `[protolock:plugin] accumulated plugin errors: | ||
| %s`, | ||
| strings.Join(errorMsgs, "\n"), | ||
| ) | ||
| } | ||
|
|
||
| return report, nil | ||
| } | ||
|
|
||
| func wrapPluginErr(name, path string, err error) error { | ||
| return fmt.Errorf("%s: %v (%s)", name, err, path) | ||
| } |
Oops, something went wrong.