diff --git a/README.md b/README.md index 9e1471e..d107fbf 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,21 @@ cat someYaml | p2 -t template.j2 -f yaml ``` ### Advanced Usage + +#### Side-effectful filters +`p2` allows enabling a suite of non-standard pongo2 filters which have +side-effects on the system. These filters add a certain amount of +minimal scripting ability to `p2`, namely by allowing a single template +to use filters which can perform operations like writing files and +creating directories. + +These are __unsafe__ filters to use with uninspected templates, and so +by default are disabled. They can be enabled on a per-filter basis with +the `--enable-filters` flag. For template debugging purposes, they can +also be enabled in no-op mode with `--enable-noops` which will allow +all filters but disable their side-effects. + +#### Passing structured data in environment variable keys It is technically possible to store complex data in environment variables. `p2` supports this use-case (without commenting if it's a good idea). To use it, pass the name of the environment variable as the `--input` and specify @@ -47,6 +62,77 @@ pass the name of the environment variable as the `--input` and specify p2 -t template.j2 -f json --use-env-key -i MY_ENV_VAR ``` +#### Multiple file templating via `write_file` +`p2` implements the custom `write_file` filter extension to pongo2. +`write_file` takes a filename as an argument (which can itself be a +templated value) and creates and outputs any input content to that +filename (overwriting the destination file). + +This makes it possible to write a template which acts more like a +script, and generates multiple output values. Example: + +`input.yml`: +```yaml +users: +- user: mike + content: This is Mike's content. +- user: sally + content: This is Sally's content. +- user: greg + content: This is greg's content. +``` + +`template.p2`: +```Django +{% macro user_content(content) %} +{{content|safe}} +{% endmacro %} + +{% for user in users %} +## {{user.name}}.txt output +{% set filename = user.name|stringformat:"%s.txt" %} +{{ user_content( user.content ) | write_file:filename }} +## +{% endfor %} +``` + +Now executing the template: +```sh +$ p2 --enable-write_file -t template.p2 -i input.yml + + + +## mike.txt output + + +This is Mike's content. + +## + +## sally.txt output + + +This is Sally's content. + +## + +## greg.txt output + + +This is greg's content. + +## + +$ ls +greg.txt input.yml mike.txt sally.txt template.p2 +``` + +We get the output, but we have also created a new set of files +containing the content from our macro. + +Note that until pongo2 supports multiple filter arguments, the file +output plugin creates files with the maximum possible umask of the user. + ## Building It is recommended to build using the included Makefile. This correctly sets up diff --git a/custom_filters.go b/custom_filters.go new file mode 100644 index 0000000..eb67fa9 --- /dev/null +++ b/custom_filters.go @@ -0,0 +1,78 @@ +package main + +import ( + "fmt" + "github.com/flosch/pongo2" + "os" +) + +// This noop filter is registered in place of custom filters which otherwise +// passthru their input (our file filters). This allows debugging and testing +// without running file operations. +func filterNoopPassthru(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { + return in, nil +} + +// This noop filter is registered in place of custom filters which otherwise +// produce no output. +func filterNoop(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { + return nil, nil +} + +// This filter writes the content of its input to the filename specified as its +// argument. The templated content is returned verbatim. +func filterWriteFile(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { + if !in.IsString() { + return nil, &pongo2.Error{ + Sender: "filter:write_file", + ErrorMsg: "Filter input must be of type 'string'.", + } + } + + if !param.IsString() { + return nil, &pongo2.Error{ + Sender: "filter:write_file", + ErrorMsg: "Filter parameter must be of type 'string'.", + } + } + + f, err := os.OpenFile(param.String(), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(0777)) + if err != nil { + return nil, &pongo2.Error{ + Sender: "filter:write_file", + ErrorMsg: fmt.Sprintf("Could not open file for output: %s", err.Error()), + } + } + defer f.Close() + + _, werr := f.WriteString(in.String()) + if werr != nil { + return nil, &pongo2.Error{ + Sender: "filter:write_file", + ErrorMsg: fmt.Sprintf("Could not write file for output: %s", werr.Error()), + } + } + + return in, nil +} + +// This filter makes a directory based on the value of its argument. It passes +// through any content without alteration. This allows chaining with write-file. +func filterMakeDirs(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { + if !param.IsString() { + return nil, &pongo2.Error{ + Sender: "filter:write_file", + ErrorMsg: "Filter parameter must be of type 'string'.", + } + } + + err := os.MkdirAll(in.String(), os.FileMode(0777)) + if err != nil { + return nil, &pongo2.Error{ + Sender: "filter:make_dirs", + ErrorMsg: fmt.Sprintf("Could not create directories: %s %s", in.String(), err.Error()), + } + } + + return in, nil +} diff --git a/p2cli.go b/p2cli.go index b54dab6..b5432c4 100644 --- a/p2cli.go +++ b/p2cli.go @@ -52,6 +52,21 @@ var dataFormats map[string]SupportedType = map[string]SupportedType{ "env": ENV, } +// Map of custom filters p2 implements. These are gated behind the --enable-filter +// command line option as they can have unexpected or even unsafe behavior (i.e. +// templates gain the ability to make filesystem modifications). +// Disabled filters are stubbed out to allow for debugging. + +type CustomFilterSpec struct { + FilterFunc pongo2.FilterFunction + NoopFunc pongo2.FilterFunction +} + +var customFilters map[string]CustomFilterSpec = map[string]CustomFilterSpec{ + "write_file": CustomFilterSpec{filterWriteFile, filterNoopPassthru}, + "make_dirs": CustomFilterSpec{filterMakeDirs, filterNoopPassthru}, +} + var ( inputData map[string]interface{} = make(map[string]interface{}) ) @@ -101,6 +116,9 @@ func main() { TemplateFile string `goptions:"-t, --template, description='Template file to process'"` DataFile string `goptions:"-i, --input, description='Input data path. Leave blank for stdin.'"` OutputFile string `goptions:"-o, --output, description='Output file. Leave blank for stdout.'"` + + CustomFilters string `goptions:"--enable-filters, description='Enable custom p2 filters.'"` + CustomFilterNoops bool `goptions:"--enable-noop-filters, description='Enable all custom filters in noop mode. Supercedes --enable-filters'"` }{ Format: "", } @@ -116,6 +134,25 @@ func main() { log.Fatalln("Template file must be specified!") } + // Register custom filter functions. + if options.CustomFilterNoops { + for filter, spec := range customFilters { + pongo2.RegisterFilter(filter, spec.NoopFunc) + } + } else { + // Register enabled custom-filters + if options.CustomFilters != "" { + for _, filter := range strings.Split(options.CustomFilters, ",") { + spec, found := customFilters[filter] + if !found { + log.Fatalln("This version of p2 does not support the", filter, "custom filter.") + } + + pongo2.RegisterFilter(filter, spec.FilterFunc) + } + } + } + // Determine mode of operations var fileFormat SupportedType = UNKNOWN var inputSource DataSource = SOURCE_ENV