From e5ac01deb8371eba90085a38cadce7140d9e1490 Mon Sep 17 00:00:00 2001 From: Sam Salisbury Date: Fri, 26 Jul 2019 17:32:13 +0100 Subject: [PATCH] ci: doc+warn about editing/merging generated file (#7201) - This is the result of calling ./install from https://github.com/samsalisbury/circleci-multi-file-config at commit 920f5330430303196781d668e8546337427cceab --- .circleci/Makefile | 18 +++++++--- .circleci/README.md | 81 +++++++++++++++++++++++++------------------- .circleci/config.yml | 5 ++- 3 files changed, 65 insertions(+), 39 deletions(-) diff --git a/.circleci/Makefile b/.circleci/Makefile index 114045597dc0..4bf067a3dfff 100644 --- a/.circleci/Makefile +++ b/.circleci/Makefile @@ -1,4 +1,5 @@ -# SHELL includes all flags for maximum compatibility, see https://fieldnotes.tech/how-to-shell-for-compatible-makefiles/ +# Set SHELL to 'strict mode' without using .SHELLFLAGS for max compatibility. +# See https://fieldnotes.tech/how-to-shell-for-compatible-makefiles/ SHELL := /usr/bin/env bash -euo pipefail -c # CONFIG is the name of the make target someone @@ -10,6 +11,8 @@ VERIFY ?= ci-verify CIRCLECI := circleci --skip-update-check +# Set up some documentation/help message variables. +# We do not attempt to install the CircleCI CLI from this Makefile. CCI_INSTALL_LINK := https://circleci.com/docs/2.0/local-cli/\#installation CCI_INSTALL_MSG := Please install CircleCI CLI. See $(CCI_INSTALL_LINK) CCI_VERSION := $(shell $(CIRCLECI) version 2> /dev/null) @@ -37,7 +40,7 @@ help: @[ -n "$(CCI_VERSION)" ] || echo " $(CCI_INSTALL_MSG)" $(SOURCE_DIR): - @echo Source directory $(SOURCE_DIR)/ not found.; exit 1 + @echo No source directory $(SOURCE_DIR) found.; exit 1 # Make sure our .tmp dir exists. $(shell [ -d .tmp ] || mkdir .tmp) @@ -49,10 +52,17 @@ $(CONFIG): $(OUT) $(VERIFY): config-up-to-date @$(CIRCLECI) config validate $(OUT) -GENERATED_FILE_HEADER := \#\#\# Generated by 'make $(CONFIG)' do not manually edit this file. +define GENERATED_FILE_HEADER +### *** +### WARNING: DO NOT manually EDIT or MERGE this file, it is generated by 'make $(CONFIG)'. +### INSTEAD: Edit or merge the source in $(SOURCE_DIR)/ then run 'make $(CONFIG)'. +### *** +endef +export GENERATED_FILE_HEADER + define GEN_CONFIG @$(CIRCLECI) config pack $(SOURCE_DIR) > $(CONFIG_PACKED) - @echo "$(GENERATED_FILE_HEADER)" > $@ + @echo "$$GENERATED_FILE_HEADER" > $@ @$(CIRCLECI) config process $(CONFIG_PACKED) >> $@ endef diff --git a/.circleci/README.md b/.circleci/README.md index ea7ed54a2b8b..1ec75cafade9 100644 --- a/.circleci/README.md +++ b/.circleci/README.md @@ -1,39 +1,60 @@ -# CircleCI config +# How to use CircleCI multi-file config -This directory contains both the source code (under `./config/`) -and the generated single-file `config.yml` -which defines the CircleCI workflows for this project. +This README and the Makefile should be in your `.circleci` directory, +in the root of your repository. +All path references in this README assume we are in this `.circleci` directory. + +The `Makefile` in this directory generates `./config.yml` in CircleCI 2.0 syntax, +from the tree rooted at `./config/`, which contains files in CircleCI 2.0 or 2.1 syntax. + + +## Quickstart + +The basic workflow is: + +- Edit source files in `./config/` +- When you are done, run `make ci-config` to update `./config.yml` +- Commit this entire `.circleci` directory, including that generated file together. +- Run `make ci-verify` to ensure the current `./config.yml` is up to date with the source. + +When merging this `.circleci` directory: + +- Do not merge the generated `./config.yml` file, instead: +- Merge the source files under `./config/`, and then +- Run `make ci-config` to re-generate the merged `./config.yml` + +And that's it, for more detail, read on! + + +## How does it work, roughly? -The Makefile in this directory generates the `./config.yml` -in CircleCI 2.0 syntax, -from the tree rooted at `./config/`, -which contains files in CircleCI 2.1 syntax. CircleCI supports [generating a single config file from many], using the `$ circleci config pack` command. It also supports [expanding 2.1 syntax to 2.0 syntax] using the `$ circleci config process` command. +We use these two commands, stitched together using the `Makefile` +to implement the workflow. [generating a single config file from many]: https://circleci.com/docs/2.0/local-cli/#packing-a-config [expanding 2.1 syntax to 2.0 syntax]: https://circleci.com/docs/2.0/local-cli/#processing-a-config + ## Prerequisites You will need the [CircleCI CLI tool] installed and working, at least version `0.1.5607`. +You can [download this tool directly from GitHub Releases]. ``` $ circleci version 0.1.5607+f705856 ``` -NOTE: It is recommended to [download this tool directly from GitHub Releases]. -Do not install it using Homebrew, as this version cannot be easily updated. -It is also not recommended to pipe curl to bash (which CircleCI recommend) for security reasons! - [CircleCI CLI tool]: https://circleci.com/docs/2.0/local-cli/ [download this tool directly from GitHub Releases]: https://github.com/CircleCI-Public/circleci-cli/releases -## How to make changes + +## Updating the config source Before making changes, be sure to understand the layout of the `./config/` file tree, as well as circleci 2.1 syntax. @@ -46,18 +67,20 @@ If that's successful, you should then commit every `*.yml` file in the tree rooted in this directory. That is: you should commit both the source under `./config/` and the generated file `./config.yml` at the same time, in the same commit. +The included git pre-commit hook will help with this. Do not edit the `./config.yml` file directly, as you will lose your changes next time `make ci-config` is run. [Syntax and layout]: #syntax-and-layout + ### Verifying `./config.yml` -To check whether or not the current `./config.yml` is up to date with the source, -and whether it is valid, run `$ make ci-verify`. +To check whether or not the current `./config.yml` is up to date with the source +and valid, run `$ make ci-verify`. Note that `$ make ci-verify` should be run in CI, -as well as by a local git commit hook, -to ensure we never commit files that are invalid or out of date. +in case not everyone has the git pre-commit hook set up correctly. + #### Example shell session @@ -71,6 +94,7 @@ Changes detected in .circleci/, running 'make -C .circleci ci-verify' --> Config file at config.yml is valid. ``` + ### Syntax and layout It is important to understand the layout of the config directory. @@ -89,15 +113,12 @@ $ tree . ├── config # The source code for config.yml is rooted here. │   ├── @config.yml # Files beginning with @ are treated specially by `circleci config pack` │   ├── commands # Subdirectories of config become top-level keys. -│   │   └── go_test.yml # Filenames (minus .yml) become top-level keys under their parent (in this case "commands"). -│ │ # The contents of go_test.yml therefore are placed at: .commands.go_test: -│   └── jobs # jobs also becomes a top-level key under config... -│   ├── build-go-dev.yml # ...and likewise filenames become keys under their parent. -│   ├── go-mod-download.yml -│   ├── install-ui-dependencies.yml -│   ├── test-go-race.yml -│   ├── test-go.yml -│   └── test-ui.yml +│   │   └── go_test.yml # Filenames (minus .yml) become top-level keys under +│   │   └── go_build.yml # their parent (in this case "commands"). +│ │ # The contents of go_test.yml therefore are placed at: .commands.go_test: +│   └── jobs # jobs also becomes a top-level key under config... +│   ├── build.yml # ...and likewise filenames become keys under their parent. +│   └── test.yml └── config.yml # The generated file in 2.0 syntax. ``` @@ -107,11 +128,3 @@ should be at the top-level, rather than underneath a key named after their filen This naming convention is unfortunate as it breaks autocompletion in bash, but there we go. -### Why not just use YAML references? - -YAML references only work within a single file, -this is because `circleci config pack` is not a text-level packer, -but rather stitches together the structures defined in each YAML -file according to certain rules. -Therefore it must parse each file separately, -and YAML references are handled by the parser. diff --git a/.circleci/config.yml b/.circleci/config.yml index f80e499b56f0..71946ed483a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,7 @@ -### Generated by 'make ci-config' do not manually edit this file. +### *** +### WARNING: DO NOT manually EDIT or MERGE this file, it is generated by 'make ci-config'. +### INSTEAD: Edit or merge the source in config/ then run 'make ci-config'. +### *** version: 2 jobs: pre-flight-checks: