diff --git a/.editorconfig b/.editorconfig index 68a67eccc..095e0c5ff 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,6 +13,7 @@ indent_size = 4 [*.yaml] intent_style = space indent_size = 2 +trim_trailing_whitespace = true [*.sh] indent_style = tab diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 32a7d894b..382ea85a7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -22,10 +22,10 @@ jobs: os: ['alpine', 'debian'] steps: - name: "Checkout source code at current commit" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: ${{ secrets.ECR_AWS_ROLE }} aws-region: ${{ env.AWS_REGION }} @@ -44,11 +44,16 @@ jobs: # We therefore designate whichever base OS version we recommend as the best supported # as the one to get the "latest" tag. Initially that will be Alpine. env: - LATEST_TAG_OS: 'alpine' + LATEST_TAG_OS: 'debian' BASE_OS: ${{matrix.os}} run: | - echo ::set-output name=publish::${{ (github.event_name == 'release' && github.event.action == 'published') || (github.event.pull_request.head.repo.full_name == github.repository) }} + echo publish=${{ (github.event_name == 'release' && github.event.action == 'published') || (github.event.pull_request.head.repo.full_name == github.repository) }} >> $GITHUB_OUTPUT + if [[ $BASE_OS == "debian" ]]; then + echo platforms="linux/amd64,linux/arm64" >> $GITHUB_OUTPUT + else + echo platforms="linux/amd64" >> $GITHUB_OUTPUT + fi COMMIT_SHA="${GITHUB_SHA}" if [[ $GITHUB_REF == refs/tags/* ]]; then VERSION=${GITHUB_REF#refs/tags/} @@ -57,7 +62,7 @@ jobs: COMMIT_SHA=${{ github.event.pull_request.head.sha }} fi printf "Version resolved to %s\n" "${VERSION}" - echo ::set-output name=version::${VERSION} + echo version=${VERSION} >> $GITHUB_OUTPUT TAGS="${{ github.repository }}:sha-${COMMIT_SHA:0:7}-${BASE_OS}" TAGS="$TAGS,${{ env.ECR_REGISTRY }}${{ github.repository }}:sha-${COMMIT_SHA:0:7}-${BASE_OS}" if [[ -n $VERSION ]]; then @@ -75,24 +80,27 @@ jobs: printf "Tagging %s with " "${BASE_OS}" if [[ "${BASE_OS}" == "$LATEST_TAG_OS" ]]; then printf "%s\n" "${LATEST_TAGS}" - echo ::set-output name=tags::${LATEST_TAGS} + echo tags=${LATEST_TAGS} >> $GITHUB_OUTPUT else printf "%s\n" "${TAGS}" - echo ::set-output name=tags::${TAGS} + echo tags=${TAGS} >> $GITHUB_OUTPUT fi + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to DockerHub if: steps.prepare.outputs.publish == 'true' - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: "Build and push docker image to DockerHub" id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: push: ${{ steps.prepare.outputs.publish == 'true' }} + platforms: ${{ steps.prepare.outputs.platforms }} tags: ${{ steps.prepare.outputs.tags }} file: ./os/${{matrix.os}}/Dockerfile.${{matrix.os}} build-args: | diff --git a/README.md b/README.md index 1fcdf1469..438e15bce 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ ![Geodesic](docs/geodesic-small.png) -Geodesic is the fastest way to get up and running with a rock solid, production grade cloud platform built entirely from Open Source technologies. +Geodesic is the fastest way to get up and running with a rock solid, production grade cloud platform built entirely from Open Source technologies. It’s a swiss army knife for creating and building consistent platforms to be shared across a team environment. @@ -38,7 +38,7 @@ It easily versions staging environments in a repeatable manner that can be follo It's a way of doing things that allows companies to collaborate on infrastructure (~snowflakes~) and radically reduce Total Cost of Ownership, along with a vibrant and active [slack community](https://slack.cloudposse.com). -It provides a fully customizable framework for defining and building cloud infrastructures backed by [AWS](https://aws.amazon.com/) and powered by [kubernetes](https://kubernetes.io/). It couples best-of-breed technologies with engineering best-practices to equip organizations with the tooling that enables clusters to be spun up in record time without compromising security. +It provides a fully customizable framework for defining and building cloud infrastructures backed by [AWS](https://aws.amazon.com/) and powered by [kubernetes](https://kubernetes.io/). It couples best-of-breed technologies with engineering best-practices to equip organizations with the tooling that enables clusters to be spun up in record time without compromising security. It's works natively with Mac OSX, Linux, and [Windows 10 (WSL)](https://docs.microsoft.com/en-us/windows/wsl/install-win10). @@ -87,23 +87,28 @@ Geodesic is composed of two parts: An organization may chose to leverage all of these components, or just the parts that make their life easier. We recommend starting by using `geodesic` as a Docker base image (e.g. `FROM cloudposse/geodesic:...` pinned to a release and base OS) in your projects. -**Note**: Starting with Geodesic version 0.138.0, we distribute 2 versions of Geodesic Docker images, one based on [Alpine](https://alpinelinux.org/) -and one based on [Debian](https://debian.org), tagged `VERSION-BASE_OS`, e.g. `0.138.0-alpine`. -Prior to this, all Docker images were based on Alpine only and simply tagged `VERSION`. At present, the Alpine version is the most thoroughly tested -and best supported version, and the special Docker tag `latest` will continue to point to the latest Alpine version while this -remains the case. However, we encourage people to use the Debian version and report any issues by opening a GitHub issue, -so that we may eventually make it the preferred version, after which point the `latest` tag will point to latest Debian image. We -will maintain the `latest-alpine` and `latest-debian` Docker tags for those who want to commit to using one base OS or -the other but still want automatic updates. - -**Note**: Geodesic is a large collection of tools. As such, support for the Apple M1 chip is not under -Cloud Posse's control, rather it depends on each tool author updating each tool for the M1 chip. -All of the compiled tools that Cloud Posse has authored and are included in Geodesic are compiled -for M1 (`darwin_arm64`), and of course all of the scripts work on M1 if the interpreters (e.g. -`bash`, `python`) are compiled for M1. Unfortunatetly, this is only a small portion of the overall -toolkit that is assembled in Geodesic. Therefore we do not advise using Geodesic on the M1 at this time -and do not anticipate M1 will be well supported before 2022. Historically, widespread support for a new -chip takes several years to establish; we hope we will not have to wait that long. +**Note**: Starting with Geodesic 2.0, we distribute Geodesic as a multi-platform (`linux/amd64`, `linux/arm64`) Debian-based +Docker image and a single-platform (`linux/amd64`) Alpine-based image. We recommend using the Debian-based image, and the +`cloudposse/geodesic:latest` image now points to it. (Previously `cloudposse/geodesic:latest` was the Alpine image.) +We have deprioritized support for Alpine and may drop it entirely at some point. + +Starting with Geodesic version 0.138.0, we distributed 2 versions of `linux/arm64` Geodesic Docker images, +one based on [Alpine](https://alpinelinux.org/) and one based on [Debian](https://debian.org), tagged `VERSION-BASE_OS`, e.g. `0.138.0-alpine`. +Prior to this, all Docker images were based on Alpine only and simply tagged `VERSION`. Prior to the release of Geodesic version 1.0, +the Alpine version was the most thoroughly tested and best supported version, and the special Docker tag `latest` continued to point +to the latest Alpine version. + +**Note**: Geodesic is a large collection of tools. To run on an Apple natively under Geodesic, binaries need to be compiled +for `linux/arm64`, while to run on Apple natively on outside of Geodesic (on macOS) they need to be +compiled for `darwin/arm64`. As such, support for the Apple M1 (or later) chip is not +fully under Cloud Posse's control, rather it depends on each tool author updating each tool. + +To be included in Geodesic and for Geodesic to support both Intel and Apple CPUs, a tool project must +distribute both a `linux/amd64` and `linux/arm64` binary. (In exceptional cases, if a tool is written +in the `go` language and distributes source code only, Cloud Posse may build the needed binaries.) +As of Geodesic 2.0, all but a small number of tools have started releasing the necessary binaries, +so we removed the ones that were not available on `linux/arm64` in order to provide a consistent +toolkit on both platforms. (See the Geodesic 2.0 [Release Notes](https://github.com/cloudposse/geodesic/releases/tag/2.0.0) for details on which tools were removed.) Want to learn more? [Check out our getting started with Geodesic guide!](https://docs.cloudposse.com/tutorials/geodesic-getting-started/) @@ -119,39 +124,20 @@ Want to learn more? [Check out our getting started with Geodesic guide!](https:/ #### docker run -This will launch the latest version of the geodesic container - +Launching Gedoesic is a bit complex, so we recommend you install a launch script by running ``` -docker run -it --rm \ - --name=custom-image-name \ - --env LS_COLORS \ - --env TERM \ - --env TERM_COLOR \ - --env TERM_PROGRAM \ - --volume=$HOME:/localhost \ - --env LOCAL_HOME=$HOME \ - --privileged \ - --publish 37049:37049 \ - --env GEODESIC_PORT=37049 \ - --env DOCKER_IMAGE=cloudposse/geodesic \ - --env DOCKER_NAME=custom-image-name \ - --env DOCKER_TAG=latest \ - --env GEODESIC_HOST_CWD=$PWD cloudposse/geodesic:latest -l +docker run --rm cloudposse/geodesic:latest-debian init | bash ``` - -#### install a script - +After that, you should be able to launch Geodesic just by typing ``` -git clone git@github.com:cloudposse/geodesic.git -cd geodesic -make all -f Makefile.custom +geodesic ``` ### Customizing your Docker image In general we recommend creating a customized version of Geodesic by creating your own `Dockerfile` starting with ``` -ARG VERSION=0.138.0 +ARG VERSION=2.0.0 ARG OS=debian FROM cloudposse/geodesic:$VERSION-$OS @@ -163,11 +149,47 @@ ENV BANNER="my-custom-geodesic" You can see some example configuration options to include in [Dockerfile.options](./Dockerfile.options). -You can also add extra commands by installing "packages". Both Alpine and Debian have a large selection +#### Multi-platform gotchas + +Although the Geodesic base image is provided in 2 architectures, when you do a local build +of your custom image, it will, by default, only be built for the architecture of the machine you are building on. +This is fine until you want to share it. You need to be aware that if you push just the image you +built with `docker build` you will only be supporting a single architecture. You should use `docker buildx` +to build a multi-platform image and push it to a Docker repository for sharing. + +If you intend to support both architectures, you need to be sure that any customizations +you install are properly installed for both architectures. Package managers handle this for you +automatically, but if you are downloading files directly, you need to be careful to select the right one. +See the use of `TARGETARCH` in [Dockerfile.debian](./os/debian/Dockerfile.debian) for some examples. + +#### Adding packages + +You can also add extra commands by installing "packages". Both Debian and Alpine have a large selection of packages to choose from. Cloud Posse also provides a large set of packages for installing common DevOps commands -and utilities via [cloudposse/packages](https://github.com/cloudposse/packages). -The package repositories are pre-installed in Geodesic, all you need to do is add the packages you want -via `RUN` commands in your Dockerfile. +and utilities via [cloudposse/packages](https://github.com/cloudposse/packages), but `linux/arm64` packages +are only provided for Debian, not Alpine. The package repositories are pre-installed in Geodesic, +all you need to do is add the packages you want via `RUN` commands in your Dockerfile. Debian +will automatically select the correct architecture for the package. + +#### Installing packages in Debian + +Debian uses [`apt`](https://wiki.debian.org/Apt) for package management and we generally recommend using +the [`apt-get`](https://www.debian.org/doc/manuals/apt-guide/ch2.en.html) command to install packages. +In addition to the default repositories, Geodesic pre-installs the Cloud Posse [package](https://github.com/cloudposse/packages) repository +and the Google Cloud SDK package repository. Unlike with `apk`, you do not need to specify a package repository when +installing a package because all repositories will be searched for it. +Also unlike `apk`, `apt-get` does not let you specify a version range on the command line, but they do +allow wildcards. Package versions include a release number (typically "1") at the end, to allow for +updated packages when there is a bug in the package itself. Therefore, best practice is to use a wildcard +for the release number when specifying a package version. For example, +to install the Google Cloud SDK at a version 300.0.0: + +``` +RUN apt-get update && apt-get install -y google-cloud-sdk="300.0.0-*" +``` + +Note the `-y` flag to `apt-get install`. That is required for scripted installation, otherwise the command +will ask for confirmation from the keyboard before installing a package. #### Installing packages in Alpine @@ -190,28 +212,10 @@ pinned to version 4.2.x (which is the last to support Alpine), we can add this t RUN apk update && apk add -u teleport@cloudposse=~4.2 ``` -#### Installing packages in Debian - -Debian uses [`apt`](https://wiki.debian.org/Apt) for package management and we generally recommend using -the [`apt-get`](https://www.debian.org/doc/manuals/apt-guide/ch2.en.html) command to install packages. -In addition to the default repositories, Geodesic pre-installs the Cloud Posse [package](https://github.com/cloudposse/packages) repository -and the Google Cloud SDK package repository. Unlike with `apk`, you do not need to specify a package repository when -installing a package because all repositories will be searched for it. -Also unlike `apk`, `apt-get` does not let you specify a version range on the command line, only an exact version. -So to install the Google Cloud SDK at a specific version, you need to include a trailing `-0` to indicate -the package version. For example, to install version Google Cloud SDK 300.0.0: - -``` -RUN apt-get update && apt-get install -y google-cloud-sdk=300.0.0-0 -``` - -Note the `-y` flag to `apt-get install`. That is required for scripted installation, otherwise the command -will ask for confirmation from the keyboard before installing a package. - ### Customizing your shell at launch time -After you have build your Docker image, or if you are using a shared Docker image, you can -add further customization at launch time. When Geodesic stars up, it looks for customization +After you have built your Docker image, or if you are using a shared Docker image, you can +add further customization at launch time. When Geodesic starts up, it looks for customization scripts and configuration so you can do things like add command aliases or override preconfigured options. Detailed information about launch-time configuration is in the [customization](./docs/customization.md) document, available from within the shell via `man customization`. @@ -311,7 +315,7 @@ In general, PRs are welcome. We follow the typical "fork-and-pull" Git workflow. ## Copyright -Copyright © 2017-2022 [Cloud Posse, LLC](https://cpco.io/copyright) +Copyright © 2017-2023 [Cloud Posse, LLC](https://cpco.io/copyright) @@ -396,7 +400,7 @@ Check out [our other projects][github], [follow us on twitter][twitter], [apply [![README Footer][readme_footer_img]][readme_footer_link] [![Beacon][beacon]][website] - + [logo]: https://cloudposse.com/logo-300x69.svg [docs]: https://cpco.io/docs?utm_source=github&utm_medium=readme&utm_campaign=cloudposse/geodesic&utm_content=docs [website]: https://cpco.io/homepage?utm_source=github&utm_medium=readme&utm_campaign=cloudposse/geodesic&utm_content=website @@ -427,3 +431,4 @@ Check out [our other projects][github], [follow us on twitter][twitter], [apply [share_googleplus]: https://plus.google.com/share?url=https://github.com/cloudposse/geodesic [share_email]: mailto:?subject=Geodesic&body=https://github.com/cloudposse/geodesic [beacon]: https://ga-beacon.cloudposse.com/UA-76589703-4/cloudposse/geodesic?pixel&cs=github&cm=readme&an=geodesic + diff --git a/README.yaml b/README.yaml index b7cc07eac..f897246b8 100644 --- a/README.yaml +++ b/README.yaml @@ -44,21 +44,21 @@ related: description: |- - Geodesic is the fastest way to get up and running with a rock solid, production grade cloud platform built entirely from Open Source technologies. - + Geodesic is the fastest way to get up and running with a rock solid, production grade cloud platform built entirely from Open Source technologies. + It’s a swiss army knife for creating and building consistent platforms to be shared across a team environment. It easily versions staging environments in a repeatable manner that can be followed by any team member. It's a way of doing things that allows companies to collaborate on infrastructure (~snowflakes~) and radically reduce Total Cost of Ownership, along with a vibrant and active [slack community](https://slack.cloudposse.com). - It provides a fully customizable framework for defining and building cloud infrastructures backed by [AWS](https://aws.amazon.com/) and powered by [kubernetes](https://kubernetes.io/). It couples best-of-breed technologies with engineering best-practices to equip organizations with the tooling that enables clusters to be spun up in record time without compromising security. + It provides a fully customizable framework for defining and building cloud infrastructures backed by [AWS](https://aws.amazon.com/) and powered by [kubernetes](https://kubernetes.io/). It couples best-of-breed technologies with engineering best-practices to equip organizations with the tooling that enables clusters to be spun up in record time without compromising security. It's works natively with Mac OSX, Linux, and [Windows 10 (WSL)](https://docs.microsoft.com/en-us/windows/wsl/install-win10). introduction: |- These days, the typical software application is distributed as a docker image and run as a container. Why should infrastructure be any different? Since everything we write is "Infrastructure as Code", we believe that it should be treated the same way. This is the "Geodesic Way". Use containers+envs instead of unconventional wrappers, complicated folder structures and symlink hacks. Geodesic is the container for all your infrastructure automation needs that enables you to truly achieve SweetOps. - + Geodesic is composed of two parts: 1. It is an interactive command-line shell. The shell includes the *ultimate* mashup of cloud orchestration tools. @@ -69,24 +69,29 @@ introduction: |- An organization may chose to leverage all of these components, or just the parts that make their life easier. We recommend starting by using `geodesic` as a Docker base image (e.g. `FROM cloudposse/geodesic:...` pinned to a release and base OS) in your projects. - **Note**: Starting with Geodesic version 0.138.0, we distribute 2 versions of Geodesic Docker images, one based on [Alpine](https://alpinelinux.org/) - and one based on [Debian](https://debian.org), tagged `VERSION-BASE_OS`, e.g. `0.138.0-alpine`. - Prior to this, all Docker images were based on Alpine only and simply tagged `VERSION`. At present, the Alpine version is the most thoroughly tested - and best supported version, and the special Docker tag `latest` will continue to point to the latest Alpine version while this - remains the case. However, we encourage people to use the Debian version and report any issues by opening a GitHub issue, - so that we may eventually make it the preferred version, after which point the `latest` tag will point to latest Debian image. We - will maintain the `latest-alpine` and `latest-debian` Docker tags for those who want to commit to using one base OS or - the other but still want automatic updates. - - **Note**: Geodesic is a large collection of tools. As such, support for the Apple M1 chip is not under - Cloud Posse's control, rather it depends on each tool author updating each tool for the M1 chip. - All of the compiled tools that Cloud Posse has authored and are included in Geodesic are compiled - for M1 (`darwin_arm64`), and of course all of the scripts work on M1 if the interpreters (e.g. - `bash`, `python`) are compiled for M1. Unfortunatetly, this is only a small portion of the overall - toolkit that is assembled in Geodesic. Therefore we do not advise using Geodesic on the M1 at this time - and do not anticipate M1 will be well supported before 2022. Historically, widespread support for a new - chip takes several years to establish; we hope we will not have to wait that long. - + **Note**: Starting with Geodesic 2.0, we distribute Geodesic as a multi-platform (`linux/amd64`, `linux/arm64`) Debian-based + Docker image and a single-platform (`linux/amd64`) Alpine-based image. We recommend using the Debian-based image, and the + `cloudposse/geodesic:latest` image now points to it. (Previously `cloudposse/geodesic:latest` was the Alpine image.) + We have deprioritized support for Alpine and may drop it entirely at some point. + + Starting with Geodesic version 0.138.0, we distributed 2 versions of `linux/arm64` Geodesic Docker images, + one based on [Alpine](https://alpinelinux.org/) and one based on [Debian](https://debian.org), tagged `VERSION-BASE_OS`, e.g. `0.138.0-alpine`. + Prior to this, all Docker images were based on Alpine only and simply tagged `VERSION`. Prior to the release of Geodesic version 1.0, + the Alpine version was the most thoroughly tested and best supported version, and the special Docker tag `latest` continued to point + to the latest Alpine version. + + **Note**: Geodesic is a large collection of tools. To run on an Apple natively under Geodesic, binaries need to be compiled + for `linux/arm64`, while to run on Apple natively on outside of Geodesic (on macOS) they need to be + compiled for `darwin/arm64`. As such, support for the Apple M1 (or later) chip is not + fully under Cloud Posse's control, rather it depends on each tool author updating each tool. + + To be included in Geodesic and for Geodesic to support both Intel and Apple CPUs, a tool project must + distribute both a `linux/amd64` and `linux/arm64` binary. (In exceptional cases, if a tool is written + in the `go` language and distributes source code only, Cloud Posse may build the needed binaries.) + As of Geodesic 2.0, all but a small number of tools have started releasing the necessary binaries, + so we removed the ones that were not available on `linux/arm64` in order to provide a consistent + toolkit on both platforms. (See the Geodesic 2.0 [Release Notes](https://github.com/cloudposse/geodesic/releases/tag/2.0.0) for details on which tools were removed.) + Want to learn more? [Check out our getting started with Geodesic guide!](https://docs.cloudposse.com/tutorials/geodesic-getting-started/) usage: |- @@ -94,39 +99,20 @@ usage: |- #### docker run - This will launch the latest version of the geodesic container - + Launching Gedoesic is a bit complex, so we recommend you install a launch script by running ``` - docker run -it --rm \ - --name=custom-image-name \ - --env LS_COLORS \ - --env TERM \ - --env TERM_COLOR \ - --env TERM_PROGRAM \ - --volume=$HOME:/localhost \ - --env LOCAL_HOME=$HOME \ - --privileged \ - --publish 37049:37049 \ - --env GEODESIC_PORT=37049 \ - --env DOCKER_IMAGE=cloudposse/geodesic \ - --env DOCKER_NAME=custom-image-name \ - --env DOCKER_TAG=latest \ - --env GEODESIC_HOST_CWD=$PWD cloudposse/geodesic:latest -l + docker run --rm cloudposse/geodesic:latest-debian init | bash ``` - - #### install a script - + After that, you should be able to launch Geodesic just by typing ``` - git clone git@github.com:cloudposse/geodesic.git - cd geodesic - make all -f Makefile.custom + geodesic ``` ### Customizing your Docker image In general we recommend creating a customized version of Geodesic by creating your own `Dockerfile` starting with ``` - ARG VERSION=0.138.0 + ARG VERSION=2.0.0 ARG OS=debian FROM cloudposse/geodesic:$VERSION-$OS @@ -138,11 +124,47 @@ usage: |- You can see some example configuration options to include in [Dockerfile.options](./Dockerfile.options). - You can also add extra commands by installing "packages". Both Alpine and Debian have a large selection + #### Multi-platform gotchas + + Although the Geodesic base image is provided in 2 architectures, when you do a local build + of your custom image, it will, by default, only be built for the architecture of the machine you are building on. + This is fine until you want to share it. You need to be aware that if you push just the image you + built with `docker build` you will only be supporting a single architecture. You should use `docker buildx` + to build a multi-platform image and push it to a Docker repository for sharing. + + If you intend to support both architectures, you need to be sure that any customizations + you install are properly installed for both architectures. Package managers handle this for you + automatically, but if you are downloading files directly, you need to be careful to select the right one. + See the use of `TARGETARCH` in [Dockerfile.debian](./os/debian/Dockerfile.debian) for some examples. + + #### Adding packages + + You can also add extra commands by installing "packages". Both Debian and Alpine have a large selection of packages to choose from. Cloud Posse also provides a large set of packages for installing common DevOps commands - and utilities via [cloudposse/packages](https://github.com/cloudposse/packages). - The package repositories are pre-installed in Geodesic, all you need to do is add the packages you want - via `RUN` commands in your Dockerfile. + and utilities via [cloudposse/packages](https://github.com/cloudposse/packages), but `linux/arm64` packages + are only provided for Debian, not Alpine. The package repositories are pre-installed in Geodesic, + all you need to do is add the packages you want via `RUN` commands in your Dockerfile. Debian + will automatically select the correct architecture for the package. + + #### Installing packages in Debian + + Debian uses [`apt`](https://wiki.debian.org/Apt) for package management and we generally recommend using + the [`apt-get`](https://www.debian.org/doc/manuals/apt-guide/ch2.en.html) command to install packages. + In addition to the default repositories, Geodesic pre-installs the Cloud Posse [package](https://github.com/cloudposse/packages) repository + and the Google Cloud SDK package repository. Unlike with `apk`, you do not need to specify a package repository when + installing a package because all repositories will be searched for it. + Also unlike `apk`, `apt-get` does not let you specify a version range on the command line, but they do + allow wildcards. Package versions include a release number (typically "1") at the end, to allow for + updated packages when there is a bug in the package itself. Therefore, best practice is to use a wildcard + for the release number when specifying a package version. For example, + to install the Google Cloud SDK at a version 300.0.0: + + ``` + RUN apt-get update && apt-get install -y google-cloud-sdk="300.0.0-*" + ``` + + Note the `-y` flag to `apt-get install`. That is required for scripted installation, otherwise the command + will ask for confirmation from the keyboard before installing a package. #### Installing packages in Alpine @@ -165,28 +187,10 @@ usage: |- RUN apk update && apk add -u teleport@cloudposse=~4.2 ``` - #### Installing packages in Debian - - Debian uses [`apt`](https://wiki.debian.org/Apt) for package management and we generally recommend using - the [`apt-get`](https://www.debian.org/doc/manuals/apt-guide/ch2.en.html) command to install packages. - In addition to the default repositories, Geodesic pre-installs the Cloud Posse [package](https://github.com/cloudposse/packages) repository - and the Google Cloud SDK package repository. Unlike with `apk`, you do not need to specify a package repository when - installing a package because all repositories will be searched for it. - Also unlike `apk`, `apt-get` does not let you specify a version range on the command line, only an exact version. - So to install the Google Cloud SDK at a specific version, you need to include a trailing `-0` to indicate - the package version. For example, to install version Google Cloud SDK 300.0.0: - - ``` - RUN apt-get update && apt-get install -y google-cloud-sdk=300.0.0-0 - ``` - - Note the `-y` flag to `apt-get install`. That is required for scripted installation, otherwise the command - will ask for confirmation from the keyboard before installing a package. - ### Customizing your shell at launch time - After you have build your Docker image, or if you are using a shared Docker image, you can - add further customization at launch time. When Geodesic stars up, it looks for customization + After you have built your Docker image, or if you are using a shared Docker image, you can + add further customization at launch time. When Geodesic starts up, it looks for customization scripts and configuration so you can do things like add command aliases or override preconfigured options. Detailed information about launch-time configuration is in the [customization](./docs/customization.md) document, available from within the shell via `man customization`. diff --git a/docs/kops.md b/docs/kops.md index 9cc0e8e41..becbdb414 100644 --- a/docs/kops.md +++ b/docs/kops.md @@ -7,10 +7,17 @@ date: May 2019 ## NAME -kops - Kubernetes Operations (kops) +kops - (Deprecated) Kubernetes Operations (kops) ## SYNOPSIS +***Deprecation notice*** CloudPosse no longer supports `kops` for managing Kubernetes clusters. +This document is retained for historical reference, but should not be used for new clusters. +Some binaries, such as `kops` itself, used to be shipped pre-installed in Geodesic, and +some, such as `direnv` and `tfenv` had special support as well. Some of these are no +longer installed, and some may not work as advertised due to bit rot. + +***Historical synopsis*** Kops is one of the easiest ways to get a production grade Kubernetes cluster up and running. The `kops` command line tool (cli) is like `kubectl` for clusters. It handles all the standard CRUD operations necessary to manage the complete life cycle of a cluster. It is possible to run any number of [kops clusters](http://github.com/kubernetes/kops) within an account. Our diff --git a/os/alpine/Dockerfile.alpine b/os/alpine/Dockerfile.alpine index a30bdba30..3b8904eab 100644 --- a/os/alpine/Dockerfile.alpine +++ b/os/alpine/Dockerfile.alpine @@ -1,7 +1,7 @@ # https://alpinelinux.org/ ARG ALPINE_VERSION=3.17.2 # https://cloud.google.com/sdk/docs/release-notes -ARG GOOGLE_CLOUD_SDK_VERSION=410.0.0 +ARG GOOGLE_CLOUD_SDK_VERSION=422.0.0 # https://github.com/ahmetb/kubectx/releases ARG KUBECTX_COMPLETION_VERSION=0.9.4 # https://github.com/jonmosco/kube-ps1/releases @@ -16,7 +16,7 @@ ARG HELM_DIFF_VERSION=3.6.0 # We had issues with helm-diff 3.1.3 + helm-git 0.9.0, # previous workaround was to pin helm-git to version 0.8.1. # We expect this has been fixed now with helm-diff 3.3.2 + helm-git 0.11.1 -ARG HELM_GIT_VERSION=0.14.0 +ARG HELM_GIT_VERSION=0.15.1 # # Python Dependencies diff --git a/os/debian/Dockerfile.debian b/os/debian/Dockerfile.debian index a2c0b633d..3159a3171 100644 --- a/os/debian/Dockerfile.debian +++ b/os/debian/Dockerfile.debian @@ -1,7 +1,10 @@ # https://www.debian.org/releases/ ARG DEBIAN_VERSION=11.6-slim +# Find the current version of Python at https://www.python.org/downloads/source/ +ARG PYTHON_VERSION=3.10.10 + # https://cloud.google.com/sdk/docs/release-notes -ARG GOOGLE_CLOUD_SDK_VERSION=410.0.0 +ARG GOOGLE_CLOUD_SDK_VERSION=422.0.0 # https://github.com/ahmetb/kubectx/releases ARG KUBECTX_COMPLETION_VERSION=0.9.4 # https://github.com/jonmosco/kube-ps1/releases @@ -16,12 +19,10 @@ ARG HELM_DIFF_VERSION=3.6.0 # We had issues with helm-diff 3.1.3 + helm-git 0.9.0, # previous workaround was to pin helm-git to version 0.8.1. # We expect this has been fixed now with helm-diff 3.3.2 + helm-git 0.11.1 -ARG HELM_GIT_VERSION=0.14.0 +ARG HELM_GIT_VERSION=0.15.1 -FROM debian:$DEBIAN_VERSION as python -# Find the current version of Python at https://www.python.org/downloads/source/ -ARG PYTHON_VERSION=3.10.8 +FROM python:$PYTHON_VERSION-slim-bullseye as python # Debian comes with minimal Locale support. See https://github.com/docker-library/docs/pull/703/files # Recommended: LC_ALL=C.UTF-8 @@ -30,39 +31,22 @@ ENV LC_ALL=C.UTF-8 # Once again, Debian is weird. Package-installed Python is system-only, and does not # play well with pip. See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=765022#30 # -# So we cannot install the packages: RUN apt-get update && apt-get install -y python3 python3-pip python3-dev -# and instead we have to build Python ourselves and install it in /usr/local/bin (by Debian design). - -# Also bad is the fact that python, being an interpreted language, installs a lot of shell scripts +# So we copy python from the Docker "Official Image" for python, (not to be confused with +# any official python image provided by python upstream). See https://hub.docker.com/_/python/ for details. +# +# Bad is the fact that python, being an interpreted language, installs a lot of shell scripts # that specifically reference the location of the python interpreter or other things, those # locations being baked in at install time. So it does not work to install python in `/dist` # and then later copy `/dist` to `/usr/local`: The installed scripts would have `!#/dist/bin/python` -# baked in and fail. So we take the somewhat unorthodox approach of deleting /usr/local entirely, -# then installing python in /usr/local, and then copying /usr/local over. +# baked in and fail. So the official image starts with a basically empty /usr/local/, +# installs python into /usr/local, and then we copy /usr/local over. # -RUN rm -rf /usr/local && mkdir /usr/local - -# Preload apt-utils -RUN apt-get update && apt-get install -y apt-utils - -# Install the packages that are needed to build python3 -RUN apt-get update && apt-get install -y build-essential zlib1g-dev libncurses5-dev libgdbm-dev \ - libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl libbz2-dev - -# Download the Python source code -RUN curl -1sLfO https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz && \ - tar -xf Python-${PYTHON_VERSION}.tar.xz - -# Build Python to be installed in /usr/local/bin -WORKDIR Python-${PYTHON_VERSION} -RUN ./configure --enable-optimizations --prefix=/usr/local && make -j $(nproc) && make install - -# Build the Python packages, configured to use /usr/local/bin/python3, and install them under /dist +# Build the Python packages, configured to use /usr/local/bin/python3, and install them under /usr/local COPY requirements.txt /requirements.txt RUN python3 -m pip install --upgrade --progress-bar off pip setuptools wheel && \ - pip install --progress-bar off -r /requirements.txt --ignore-installed --prefix=/usr/local --no-build-isolation + pip install --progress-bar off -r /requirements.txt --prefix=/usr/local --no-build-isolation # Remove Python cache files RUN find / -name __pycache__ -exec rm -rf {} \; -prune @@ -132,21 +116,6 @@ RUN apt-get update && apt-get install -y \ mkdir -p /etc/bash_completion.d/ /etc/profile.d/ /conf && \ touch /conf/.gitconfig -# -# Install Google Cloud SDK -# This is separate so that updating it does not invalidate the Docker cache layer with all the packages installed above -# https://cloud.google.com/sdk/docs/release-notes -ARG GOOGLE_CLOUD_SDK_VERSION -ENV CLOUDSDK_CONFIG=/localhost/.config/gcloud/ - -RUN apt-get update && apt-get install -y google-cloud-sdk=${GOOGLE_CLOUD_SDK_VERSION}-\* - -# gcloud config writes successful status updates to stderr, but we want to preserve -# stderr for real errors in need of action. -RUN { gcloud config set core/disable_usage_reporting true --installation && \ - gcloud config set component_manager/disable_update_check true --installation && \ - gcloud config set metrics/environment github_docker_image --installation; } 2>&1 - # Here is where we would confirm that the package repo checksum is what we expect (not mismatched due to Docker layer cache) # Using the en_US.UTF-8 local breaks our login setup because it changes the sort order, @@ -156,7 +125,8 @@ RUN { gcloud config set core/disable_usage_reporting true --installation && \ # "locales" package and run the following command: # RUN localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 -RUN echo "net.ipv6.conf.all.disable_ipv6=0" > /etc/sysctl.d/00-ipv6.conf +# Not sure why we were disabling IPv6, but we should not do that anymore, as IPv6 is stable and useful +# RUN echo "net.ipv6.conf.all.disable_ipv6=0" > /etc/sysctl.d/00-ipv6.conf # Disable vim from reading a swapfile (incompatible with goofys) RUN echo 'set noswapfile' >> /etc/vim/vimrc @@ -166,6 +136,21 @@ WORKDIR /tmp # Copy python dependencies COPY --from=python /usr/local/ /usr/local/ +# +# Install Google Cloud SDK (requires Python) +# This is separate so that updating it does not invalidate the Docker cache layer with all the packages installed above +# https://cloud.google.com/sdk/docs/release-notes +ARG GOOGLE_CLOUD_SDK_VERSION +ENV CLOUDSDK_CONFIG=/localhost/.config/gcloud/ + +RUN apt-get update && apt-get install -y google-cloud-sdk=${GOOGLE_CLOUD_SDK_VERSION}-\* + +# gcloud config writes successful status updates to stderr, but we want to preserve +# stderr for real errors in need of action. +RUN { gcloud config set core/disable_usage_reporting true --installation && \ + gcloud config set component_manager/disable_update_check true --installation && \ + gcloud config set metrics/environment github_docker_image --installation; } 2>&1 + # Explicitly set KUBECONFIG to enable kube_ps1 prompt ENV KUBECONFIG=/conf/.kube/config # Install an empty kubeconfig to suppress some warnings @@ -274,6 +259,9 @@ ENV MAKEFLAGS="--no-print-directory" COPY rootfs/ / COPY os/debian/rootfs/ / + +ARG TARGETARCH + # Move AWS CLI v1 aside and install AWS CLI v2 as default, leaving both available as alternatives. # We do this at the end because we need cache busting from above to get us the latest AWS CLI v2 @@ -285,7 +273,9 @@ RUN mv /usr/local/bin/aws /usr/local/bin/aws1 && \ # but it is updated several times a week, so we choose to just get the latest. # ARG AWS_CLI_VERSION=2.2.8 RUN AWSTMPDIR=$(mktemp -d -t aws-inst-XXXXXXXXXX) && \ - curl -sSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64${AWS_CLI_VERSION:+-${AWS_CLI_VERSION}}.zip" -o "$AWSTMPDIR/awscliv2.zip" && \ + if [ "$TARGETARCH" = "amd64" ]; then \ + AWS_ARCH=x86_64; else AWS_ARCH=aarch64; fi && \ + curl -sSL "https://awscli.amazonaws.com/awscli-exe-linux-${AWS_ARCH}${AWS_CLI_VERSION:+-${AWS_CLI_VERSION}}.zip" -o "$AWSTMPDIR/awscliv2.zip" && \ cd $AWSTMPDIR && \ unzip -qq awscliv2.zip && \ ./aws/install -i /usr/share/aws/v2 -b /usr/share/aws/v2/bin && \ @@ -306,7 +296,9 @@ ENV AWS_PAGER="less -FRX" # Install AWS Session Manager Plugin ARG SESSION_MANAGER_PLUGIN_VERSION -RUN curl -L "https://s3.amazonaws.com/session-manager-downloads/plugin/${SESSION_MANAGER_PLUGIN_VERSION}/ubuntu_64bit/session-manager-plugin.deb" \ +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + AWS_ARCH=64bit; else AWS_ARCH="$TARGETARCH"; fi && \ + curl -L "https://s3.amazonaws.com/session-manager-downloads/plugin/${SESSION_MANAGER_PLUGIN_VERSION}/ubuntu_${AWS_ARCH}/session-manager-plugin.deb" \ -o "/tmp/session-manager-plugin.deb" && \ sudo dpkg -i /tmp/session-manager-plugin.deb && \ rm /tmp/session-manager-plugin.deb diff --git a/packages.txt b/packages.txt index bedbba344..4803635c9 100644 --- a/packages.txt +++ b/packages.txt @@ -1,10 +1,10 @@ -# Essential alpine packages -awless@cloudposse +# Essential packages +# no arm64 awless@cloudposse aws-iam-authenticator@cloudposse bash bash-completion bats@community -cfssl@cloudposse +# no arm64 cfssl@cloudposse coreutils chamber@cloudposse curl @@ -20,7 +20,7 @@ fuse fzf@cloudposse gettext git -github-commenter@cloudposse +# no arm64 github-commenter@cloudposse gomplate@cloudposse goofys@cloudposse gosu@cloudposse @@ -29,7 +29,6 @@ helm@cloudposse helm3@cloudposse helmfile@cloudposse jq -kops@cloudposse # The latest version of kubectl is wrong for most of our customers, but the # version-specific packages are organized to automatically configure the latest # installed version as the default, meaning if we install a version and a user @@ -53,10 +52,10 @@ pandoc@cloudposse postgresql-client pwgen python3 -rakkess@cloudposse +# no arm64 rakkess@cloudposse rbac-lookup@cloudposse retry@cloudposse -scenery@cloudposse +# abandoned 2021-03-08 scenery@cloudposse shellcheck@cloudposse shfmt@cloudposse sops@cloudposse @@ -71,8 +70,8 @@ terraform@cloudposse # will cause problems for people who install one or the other terragrunt@cloudposse terrahelp@cloudposse -tfenv@cloudposse -tfmask@cloudposse +# no arm64 tfenv@cloudposse +# no arm64 tfmask@cloudposse unzip util-linux variant@cloudposse diff --git a/rootfs/etc/profile.d/tfenv.sh b/rootfs/etc/profile.d/tfenv.sh deleted file mode 100755 index 70aa56b12..000000000 --- a/rootfs/etc/profile.d/tfenv.sh +++ /dev/null @@ -1,12 +0,0 @@ -# Automatically export the current environment to `TF_VAR` -# Use a regex defined in the `TFENV_WHITELIST` and `TFENV_BLACKLIST` environment variables to include and exclude variables -# - -# Prevent tfenv from exporting sensitive environment variables -[[ -z "$TFENV_BLACKLIST" ]] && export TFENV_BLACKLIST="^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_SECURITY_TOKEN|AWS_SESSION_TOKEN|ATLANTIS_.*|GITHUB_.*)$" - -if [[ ${TFENV_LOAD_GLOBALLY:-false} == "true" ]]; then - echo $(red Global use of) tfenv $(red is not recommended) && - echo $(red use the) '"use tfenv"' $(red function of) '"direnv"' $(red to automatically execute it in certain directories) && - source <(tfenv) -fi diff --git a/rootfs/usr/local/bin/init-terraform b/rootfs/usr/local/bin/init-terraform deleted file mode 100755 index c0adb01c9..000000000 --- a/rootfs/usr/local/bin/init-terraform +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# This script sets up a Terraform backend for a given project using envs - -echo "DEPRECATED: $0 is no longer needed. Use \`tfenv\` instead." - -# Settings have moved to `direnvrc` -source /etc/direnv/direnvrc - -# When using globs, return an empty string when no files found -shopt -s nullglob -TF_FILES=(*.tf *.tfvars) -shopt -u nullglob - -# Exit on all errors -set -e - -# Do not attempt to initialize non-terraform projects -if [ -z "${TF_FILES}" ] && [ -z "${TF_FROM_MODULE}" ]; then - echo "Run $0 from a terraform project directory" - exit 1 -fi - -# Mount all filesystems (E.g. `goofys` S3 filesystem) -[ $UID -ne 0 ] || mount -a - -use_terraform - -source <(tfenv) - -# Initialize terraform state -terraform init