use pkg-builder to build packages

.gitattributes

@@ -0,0 +1,2 @@
+*.sss linguist-language=toml
+*.sps linguist-language=toml

.github/workflows/debian-12-packaging.yml

@@ -0,0 +1,129 @@
+name: Debian 12 packaging
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main     
+    paths:
+      - 'debian-12/**'  
+
+jobs:
+
+  eth-node-besu:
+    strategy:
+      matrix:
+        package-name: ["eth-node-besu"]
+        package-version-with-revision: ["23.10.1-1"]
+        codename: ["debian-12"]
+      fail-fast: false
+    uses: ./.github/workflows/pkgbuilder-deps.yml
+    with:
+      package-name: ${{ matrix.package-name }}
+      package-version-with-revision: ${{ matrix.package-version-with-revision }}
+      codename:  ${{ matrix.codename }}
+
+  eth-node-erigon:
+    strategy:
+      matrix:
+        package-name: ["eth-node-erigon"]
+        package-version-with-revision: ["2.53.2-1"]
+        codename: ["debian-12"]
+      fail-fast: false
+    uses: ./.github/workflows/pkgbuilder-deps.yml
+    with:
+      package-name: ${{ matrix.package-name }}
+      package-version-with-revision: ${{ matrix.package-version-with-revision }}
+      codename:  ${{ matrix.codename }}
+
+  eth-node-geth:
+    strategy:
+      matrix:
+        package-name: ["eth-node-geth"]
+        package-version-with-revision: ["1.13.4-1"]
+        codename: ["debian-12"]
+      fail-fast: false
+    uses: ./.github/workflows/pkgbuilder-deps.yml
+    with:
+      package-name: ${{ matrix.package-name }}
+      package-version-with-revision: ${{ matrix.package-version-with-revision }}
+      codename:  ${{ matrix.codename }}
+
+  eth-node-lighthouse:
+    strategy:
+      matrix:
+        package-name: ["eth-node-lighthouse"]
+        package-version-with-revision: ["4.5.0-1"]
+        codename: ["debian-12"]
+    uses: ./.github/workflows/pkgbuilder-deps.yml
+    with:
+      package-name: ${{ matrix.package-name }}
+      package-version-with-revision: ${{ matrix.package-version-with-revision }}
+      codename:  ${{ matrix.codename }}    
+
+  eth-node-lodestar:
+    strategy:
+      matrix:
+        package-name: ["eth-node-lodestar"]
+        package-version-with-revision: ["1.11.3-1"]
+        codename: ["debian-12"]
+      fail-fast: false
+    uses: ./.github/workflows/pkgbuilder-deps.yml
+    with:
+        package-name: ${{ matrix.package-name }}
+        package-version-with-revision: ${{ matrix.package-version-with-revision }}
+        codename:  ${{ matrix.codename }}   
+
+  eth-node-nethermind:
+    strategy:
+      matrix:
+        package-name: ["eth-node-nethermind"]
+        package-version-with-revision: ["1.21.1-1"]
+        codename: ["debian-12"]
+      fail-fast: false
+    uses: ./.github/workflows/pkgbuilder-deps.yml
+    with:
+      package-name: ${{ matrix.package-name }}
+      package-version-with-revision: ${{ matrix.package-version-with-revision }}
+      codename:  ${{ matrix.codename }}      
+
+  eth-node-nimbus-eth2:
+    strategy:
+      matrix:
+        package-name: ["eth-node-nimbus-eth2"]
+        package-version-with-revision: ["24.3.0-1"]
+        codename: ["debian-12"]
+      fail-fast: false 
+    uses: ./.github/workflows/pkgbuilder-deps.yml
+    with:
+      package-name: ${{ matrix.package-name }}
+      package-version-with-revision: ${{ matrix.package-version-with-revision }}
+      codename:  ${{ matrix.codename }}    
+
+  eth-node-prysm:
+    strategy:
+      matrix:
+        package-name: ["eth-node-prysm"]
+        package-version-with-revision: ["4.1.1-1"]
+        codename: ["debian-12"]
+      fail-fast: false
+    uses: ./.github/workflows/pkgbuilder-deps.yml
+    with:
+      package-name: ${{ matrix.package-name }}
+      package-version-with-revision: ${{ matrix.package-version-with-revision }}
+      codename:  ${{ matrix.codename }}
+
+  eth-node-teku:
+    strategy:
+      matrix:
+        package-name: ["eth-node-teku"]
+        package-version-with-revision: ["23.10.0-1"]
+        codename: ["debian-12"]
+      fail-fast: false
+    uses: ./.github/workflows/pkgbuilder-deps.yml
+    with:
+      package-name: ${{ matrix.package-name }}
+      package-version-with-revision: ${{ matrix.package-version-with-revision }}
+      codename:  ${{ matrix.codename }}    

.github/workflows/pkgbuilder-deps.yml

@@ -0,0 +1,126 @@
+name: pkbuilder-deps
+
+on:
+  workflow_call:
+    inputs:
+      package-name:
+        required: true
+        type: string
+      package-version-with-revision:
+          required: true
+          type: string  
+      codename:
+          required: true
+          type: string      
+
+jobs:
+  package:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Sbuild setup
+      run: |
+        sudo apt-get update
+        # Note this is an older version of sbuild, no need to patch it, yet
+        sudo apt install -y debhelper schroot ubuntu-dev-tools piuparts
+        sudo apt-get -y install pkg-config libssl-dev uidmap
+        sudo apt-get install -y libfilesys-df-perl libmime-lite-perl
+        sudo apt-get install -y autopkgtest vmdb2 qemu-system-x86
+        wget -q https://github.com/eth-pkg/sbuild-ubuntu/releases/download/0.85-6-1/sbuild_0.85.6_all.deb
+        wget -q https://github.com/eth-pkg/sbuild-ubuntu/releases/download/0.85-6-1/libsbuild-perl_0.85.6_all.deb
+        sudo dpkg -i sbuild_0.85.6_all.deb libsbuild-perl_0.85.6_all.deb || true
+
+
+    # - name: pkg-builder
+    #   run: |
+    #     wget -q https://github.com/eth-pkg/pkg-builder/releases/download/v0.1/pkg-builder
+    #     wget -q https://github.com/eth-pkg/pkg-builder/releases/download/v0.1/debcrafter
+    #     mkdir -p ${HOME}/.local/bin 
+    #     mv pkg-builder ${HOME}/.local/bin
+    #     mv debcrafter ${HOME}/.local/bin 
+    #     chmod +x ${HOME}/.local/bin/pkg-builder
+    #     chmod +x ${HOME}/.local/bin/debcrafter
+    #     echo "${HOME}/.local/bin" >> $GITHUB_PATH
+
+    - name: Checkout dependency repository
+      uses: actions/checkout@v4
+      with:
+        repository: eenagy/pkg-builder  
+        ref: main
+        path: pkg-builder
+
+    - name: Install pkg-builder
+      run: |
+        cd pkg-builder
+        cargo build --release
+        mkdir -p ${HOME}/.local/bin 
+        mv target/release/pkg-builder ${HOME}/.local/bin
+        chmod +x ${HOME}/.local/bin/pkg-builder
+        echo "${HOME}/.local/bin" >> $GITHUB_PATH
+        cd .. && rm -rf pkg-builder
+
+    - name: Install debcrafter
+      run: |
+        wget -q https://github.com/eth-pkg/pkg-builder/releases/download/v0.1/debcrafter
+        mkdir -p ${HOME}/.local/bin 
+        mv debcrafter ${HOME}/.local/bin
+        chmod +x ${HOME}/.local/bin/debcrafter
+        echo "${HOME}/.local/bin" >> $GITHUB_PATH
+
+    - name: Checkout repository
+      uses: actions/checkout@v4    
+
+    - name: Create chroot env
+      run: |
+        pkg-builder env create ${{ inputs.codename }}/x86_64/${{inputs.package-name}}/${{ inputs.package-version-with-revision }}/pkg-builder.toml
+        echo "${HOME}/.cache/sbuild/bookworm-amd64.tar.gz" >> $GITHUB_PATH
+
+    - name: package
+      run: |
+        pkg-builder package --run-piuparts false --run-autopkgtests false --run-lintian false ${{ inputs.codename }}/x86_64/${{inputs.package-name}}/${{ inputs.package-version-with-revision }}/pkg-builder.toml 
+
+
+    - name: lintian
+      run: pkg-builder lintian ${{ inputs.codename }}/x86_64/${{inputs.package-name}}/${{ inputs.package-version-with-revision }}/pkg-builder.toml
+
+
+    - name: piuparts
+      run: |
+        # Install debian-archive-keyring but fails, as it is already installed
+        # sudo apt-get install -y debian-archive-keyring
+        ${HOME}/.local/bin/pkg-builder piuparts ${{ inputs.codename }}/x86_64/${{inputs.package-name}}/${{ inputs.package-version-with-revision }}/pkg-builder.toml
+
+    - name: autopkgtests
+      run: |
+        # Copy files under root
+        sudo cp -R ${HOME}/.pkg-builder /root
+        sudo ${HOME}/.local/bin/pkg-builder autopkgtest ${{ inputs.codename }}/x86_64/${{inputs.package-name}}/${{ inputs.package-version-with-revision }}/pkg-builder.toml
+
+    - name: verify
+      run: |
+        ${HOME}/.local/bin/pkg-builder verify --config-file ${{ inputs.codename }}/x86_64/${{inputs.package-name}}/${{ inputs.package-version-with-revision }}/pkg-builder.toml  --verify-config-file ${{ inputs.codename }}/x86_64/${{inputs.package-name}}/${{ inputs.package-version-with-revision }}/pkg-builder-verify.toml  --no-package true # do not need to repackage it, as we run it multiple times on CI, and hashes should come from locally built sources
+
+    - name: upload cleanup 
+      run: | 
+        version=$(echo ${{inputs.package-version-with-revision}} | cut -d'-' -f1)
+        rm -rf /home/runner/.pkg-builder/packages/${{inputs.package-name}}/${{inputs.package-name}}-$version
+        rm -rf /home/runner/.pkg-builder/packages/${{inputs.package-name}}/${{inputs.package-name}}
+        # copy pkg-builder configs to artifacts
+        cp ${{ inputs.codename }}/x86_64/${{inputs.package-name}}/${{ inputs.package-version-with-revision }}/pkg-builder.toml /home/runner/.pkg-builder/packages/${{inputs.package-name}}
+        cp ${{ inputs.codename }}/x86_64/${{inputs.package-name}}/${{ inputs.package-version-with-revision }}/pkg-builder-verify.toml /home/runner/.pkg-builder/packages/${{inputs.package-name}}
+        cd /home/runner/.pkg-builder/packages/${{inputs.package-name}}
+        rm *:*
+        rm *dbgsym* || true # possible there is none
+
+
+          # This should come after verify, but sometimes it is good to have it, to check why the verify fails
+    - name: Save results
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{inputs.package-name}}-${{ inputs.codename }}-${{ inputs.package-version-with-revision }}
+        path: /home/runner/.pkg-builder/packages/${{inputs.package-name}}  
+
+
+
+
+
+

README.md

@@ -1,52 +1,44 @@
-# Ethereum Node Packaging
+## Ethereum Node Packaging
 
-This project is designed to simplify the process of packaging different Ethereum nodes for Debian-based systems. It provides a structured way to create Debian packages for multiple Ethereum clients. 
+This initiative streamlines the process of packaging diverse Ethereum nodes for Debian-based systems. It offers a systematic approach to generate Debian packages for various Ethereum clients.
 
-## Usage
+### Usage
 
-### Build debian packages
+#### Prerequisites
 
-#### Create sbuild env 
+Begin by installing `pkg-builder`. Refer to the README for installation instructions: [https://github.com/eth-pkg/pkg-builder](https://github.com/eth-pkg/pkg-builder)
 
-Sbuild env
-```bash
-bash build-systems/v1/create-chroot-no-deps.sh bookworm amd64
-```
-Sbuild env for java/besu
-```bash
-bash build-systems/v1/create-chroot-oracle-jdk.sh  bookworm amd64
-```
+#### Building Debian Packages
 
-Sbuild env for dotnet/nethermind
-```bash
-bash build-systems/v1/create-chroot-dotnet.sh  bookworm amd64
-```
+**Building Debian Packages**:
 
-Sbuild env for node.js/lodestar
 ```bash
-bash build-systems/v1/create-chroot-node.sh  bookworm amd64
+# Change into the directoy which you want to package
+cd debian-12/x86_64/eth-node-nimbus-eth2/24.3.0-1
+# Create an environment once for each distribution/architecture pairing
+pkg-builder env create pkg-builder-verify.toml
+
+# This command installs dependencies, downloads source code, builds the client, and conducts tests against it
+# Note: Autopkgtests and piuparts may require elevated permissions
+pkg-builder package pkg-builder.toml
+
+# To verify a successful build without running tests, you can use this command
+pkg-builder verify --config-file pkg-builder.toml --verfiy-config-file pkg-builder-verify.toml 
 ```
 
-Sbuild env for lighthouse/docker
+If you're not building but simply ensuring the built client matches a specific hash:
+
 ```bash
-bash build-systems/v1/create-chroot-docker.sh  bookworm amd64
-```
+debian-12/x86_64/eth-node-nimbus-eth2/24.3.0-1
+# Create an environment once for each distribution/architecture pairing
+pkg-builder env create pkg-builder-verify.toml
 
-**Build the Debian Packages**: Build Debian packages for each client using the generated variables.
-   ```bash
-   make build PACKAGE=eth-node VERSION=0.1-1
-   make build PACKAGE=eth-node-besu VERSION=23.10.1-1
-   make build PACKAGE=eth-node-erigon VERSION=2.53.2-1
-   make build PACKAGE=eth-node-geth VERSION=1.13.4-1
-   make build PACKAGE=eth-node-lighthouse VERSION=4.5.0-1
-   make build PACKAGE=eth-node-lodestar VERSION=1.11.3-1
-   make build PACKAGE=eth-node-nethermind VERSION=1.21.1-1
-   make build PACKAGE=eth-node-nimbus-eth2 VERSION=23.10.0-1
-   make build PACKAGE=eth-node-prysm VERSION=4.1.1-1
-   make build PACKAGE=eth-node-service VERSION=0.1-1
-   make build PACKAGE=eth-node-teku VERSION=23.10.0-1
+# Verify the successful build without running tests
+pkg-builder verify --config-file pkg-builder.toml --verfiy-config-file pkg-builder-verify.toml 
+```
 
-   ```
-  This will generate the appropiate .deb files for each package.
+### How It Works
 
+This process leverages `debcrafter` and `pkg-builder` to establish reproducible environments. Debcrafter aids in creating reproducible Debian directories based on detailed specification files ending with `.sss` and `.sps`. Meanwhile, pkg-builder utilizes debcrafter, and extends it to setup minimal environments to build and adheres to Debian's best practices, including `sbuild`, `piuparts`, `lintian`, and `autopkgtest`, to build the packages and test them thoroughly, ensuring they are not merely packages but functional ones.
 
+Currently, a significant obstacle in Debian packaging is the requirement for a separate git repository per package, which might hinder the support for numerous applications. Despite Debian packaging already facilitating reproducible builds, this aspect is still in its infancy. This project aims to adhere to distribution best practices to the fullest extent possible, only deviating when necessary or when certain support structures are not yet in place.