Merge remote-tracking branch 'upstream/master' into HEAD

.dockerignore

@@ -1,3 +1,4 @@
 Dockerfile*
 build
 .*.swp
+docker/Dockerfile*

.github/actions/build-container/action.yml

@@ -0,0 +1,52 @@
+name: "Build/Push container"
+description: "Build a BCC CI container and push it when not a pull-request."
+
+inputs:
+  os_distro:
+    description: "OS Disctribution. Ex: ubuntu"
+    required: true
+  os_version:
+    description: "Version of the OS. Ex: 20.04"
+    required: true
+  os_nick:
+    description: "Nickname of the OS. Ex: focal"
+    required: true
+  llvm_versions:
+    description: "Space separated list of llvm versions to install in the container. Only supported for Ubuntu containers."
+    type: string
+    default: "12"
+  registry:
+    description: "Registry where to push images"
+    default: ghcr.io
+  password:
+    description: "Password used to log into the docker registry."
+  push:
+    description: "Whether or not to push the build image"
+    type: boolean
+    default: false
+
+runs:
+  using: "composite"
+  steps:
+    # Login against registry except on PR
+    # https://github.com/docker/login-action
+    - name: Log into registry ${{ inputs.registry }}
+      if: ${{ inputs.push == 'true' && github.event_name != 'pull_request' }}
+
+      uses: docker/login-action@v2
+      with:
+        registry: ${{ inputs.registry }}
+        username: ${{ github.actor }}
+        password: ${{ inputs.password }}
+
+    - name: Build and push
+      uses: docker/build-push-action@v3
+      with:
+        push: ${{ inputs.push == 'true' && github.event_name != 'pull_request' }}
+        build-args: |
+          VERSION=${{ inputs.os_version }}
+          SHORTNAME=${{ inputs.os_nick }}
+          LLVM_VERSION=${{ inputs.llvm_versions }}
+        file: docker/build/Dockerfile.${{ inputs.os_distro }}
+        tags: ${{ inputs.registry }}/${{ github.repository }}:${{ inputs.os_distro }}-${{ inputs.os_version }}
+

.github/workflows/bcc-test.yml

@@ -1,29 +1,66 @@
 name: BCC Build and tests
 
-on: push
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+permissions:
+  contents: read # to fetch code (actions/checkout)
+  pull-requests: read # to read pull requests (dorny/paths-filter)
 
 jobs:
   test_bcc:
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
-        os: [ubuntu-16.04, ubuntu-18.04] # 16.04.4 release has 4.15 kernel
-                                         # 18.04.3 release has 5.0.0 kernel
+        os: [{distro: "ubuntu", version: "20.04", nick: focal}]
+        llvm_version: [12, 15, 17]
         env:
         - TYPE: Debug
           PYTHON_TEST_LOGFILE: critical.log
+          RW_ENGINE_ENABLED: ON
+        - TYPE: Debug
+          PYTHON_TEST_LOGFILE: critical.log
+          RW_ENGINE_ENABLED: OFF
         - TYPE: Release
           PYTHON_TEST_LOGFILE: critical.log
+          RW_ENGINE_ENABLED: ON
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
+    - uses: dorny/paths-filter@v3
+      id: changes
+      with:
+        filters: |
+          docker:
+            - 'docker/build/**'
     - name: System info
       run: |
         uname -a
         ip addr
-    - name: Build docker container with all deps
+    - name: Pull docker container
+      if: steps.changes.outputs.docker == 'false'
       run: |
-        docker build -t bcc-docker -f Dockerfile.tests .
-    - name: Run bcc build
+        docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.os.distro }}-${{ matrix.os.version }}
+    - name: Build docker container
+      if: steps.changes.outputs.docker == 'true'
+      uses: ./.github/actions/build-container
+      with:
+        os_distro: ${{ matrix.os.distro }}
+        os_version: ${{ matrix.os.version }}
+        os_nick: ${{ matrix.os.nick }}
+        llvm_versions: ${{ matrix.llvm_version }}
+    - name: Tag docker container
+      run: |
+        docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.os.distro }}-${{ matrix.os.version }} bcc-docker
+    - name: Run bcc build - llvm-${{ matrix.llvm_version }}
       env: ${{ matrix.env }}
       run: |
         /bin/bash -c \
@@ -37,7 +74,7 @@ jobs:
                    bcc-docker \
                    /bin/bash -c \
                    'mkdir -p /bcc/build && cd /bcc/build && \
-                    cmake -DCMAKE_BUILD_TYPE=${TYPE} .. && make -j9'"
+                    LLVM_ROOT=/usr/lib/llvm-${{ matrix.llvm_version }} cmake -DCMAKE_BUILD_TYPE=${TYPE} -DENABLE_LLVM_NATIVECODEGEN=${RW_ENGINE_ENABLED} .. && make -j9'"
     - name: Run bcc's cc tests
       env: ${{ matrix.env }}
       # tests are wrapped with `script` as a hack to get a TTY as github actions doesn't provide this
@@ -83,10 +120,123 @@ jobs:
         echo "There were $critical_count critical tests skipped with @mayFail:"
         grep -A2 @mayFail tests/python/critical.log
 
-    - uses: actions/upload-artifact@v1
+    - uses: actions/upload-artifact@v4
       with:
-        name: critical-tests-${{ matrix.env['TYPE'] }}-${{ matrix.os }}
+        name: critical-tests-${{ matrix.env['TYPE'] }}-${{ matrix.os.version }}
         path: tests/python/critical.log
+        overwrite: true
+
+  test_bcc_fedora:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        os: [{distro: "fedora", version: "38", nick: "f38"}]
+        env:
+        - TYPE: Debug
+          PYTHON_TEST_LOGFILE: critical.log
+        - TYPE: Release
+          PYTHON_TEST_LOGFILE: critical.log
+    steps:
+    - uses: actions/checkout@v4
+    - uses: dorny/paths-filter@v3
+      id: changes
+      with:
+        filters: |
+          docker:
+            - 'docker/build/**'
+    - name: System info
+      run: |
+        uname -a
+        ip addr
+    - name: Pull docker container
+      if: steps.changes.outputs.docker == 'false'
+      run: |
+        docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.os.distro }}-${{ matrix.os.version }}
+    - name: Build docker container
+      if: steps.changes.outputs.docker == 'true'
+      uses: ./.github/actions/build-container
+      with:
+        os_distro: ${{ matrix.os.distro }}
+        os_version: ${{ matrix.os.version }}
+        os_nick: ${{ matrix.os.nick }}
+    - name: Tag docker container
+      run: |
+        docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.os.distro }}-${{ matrix.os.version }} bcc-docker
+    - name: Run bcc build
+      env: ${{ matrix.env }}
+      run: |
+        /bin/bash -c \
+                   "docker run --privileged \
+                   --pid=host \
+                   -v $(pwd):/bcc \
+                   -v /sys/kernel/debug:/sys/kernel/debug:rw \
+                   -v /lib/modules:/lib/modules:ro \
+                   -v /usr/src:/usr/src:ro \
+                   -v /usr/include/linux:/usr/include/linux:ro \
+                   bcc-docker \
+                   /bin/bash -c \
+                   'mkdir -p /bcc/build && cd /bcc/build && \
+                    cmake -DCMAKE_BUILD_TYPE=${TYPE} -DENABLE_LLVM_SHARED=ON -DRUN_LUA_TESTS=OFF .. && make -j9'"
+    - name: Run libbpf-tools build
+      env: ${{ matrix.env }}
+      run: |
+        /bin/bash -c \
+                   "docker run --privileged \
+                   --pid=host \
+                   -v $(pwd):/bcc \
+                   -v /sys/kernel/debug:/sys/kernel/debug:rw \
+                   -v /lib/modules:/lib/modules:ro \
+                   -v /usr/src:/usr/src:ro \
+                   -v /usr/include/linux:/usr/include/linux:ro \
+                   bcc-docker \
+                   /bin/bash -c \
+                   'cd /bcc/libbpf-tools && make -j9'"
+
+    - name: Run bcc's cc tests
+      env: ${{ matrix.env }}
+      # tests are wrapped with `script` as a hack to get a TTY as github actions doesn't provide this
+      # see https://github.com/actions/runner/issues/241
+      run: |
+        script -e -c /bin/bash -c \
+        "docker run -ti \
+                    --privileged \
+                    --network=host \
+                    --pid=host \
+                    -v $(pwd):/bcc \
+                    -v /sys/kernel/debug:/sys/kernel/debug:rw \
+                    -v /lib/modules:/lib/modules:ro \
+                    -v /usr/src:/usr/src:ro \
+                    -e CTEST_OUTPUT_ON_FAILURE=1 \
+                    bcc-docker \
+                    /bin/bash -c \
+                    '/bcc/build/tests/wrapper.sh \
+                        c_test_all sudo /bcc/build/tests/cc/test_libbcc'"
+
+    - name: Run all tests
+      env: ${{ matrix.env }}
+      run: |
+        script -e -c /bin/bash -c \
+        "docker run -ti \
+                    --privileged \
+                    --network=host \
+                    --pid=host \
+                    -v $(pwd):/bcc \
+                    -v /sys/kernel/debug:/sys/kernel/debug:rw \
+                    -v /lib/modules:/lib/modules:ro \
+                    -v /usr/src:/usr/src:ro \
+                    -e CTEST_OUTPUT_ON_FAILURE=1 \
+                    bcc-docker \
+                    /bin/bash -c \
+                    'cd /bcc/build && \
+                     make test PYTHON_TEST_LOGFILE=$PYTHON_TEST_LOGFILE ARGS=-V'"
+
+    - name: Check critical tests
+      env: ${{ matrix.env }}
+      run: |
+        critical_count=$(grep @mayFail tests/python/critical.log | wc -l)
+        echo "There were $critical_count critical tests skipped with @mayFail:"
+        grep -A2 @mayFail tests/python/critical.log
+
 
 # To debug weird issues, you can add this step to be able to SSH to the test environment
 #     https://github.com/marketplace/actions/debugging-with-tmate

.github/workflows/publish-build-containers.yml

@@ -0,0 +1,43 @@
+name: Publish Build Containers
+
+on:
+  # We want to update this image regularly and when updating master
+  schedule:
+    - cron: '00 18 * * *'
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - 'docker/build/**'
+
+permissions: {}
+
+jobs:
+
+  publish_ghcr:
+    permissions:
+      contents: read # to fetch code (actions/checkout)
+      packages: write # to push container
+    name: Publish To GitHub Container Registry
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [
+          {distro: "ubuntu", version: "20.04", nick: focal, installed_llvm_versions: "12 15 17"},
+          {distro: "fedora", version: "38", nick: "f38", installed_llvm_versions: "this is not used"},
+        ]
+
+    steps:
+
+    - uses: actions/checkout@v4
+
+    - name: Build and push
+      uses: ./.github/actions/build-container
+      with:
+        os_distro: ${{ matrix.os.distro }}
+        os_version: ${{ matrix.os.version }}
+        os_nick: ${{ matrix.os.nick }}
+        llvm_versions: ${{ matrix.os.installed_llvm_versions }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+        push: true

.github/workflows/publish.yml

@@ -0,0 +1,54 @@
+name: Publish Build Artifacts
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
+jobs:
+  # Optionally publish container images to custom docker repository,
+  # guarded by presence of all required github secrets.
+  # GitHub secrets can be configured as follows:
+  #   - DOCKER_IMAGE = docker.io/myorg/bcc
+  #   - DOCKER_USERNAME = username
+  #   - DOCKER_PASSWORD = password
+  publish_dockerhub:
+    name: Publish To Dockerhub
+    runs-on: ubuntu-latest
+    steps:
+
+    - uses: actions/checkout@v1
+
+    - name: Initialize workflow variables
+      id: vars
+      shell: bash
+      run: |
+          if [ -n "${DOCKER_IMAGE}" ] && \
+             [ -n "${DOCKER_USERNAME}" ] && \
+             [ -n "${DOCKER_PASSWORD}" ];then
+            echo "Custom docker credentials set, will push an image"
+            echo ::set-output name=DOCKER_PUBLISH::true
+          else
+            echo "Custom docker credentials not, skipping"
+          fi
+      env:
+        DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }}
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+
+    - name: Build container image and publish to registry
+      id: publish-registry
+      uses: elgohr/Publish-Docker-Github-Action@v5
+      if: ${{ steps.vars.outputs.DOCKER_PUBLISH }}
+      with:
+        name: ${{ secrets.DOCKER_IMAGE }}
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        workdir: .
+        dockerfile: Dockerfile.ubuntu
+        snapshot: true
+        cache: ${{ github.event_name != 'schedule' }}

.gitignore

@@ -12,3 +12,10 @@ debian/**/*.log
 *critical.log
 obj-x86_64-linux-gnu
 examples/cgroupid/cgroupid
+
+# Output from docker builds
+scripts/docker/output/
+/output/
+
+# UAPI header generated for libbpf package-based builds
+src/cc/compat/linux/bpf.h

.gitmodules

@@ -1,3 +1,9 @@
 [submodule "src/cc/libbpf"]
 	path = src/cc/libbpf
 	url = https://github.com/libbpf/libbpf.git
+[submodule "libbpf-tools/bpftool"]
+	path = libbpf-tools/bpftool
+	url = https://github.com/libbpf/bpftool
+[submodule "libbpf-tools/blazesym"]
+	path = libbpf-tools/blazesym
+	url = https://github.com/libbpf/blazesym