Merge remote-tracking branch 'origin/topic/robin/linux-ebpf'

* origin/topic/robin/linux-ebpf:
  Bump IXWebSocket.
  Update README.
  Add BPF-based `processes_events` table on Linux.
  Add BPF-based `sockets_events` table on Linux.
  Add central manager for handling BPF programs.
  Integrate bpftool.
  Add function retrieve Linux kernel version.

.clang-tidy

@@ -9,38 +9,42 @@ Checks: 'bugprone-*,
          portability-*,
          readability-*,
 
+         -bugprone-assignment-in-if-condition,
          -bugprone-easily-swappable-parameters,
+         -bugprone-reserved-identifier,
          -bugprone-unchecked-optional-access,
          -cert-err58-cpp,
          -clang-analyzer-cplusplus.NewDeleteLeaks,
          -clang-diagnostic-c++2a-designator,
          -clang-diagnostic-deprecated-copy,
          -clang-diagnostic-range-loop-analysis,
          -concurrency-mt-unsafe,
+         -misc-const-correctnes,
          -misc-macro-parentheses,
-         -misc-non-private-member-variables-in-classes,
          -misc-no-recursion,
+         -misc-non-private-member-variables-in-classes,
          -misc-suspicious-semicolon,
          -misc-unused-parameters,
-         -misc-const-correctnes,
          -misc-use-anonymous-namespace
          -modernize-avoid-c-arrays,
+         -modernize-macro-to-enum,
          -modernize-use-equals-default,
          -modernize-use-nodiscard,
          -modernize-use-trailing-return-type,
+         -performance-no-int-to-ptr,
          -readability-braces-around-statements,
          -readability-container-size-empty,
          -readability-convert-member-functions-to-static,
          -readability-else-after-return,
+         -readability-function-cognitive-complexity,
+         -readability-function-size,
+         -readability-identifier-length,
          -readability-implicit-bool-conversion,
+         -readability-isolate-declaration,
          -readability-magic-numbers,
          -readability-make-member-function-const,
          -readability-named-parameter,
          -readability-qualified-auto,
-         -readability-function-cognitive-complexity,
-         -readability-function-size,
-         -readability-isolate-declaration,
-         -readability-identifier-length,
          -readability-static-definition-in-anonymous-namespace,÷
          '
 

.github/workflows/main.yml

@@ -3,8 +3,8 @@ name: CI pipeline
 on: push
 
 jobs:
-  debug_ubuntu_latest:
-    runs-on: ubuntu-latest
+  debug_ubuntu_22:
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
@@ -14,16 +14,27 @@ jobs:
       ZEEK_VERSION: ${{ matrix.zeek.version }}
       ZEEK_TAG: ${{ matrix.zeek.tag }}
       ZEEK_AGENT_CONFIGURE_ADDL: ${{ matrix.configure }}
+      LD_LIBRARY_PATH: /usr/lib/llvm-17/lib/clang/17/lib/linux
 
     steps:
       - name: Prepare
         run: |
           export DEBIAN_FRONTEND=noninteractive
           sudo apt-get update
-          sudo apt-get install -y ninja-build ccache curl ca-certificates clang-tidy-12
-          sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-12 100
+          sudo apt-get install -y ninja-build ccache curl ca-certificates
           sudo pip3 install btest zkg pre-commit
 
+          # LLVM toolchain
+          echo 'deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main' | sudo tee -a /etc/apt/sources.list.d/llvm17.list
+          echo 'deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main' | sudo tee -a /etc/apt/sources.list.d/llvm17.list
+          sudo curl https://apt.llvm.org/llvm-snapshot.gpg.key -o /etc/apt/trusted.gpg.d/llvm.asc
+          sudo apt-get update
+          sudo apt-get install -y llvm-17-dev clang-17 libclang-17-dev clang-format-17 clang-tidy-17 libclang-rt-17-dev
+
+          # for bpftool
+          sudo ln -s $(which llvm-strip-17) /usr/local/bin/llvm-strip
+          sudo apt-get install libelf-dev gcc-multilib
+
       - name: Install Zeek
         run: |
           (cd /tmp && curl -L -O https://download.zeek.org/binary-packages/xUbuntu_22.04/amd64/zeek${ZEEK_TAG}-core_${ZEEK_VERSION}_amd64.deb)
@@ -51,7 +62,7 @@ jobs:
 
       - name: Check code
         run: |
-          SKIP=autogen-readme pre-commit run -a --show-diff-on-failure
+          pre-commit run -a --show-diff-on-failure
           # TODO: tidy fails in Broker currently
           # ninja -C build tidy
 
@@ -64,19 +75,22 @@ jobs:
             zeek-agent/tests/.tmp
 
   # TODO: Install Zeek and run Zeek tests.
-  release_alpine_3_15_static:
-    runs-on: ubuntu-latest
+  release_alpine_3_19_static:
+    runs-on: ubuntu-22.04
     environment: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/topic/ci-release-test' || startsWith(github.ref, 'refs/tags/v')) && 'release' || '' }}
 
     container:
-      image: alpine:3.15
+      image: alpine:3.19
 
     steps:
       - name: Prepare
         run: |
           apk update
-          apk add linux-headers ccache cmake g++ gcc git make ninja tar zlib-static zlib-dev openssl-libs-static openssl-dev python3 py3-pip bash
-          pip3 install btest zkg
+          apk add linux-headers ccache cmake g++ gcc git make ninja tar zlib-static zlib-dev openssl-libs-static openssl-dev zstd-static python3 py3-pip bash
+          pip3 install --break-system-packages btest zkg
+
+          # for bpftool
+          apk add clang llvm libelf elfutils-dev
 
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -317,7 +331,7 @@ jobs:
           path: build/dist/${{env.ZA_DIST}}
 
   release_source:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     environment: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/topic/ci-release-test' || startsWith(github.ref, 'refs/tags/v')) && 'release' || '' }}
 
     steps:
@@ -327,6 +341,9 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y ninja-build ccache curl ca-certificates
 
+          # for bpftool
+          sudo apt-get install llvm libelf-dev gcc-multilib
+
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
@@ -360,9 +377,9 @@ jobs:
   publish_release:
     permissions:
       contents: write
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     if: (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev'))
-    needs: [debug_ubuntu_latest, release_alpine_3_15_static, release_macos_13, release_windows_2022, release_source]
+    needs: [debug_ubuntu_22, release_alpine_3_19_static, release_macos_13, release_windows_2022, release_source]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

.gitmodules

@@ -49,3 +49,6 @@
 [submodule "3rdparty/glob"]
 	path = 3rdparty/glob
 	url = https://github.com/p-ranav/glob.git
+[submodule "3rdparty/bpftool"]
+	path = 3rdparty/bpftool
+	url = https://github.com/libbpf/bpftool

3rdparty/CMakeLists.txt

@@ -89,10 +89,56 @@ if ( HAVE_LINUX )
     set(BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE)
     add_subdirectory(pfs EXCLUDE_FROM_ALL)
     target_include_directories(pfs PUBLIC pfs/include)
+    target_compile_definitions(pfs PRIVATE preadv64=preadv) # work-around preadv64 not being defined on Alpine 3.19.
 endif()
 
 if ( NOT HAVE_GETOPT_LONG )
     add_library(3rdparty STATIC bsd-getopt-long.c)
     target_include_directories(pathfind PUBLIC .)
     add_library(3rdparty:3rdparty ALIAS 3rdparty)
 endif()
+
+# On Linux, build libbpf.
+if ( HAVE_LINUX )
+    include(ExternalProject)
+
+    find_program (llvm_strip llvm-strip REQUIRED) # needed by bpftools, ensure we have it
+    find_library(elf NAMES elf REQUIRED)
+    find_library(zstd NAMES zstd)
+
+    set(bpftool_source "${CMAKE_CURRENT_SOURCE_DIR}/bpftool")
+    set(bpftool_build "${CMAKE_CURRENT_BINARY_DIR}/bpftool")
+    set(libbpf_lib "${bpftool_build}/bootstrap/libbpf")
+
+    set(libbpf_include "${bpftool_build}/libbpf/include")
+    file(MAKE_DIRECTORY ${libbpf_include})
+
+    ExternalProject_Add(bpftool
+        SOURCE_DIR "${bpftool_source}"
+        BINARY_DIR "${bpftool_build}"
+        BUILD_COMMAND make -C ${bpftool_source}/src OUTPUT=${bpftool_build}/
+        DOWNLOAD_COMMAND ""
+        CONFIGURE_COMMAND ""
+        INSTALL_COMMAND make -C ${bpftool_source}/src OUTPUT=${bpftool_build}/ DESTDIR=${CMAKE_BINARY_DIR}/ prefix= install-bin
+        BUILD_BYPRODUCTS ${libbpf_lib}/libbpf.a
+        LOG_BUILD ON
+        LOG_INSTALL ON
+        LOG_OUTPUT_ON_FAILURE ON
+    )
+
+    # bpftools already comes with libbpf, and it conviniently builds a static
+    # library as well. We'll just use that instead of re-building our own.
+    add_library(bpf STATIC IMPORTED GLOBAL)
+    set_target_properties(bpf PROPERTIES IMPORTED_LOCATION "${libbpf_lib}/libbpf.a")
+    target_include_directories(bpf INTERFACE "${libbpf_include}")
+    target_link_libraries(bpf INTERFACE ${elf})
+    if ( zstd )
+        target_link_libraries(bpf INTERFACE ${zstd})
+    endif ()
+    add_dependencies(bpf bpftool)
+
+    set(BPFTOOL "${CMAKE_BINARY_DIR}/sbin/bpftool" PARENT_SCOPE)
+    set(BPF_INCLUDE_DIR "${libbpf_include}" PARENT_SCOPE)
+endif ()
+
+

CHANGES

@@ -1,3 +1,21 @@
+2.3.0-dev.90 | 2024-01-12 14:49:01 +0100
+
+  * Linux: Integrate `bpftools` and add central manager for handling
+    BPF programs. We now require `clang` for building the agent on
+    Linux.
+
+  * Linux: Add BPF-based `processes_events` and `sockets_events`
+    tables. These remain experimental for now and may require further
+    fine tuning.
+
+  * Linux: Add function retrieve kernel version.
+
+  * Add Dependabot configuration.
+
+  * Update README.
+
+  * Bump IXWebSocket.
+
 2.3.0-dev.80 | 2024-01-09 09:15:06 +0100
 
   * Add `processes_events` and `sockets_events` tables on macOS. The

CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2021 by the Zeek Project. See LICENSE for details.
+# Copyright (c) 2021-2024 by the Zeek Project. See LICENSE for details.
 
 cmake_minimum_required(VERSION 3.15.1)
 
@@ -152,6 +152,8 @@ message(
     "\nInstall prefix:        ${CMAKE_INSTALL_PREFIX}"
     "\n"
     "\nHost system:           ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_VERSION} (${CMAKE_SYSTEM_PROCESSOR})"
+    "\nC Compiler:            ${CMAKE_C_COMPILER} (${CMAKE_C_COMPILER_VERSION})"
+    "\nC++ Compiler:          ${CMAKE_CXX_COMPILER} (${CMAKE_CXX_COMPILER_VERSION})"
     "\n"
     "\nBuild with tests:      ${USE_DOCTEST}"
     "\nBuild with Broker:     ${USE_BROKER}"