Add processes_events and sockets_events tables on macOS.

The former uses Endpoint Security, the latter a Network Extension.

These remain experimental for now.

.clang-tidy

@@ -23,6 +23,7 @@ Checks: 'bugprone-*,
          -misc-suspicious-semicolon,
          -misc-unused-parameters,
          -misc-const-correctnes,
+         -misc-use-anonymous-namespace
          -modernize-avoid-c-arrays,
          -modernize-use-equals-default,
          -modernize-use-nodiscard,
@@ -40,6 +41,7 @@ Checks: 'bugprone-*,
          -readability-function-size,
          -readability-isolate-declaration,
          -readability-identifier-length,
+         -readability-static-definition-in-anonymous-namespace,÷
          '
 
 HeaderFilterRegex: '/src'

.github/workflows/main.yml

@@ -51,7 +51,7 @@ jobs:
 
       - name: Check code
         run: |
-          pre-commit run -a --show-diff-on-failure
+          SKIP=autogen-readme pre-commit run -a --show-diff-on-failure
           # TODO: tidy fails in Broker currently
           # ninja -C build tidy
 

README.md

@@ -301,6 +301,30 @@ the endpoint at the time of the query.
 | `stime` | interval | system CPU time |
 </details>
 
+<details>
+<summary><tt>processes_events:</tt> process activity [macOS]</summary><br />
+
+The table reports processes starting and stopping on the endpoint.
+
+| Column | Type | Description
+| --- | --- | --- |
+| `time` | time | timestamp |
+| `name` | text | name of process |
+| `pid` | count | process ID |
+| `ppid` | count | parent's process ID |
+| `uid` | count | effective user ID |
+| `gid` | count | effective group ID |
+| `ruid` | count | real user ID |
+| `rgid` | count | real group ID |
+| `priority` | text | process priority (representation is platform-specific) |
+| `duration` | interval | interval since started |
+| `vsize` | count | virtual memory size |
+| `rsize` | count | resident memory size |
+| `utime` | interval | user CPU time |
+| `stime` | interval | system CPU time |
+| `state` | text | state of process |
+</details>
+
 <details>
 <summary><tt>sockets:</tt> open network sockets [Linux, Windows, macOS]</summary><br />
 
@@ -320,6 +344,27 @@ the endpoint at the time of the query.
 | `state` | text | state of socket |
 </details>
 
+<details>
+<summary><tt>sockets_events:</tt> open network sockets [macOS]</summary><br />
+
+The table reports IP sockets opening and closing on the endpoint.
+
+| Column | Type | Description
+| --- | --- | --- |
+| `time` | time | timestamp |
+| `pid` | count | ID of process holding socket |
+| `process` | text | name of process holding socket |
+| `uid` | count | user ID of process |
+| `gid` | count | group ID of process |
+| `family` | text | `IPv4` or `IPv6` |
+| `protocol` | count | transport protocol |
+| `local_addr` | address | local IP address |
+| `local_port` | count | local port number |
+| `remote_addr` | address | remote IP address |
+| `remote_port` | count | remote port number |
+| `state` | text | state of socket |
+</details>
+
 <details>
 <summary><tt>system_logs_events:</tt> log messages recorded by the operating systems [Linux, Windows, macOS]</summary><br />
 

src/platform/darwin/CMakeLists.txt

@@ -4,5 +4,6 @@ target_sources(zeek-agent PRIVATE platform.mm endpoint-security.mm network-exten
 target_link_libraries(zeek-agent PRIVATE "EndpointSecurity")
 target_link_libraries(zeek-agent PRIVATE "-framework NetworkExtension")
 target_link_libraries(zeek-agent PRIVATE "-framework Cocoa")
+target_link_libraries(zeek-agent PRIVATE "bsm")
 
 add_subdirectory(ZeekAgent.app)

src/platform/darwin/network-extension.h

@@ -2,31 +2,97 @@
 
 #pragma once
 
+#include "core/table.h"
+#include "platform/darwin/platform.h"
 #include "util/pimpl.h"
 #include "util/result.h"
 
+#include <list>
+#include <memory>
+#include <string>
+#include <utility>
+
 namespace zeek::agent::platform::darwin {
 
+class NetworkExtension;
+
+namespace ne {
+
+struct Flow {
+    pid_t pid;
+    ProcessInfo process;
+
+    Value local_addr;
+    Value local_port;
+    Value remote_addr;
+    Value remote_port;
+    Value protocol;
+    Value family;
+    Value state;
+};
+
+/**
+ * State for an active subscription to NetworkExtension events. An instance of
+ * this class is creating when subscribing to NE events, and the subscription
+ * is kept active as long as the instance exists.
+ */
+class Subscriber {
+public:
+    using Callback = std::function<void(const Flow& flow)>;
+
+    ~Subscriber();
+
+private:
+    friend class darwin::NetworkExtension;
+    Subscriber(NetworkExtension* ne, std::string tag, Callback cb)
+        : _ne(ne), _tag(std::move(tag)), _callback(std::move(cb)) {}
+
+    NetworkExtension* _ne;
+    std::string _tag;
+    Callback _callback;
+};
+
+} // namespace ne
+
 /**
  * Wrapper around macOS's Network Extension API. This encapsulates API state
  * across multiple clients, maintaining just a single internal copy.
  */
 class NetworkExtension : public Pimpl<NetworkExtension> {
 public:
+    using Callback = ne::Subscriber::Callback;
+
     ~NetworkExtension();
 
     /**
      * Returns success if the Network Extension has been initialized
      * successfully, or an error otherwise.
      *
-     * @returns success or an appropiate error message if NE isn't available;
-     * then no functionlity must be used
+     * @returns success or an appropriate error message if NE isn't
+     * available; then no functionality must be used
      */
     Result<Nothing> isAvailable();
 
+    /**
+     * Creates a subscription to EndpointSecurity events.
+     *
+     * @param tag descriptive tag identifying the subscription in log messages
+     *
+     * @param callback callback to invoke when an event is received
+     *
+     * @returns a subscription object that keeps the subscription active as long as it exists
+     */
+    std::unique_ptr<ne::Subscriber> subscribe(std::string tag, Callback callback);
+
+    void newFlow(const ne::Flow& flow);
+
 private:
+    friend ne::Subscriber;
     friend NetworkExtension* networkExtension();
+
     NetworkExtension();
+
+    std::list<ne::Subscriber*> _subscribers;
 };
 
 /** Returns the global `NetworkExtension` singleton. */

src/platform/darwin/network-extension.mm

@@ -9,26 +9,49 @@
 #include "core/logger.h"
 
 #include <NetworkExtension/NetworkExtension.h>
+#include <bsm/libbsm.h>
 
 using namespace zeek::agent;
 using namespace zeek::agent::platform::darwin;
 
+ne::Subscriber::~Subscriber() { _ne->_subscribers.remove(this); }
+
 template<>
 struct Pimpl<NetworkExtension>::Implementation {
     bool running = false;
 };
 
 Result<Nothing> NetworkExtension::isAvailable() {
-    if ( pimpl()->running )
+    if ( pimpl()->running ) {
+        ZEEK_AGENT_DEBUG("darwin", "[NetworkExtension] available");
         return Nothing();
-    else
-        return result::Error("network extension  not running");
+    }
+    else {
+        ZEEK_AGENT_DEBUG("darwin", "[NetworkExtension] not available");
+        return result::Error("network extension not running");
+    }
 }
 
 NetworkExtension::NetworkExtension() {}
 
 NetworkExtension::~NetworkExtension() {}
 
+std::unique_ptr<ne::Subscriber> NetworkExtension::subscribe(std::string tag, Callback callback) {
+    ZEEK_AGENT_DEBUG("NetwokExtension", frmt("new subscriber: %s", tag));
+    auto subscriber = std::unique_ptr<ne::Subscriber>(new ne::Subscriber(this, std::move(tag), std::move(callback)));
+    _subscribers.push_back(subscriber.get());
+    return std::move(subscriber);
+}
+
+void NetworkExtension::newFlow(const ne::Flow& flow) {
+    ZEEK_AGENT_DEBUG("darwin",
+                     frmt("[NetworkExtension] new flow: {}/{} -> {}/{}", to_string(flow.local_addr),
+                          to_string(flow.local_port), to_string(flow.remote_addr), to_string(flow.remote_port)));
+
+    for ( const auto& s : _subscribers )
+        s->_callback(flow);
+}
+
 NetworkExtension* platform::darwin::networkExtension() {
     static auto ne = std::unique_ptr<NetworkExtension>{};
 
@@ -43,6 +66,7 @@ @interface FilterDataProvider : NEFilterDataProvider
 
 @implementation FilterDataProvider
 - (void)startFilterWithCompletionHandler:(void (^)(NSError* error))completionHandler {
+    ZEEK_AGENT_DEBUG("darwin", "[NetworkExtension] starting");
     networkExtension()->pimpl()->running = true;
 
     // Create a filter that matches all traffic and let it all pass through our
@@ -69,13 +93,41 @@ - (void)startFilterWithCompletionHandler:(void (^)(NSError* error))completionHan
 }
 
 - (void)stopFilterWithReason:(NEProviderStopReason)reason completionHandler:(void (^)(void))completionHandler {
+    ZEEK_AGENT_DEBUG("darwin", "[NetworkExtension] stopping");
     networkExtension()->pimpl()->running = false;
     completionHandler();
 }
 
 - (NEFilterNewFlowVerdict*)handleNewFlow:(NEFilterFlow*)flow {
-    // TODO: Report new flow to subscribers here.
-    // logger()->info("new flow");
+    ZEEK_AGENT_DEBUG("darwin", "[NetworkExtension] got flow");
+
+    if ( [flow isKindOfClass:[NEFilterSocketFlow class]] ) {
+        auto sf = (NEFilterSocketFlow*)flow;
+        auto local = (NWHostEndpoint*)sf.localEndpoint;
+        auto remote = (NWHostEndpoint*)sf.remoteEndpoint;
+        auto token = reinterpret_cast<const audit_token_t*>(sf.sourceAppAuditToken.bytes);
+
+        ne::Flow nf;
+        nf.pid = audit_token_to_pid(*token);
+
+        if ( auto p = platform::darwin::getProcessInfo(nf.pid) )
+            nf.process = *p;
+
+        nf.local_addr = [local.hostname UTF8String];
+        nf.local_port = std::stoi([local.port UTF8String]);
+        nf.remote_addr = [remote.hostname UTF8String];
+        nf.remote_port = std::stoi([remote.port UTF8String]);
+        nf.protocol = static_cast<int64_t>(sf.socketProtocol);
+        nf.state = Value(); // TODO: Can we get this?
+
+        switch ( sf.socketFamily ) {
+            case PF_INET: nf.family = "IPv4"; break;
+            case PF_INET6: nf.family = "IPv6"; break;
+        }
+
+        networkExtension()->newFlow(nf);
+    }
+
     return [NEFilterNewFlowVerdict allowVerdict];
 }
 

src/platform/darwin/platform.h

@@ -2,7 +2,10 @@
 
 #pragma once
 
+#include "core/table.h"
+
 #include <optional>
+#include <vector>
 
 #include <util/filesystem.h>
 
@@ -14,4 +17,37 @@ namespace zeek::agent::platform::darwin {
  */
 extern std::optional<filesystem::path> getApplicationSupport();
 
+struct ProcessInfo {
+    Value name;
+    Value pid;
+    Value ppid;
+    Value uid;
+    Value gid;
+    Value ruid;
+    Value rgid;
+    Value priority;
+    Value startup;
+    Value vsize;
+    Value rsize;
+    Value utime;
+    Value stime;
+};
+
+/**
+ * Retrieves a list of all currently running processes
+ *
+ * @return a list of the PIDs of all processes, or an error if the list cannot
+ * be obtained
+ */
+Result<std::vector<pid_t>> getProcesses();
+
+/**
+ * Given a process ID, returns information about the process.
+ *
+ * @param pid the process ID to retrieve information for @return information
+ * about the process, or an error if the information cannot be obtained; even
+ * if successful, the struct may have been filled only partially
+ */
+Result<ProcessInfo> getProcessInfo(pid_t pid);
+
 } // namespace zeek::agent::platform::darwin