ROX-30714: use cgroup id for container id resolution

This approach is based on using the cgroup id of processes to resolve
the container ID in userspace. In order to achieve this, a LRU BPF map
is used to keep a mapping of cgroup id to cgroup "path", send the cgroup
id as part of file events and parsing the container id by directly
accessing the map from userspace. The LRU map is filled from userspace
at startup and then kept up to date by a BPF program on the
cgroup_attach_task tracepoint, inserting new cgroup ids as we find them.

Author: Molter73

fact-ebpf/src/bpf/builtins.h

@@ -3,7 +3,6 @@
 
 #include "file.h"
 #include "types.h"
-#include "process.h"
 #include "maps.h"
 #include "events.h"
 #include "bound_path.h"
@@ -103,3 +102,38 @@ int BPF_PROG(trace_path_unlink, struct path* dir, struct dentry* dentry) {
   m->path_unlink.error++;
   return 0;
 }
+
+SEC("tp_btf/cgroup_attach_task")
+int BPF_PROG(trace_cgroup_attach_task, struct cgroup* dst_cgrp, const char* path, struct task_struct* _task, bool _threadgroup) {
+  struct metrics_t* m = get_metrics();
+  if (m == NULL) {
+    bpf_printk("Failed to get metrics entry");
+    return 0;
+  }
+
+  m->cgroup_attach_task.total++;
+
+  u64 id = dst_cgrp->kn->id;
+  if (bpf_map_lookup_elem(&cgroup_map, &id) != NULL) {
+    // Already have the entry
+    m->cgroup_attach_task.ignored++;
+    return 0;
+  }
+
+  struct helper_t* helper = get_helper();
+  if (helper == NULL) {
+    bpf_printk("Failed to get helper entry");
+    m->cgroup_attach_task.error++;
+    return 0;
+  }
+
+  bpf_core_read_str(helper->cgroup_entry.path, PATH_MAX, path);
+  helper->cgroup_entry.parsed = false;
+  int res = bpf_map_update_elem(&cgroup_map, &id, &helper->cgroup_entry, BPF_NOEXIST);
+  if (res != 0) {
+    bpf_printk("Failed to update path for %d", id);
+    m->cgroup_attach_task.error++;
+  }
+
+  return 0;
+}

fact-ebpf/src/bpf/main.c

@@ -11,8 +11,10 @@
  * Helper struct with buffers for various operations
  */
 struct helper_t {
-  char buf[PATH_MAX * 2];
-  const unsigned char* array[16];
+  union {
+    cgroup_entry_t cgroup_entry;
+    char buf[PATH_MAX * 2];
+  };
 };
 
 struct {
@@ -104,6 +106,13 @@ __always_inline static struct metrics_t* get_metrics() {
   return bpf_map_lookup_elem(&metrics, &zero);
 }
 
+struct {
+  __uint(type, BPF_MAP_TYPE_LRU_HASH);
+  __type(key, __u64);
+  __type(value, cgroup_entry_t);
+  __uint(max_entries, (2^16)-1);
+} cgroup_map SEC(".maps");
+
 uint64_t host_mount_ns;
 volatile const bool path_unlink_supports_bpf_d_path;
 

fact-ebpf/src/bpf/maps.h

@@ -11,72 +11,6 @@
 #include <bpf/bpf_core_read.h>
 // clang-format on
 
-__always_inline static const char* get_memory_cgroup(struct helper_t* helper) {
-  if (!bpf_core_enum_value_exists(enum cgroup_subsys_id, memory_cgrp_id)) {
-    return NULL;
-  }
-
-  struct task_struct* task = (struct task_struct*)bpf_get_current_task();
-
-  // We're guessing which cgroup controllers are enabled for this task. The
-  // assumption is that memory controller is present more often than
-  // cpu & cpuacct.
-  struct kernfs_node* kn = BPF_CORE_READ(task, cgroups, subsys[memory_cgrp_id], cgroup, kn);
-  if (kn == NULL) {
-    return NULL;
-  }
-
-  int i = 0;
-  for (; i < 16; i++) {
-    helper->array[i] = (const unsigned char*)BPF_CORE_READ(kn, name);
-    if (bpf_core_field_exists(kn->__parent)) {
-      kn = BPF_CORE_READ(kn, __parent);
-    } else {
-      struct kernfs_node___pre6_15 {
-        struct kernfs_node* parent;
-      };
-      struct kernfs_node___pre6_15* kn_old = (void*)kn;
-      kn = BPF_CORE_READ(kn_old, parent);
-    }
-    if (kn == NULL) {
-      break;
-    }
-  }
-
-  if (i == 16) {
-    i--;
-  }
-
-  int offset = 0;
-  for (; i >= 0 && offset < PATH_MAX; i--) {
-    // Skip empty directories
-    if (helper->array[i] == NULL) {
-      continue;
-    }
-
-    helper->buf[offset & (PATH_MAX - 1)] = '/';
-    if (++offset >= PATH_MAX) {
-      return NULL;
-    }
-
-    int len = bpf_probe_read_kernel_str(&helper->buf[offset & (PATH_MAX - 1)], PATH_MAX, helper->array[i]);
-    if (len < 0) {
-      // We should have skipped all empty entries, any other error is a genuine
-      // problem, stop processing.
-      return NULL;
-    }
-
-    if (len == 1) {
-      offset--;
-      continue;
-    }
-
-    offset += len - 1;
-  }
-
-  return helper->buf;
-}
-
 __always_inline static void process_fill_lineage(process_t* p, struct helper_t* helper, bool use_bpf_d_path) {
   struct task_struct* task = (struct task_struct*)bpf_get_current_task_btf();
   p->lineage_len = 0;
@@ -109,6 +43,7 @@ __always_inline static int64_t process_fill(process_t* p, bool use_bpf_d_path) {
   p->gid = (uid_gid >> 32) & 0xFFFFFFFF;
   p->login_uid = task->loginuid.val;
   p->pid = (bpf_get_current_pid_tgid() >> 32) & 0xFFFFFFFF;
+  p->cgroup_id = bpf_get_current_cgroup_id();
   u_int64_t err = bpf_get_current_comm(p->comm, TASK_COMM_LEN);
   if (err != 0) {
     bpf_printk("Failed to fill task comm");
@@ -133,11 +68,6 @@ __always_inline static int64_t process_fill(process_t* p, bool use_bpf_d_path) {
 
   d_path(&task->mm->exe_file->f_path, p->exe_path, PATH_MAX, use_bpf_d_path);
 
-  const char* cg = get_memory_cgroup(helper);
-  if (cg != NULL) {
-    bpf_probe_read_str(p->memory_cgroup, PATH_MAX, cg);
-  }
-
   p->in_root_mount_ns = get_mount_ns() == host_mount_ns;
 
   process_fill_lineage(p, helper, use_bpf_d_path);

fact-ebpf/src/bpf/process.h

@@ -22,7 +22,7 @@ typedef struct process_t {
   char args[4096];
   unsigned int args_len;
   char exe_path[PATH_MAX];
-  char memory_cgroup[PATH_MAX];
+  unsigned long long cgroup_id;
   unsigned int uid;
   unsigned int gid;
   unsigned int login_uid;
@@ -73,4 +73,10 @@ struct metrics_by_hook_t {
 struct metrics_t {
   struct metrics_by_hook_t file_open;
   struct metrics_by_hook_t path_unlink;
+  struct metrics_by_hook_t cgroup_attach_task;
 };
+
+typedef struct cgroup_entry_t {
+  char parsed;
+  char path[PATH_MAX];
+} cgroup_entry_t;

fact-ebpf/src/bpf/types.h

@@ -81,11 +81,13 @@ impl metrics_t {
         let mut m = metrics_t { ..*self };
         m.file_open = m.file_open.accumulate(&other.file_open);
         m.path_unlink = m.path_unlink.accumulate(&other.path_unlink);
+        m.cgroup_attach_task = m.cgroup_attach_task.accumulate(&other.cgroup_attach_task);
         m
     }
 }
 
 unsafe impl Pod for metrics_t {}
+unsafe impl Pod for cgroup_entry_t {}
 
 pub const EBPF_OBJ: &[u8] = aya::include_bytes_aligned!(concat!(env!("OUT_DIR"), "/main.o"));
 pub const CHECKS_OBJ: &[u8] = aya::include_bytes_aligned!(concat!(env!("OUT_DIR"), "/checks.o"));

fact-ebpf/src/lib.rs

@@ -3,7 +3,7 @@ use std::{io, path::PathBuf, sync::Arc};
 use anyhow::{bail, Context};
 use aya::{
     maps::{Array, LpmTrie, MapData, PerCpuArray, RingBuf},
-    programs::Lsm,
+    programs::Program,
     Btf, Ebpf,
 };
 use checks::Checks;
@@ -15,9 +15,13 @@ use tokio::{
     task::JoinHandle,
 };
 
-use crate::{event::Event, host_info, metrics::EventCounter};
+use crate::{
+    event::{self, Event},
+    host_info,
+    metrics::EventCounter,
+};
 
-use fact_ebpf::{event_t, metrics_t, path_prefix_t, LPM_SIZE_MAX};
+use fact_ebpf::{cgroup_entry_t, event_t, metrics_t, path_prefix_t, LPM_SIZE_MAX};
 
 mod checks;
 
@@ -30,6 +34,8 @@ pub struct Bpf {
 
     paths: Vec<path_prefix_t>,
     paths_config: watch::Receiver<Vec<PathBuf>>,
+
+    event_parser: event::parser::Parser,
 }
 
 impl Bpf {
@@ -44,7 +50,7 @@ impl Bpf {
 
         // Include the BPF object as raw bytes at compile-time and load it
         // at runtime.
-        let obj = aya::EbpfLoader::new()
+        let mut obj = aya::EbpfLoader::new()
             .set_global("host_mount_ns", &host_info::get_host_mount_ns(), true)
             .set_global(
                 "path_unlink_supports_bpf_d_path",
@@ -56,11 +62,17 @@ impl Bpf {
 
         let paths = Vec::new();
         let (tx, _) = broadcast::channel(100);
+        let Some(cgroup_map) = obj.take_map("cgroup_map") else {
+            bail!("Failed to get cgroup_map");
+        };
+        let cgroup_map: aya::maps::HashMap<MapData, u64, cgroup_entry_t> = cgroup_map.try_into()?;
+        let event_parser = event::parser::Parser::new(cgroup_map);
         let mut bpf = Bpf {
             obj,
             tx,
             paths,
             paths_config,
+            event_parser,
         };
 
         bpf.load_paths()?;
@@ -138,24 +150,42 @@ impl Bpf {
         Ok(())
     }
 
-    fn load_lsm_prog(&mut self, name: &str, hook: &str, btf: &Btf) -> anyhow::Result<()> {
+    fn load_prog(&mut self, name: &str, hook: &str, btf: &Btf) -> anyhow::Result<()> {
         let Some(prog) = self.obj.program_mut(name) else {
             bail!("{name} program not found");
         };
-        let prog: &mut Lsm = prog.try_into()?;
-        prog.load(hook, btf)?;
+        match prog {
+            Program::Lsm(prog) => prog.load(hook, btf)?,
+            Program::BtfTracePoint(prog) => prog.load(hook, btf)?,
+            _ => todo!(),
+        }
         Ok(())
     }
 
     fn load_progs(&mut self, btf: &Btf) -> anyhow::Result<()> {
-        self.load_lsm_prog("trace_file_open", "file_open", btf)?;
-        self.load_lsm_prog("trace_path_unlink", "path_unlink", btf)
+        let progs = [
+            ("trace_file_open", "file_open"),
+            ("trace_path_unlink", "path_unlink"),
+            ("trace_cgroup_attach_task", "cgroup_attach_task"),
+        ];
+
+        for (name, hook) in progs {
+            self.load_prog(name, hook, btf)?;
+        }
+        Ok(())
     }
 
     fn attach_progs(&mut self) -> anyhow::Result<()> {
         for (_, prog) in self.obj.programs_mut() {
-            let prog: &mut Lsm = prog.try_into()?;
-            prog.attach()?;
+            match prog {
+                Program::Lsm(prog) => {
+                    prog.attach()?;
+                }
+                Program::BtfTracePoint(prog) => {
+                    prog.attach()?;
+                }
+                _ => todo!(),
+            }
         }
         Ok(())
     }
@@ -165,8 +195,10 @@ impl Bpf {
         mut self,
         mut running: watch::Receiver<bool>,
         event_counter: EventCounter,
+        parser_counter: EventCounter,
     ) -> JoinHandle<anyhow::Result<()>> {
         info!("Starting BPF worker...");
+        self.event_parser.set_metrics(parser_counter);
 
         tokio::spawn(async move {
             self.attach_progs()
@@ -183,7 +215,7 @@ impl Bpf {
                         let ringbuf = guard.get_inner_mut();
                         while let Some(event) = ringbuf.next() {
                             let event: &event_t = unsafe { &*(event.as_ptr() as *const _) };
-                            let event = match Event::try_from(event) {
+                            let event = match self.event_parser.parse(event) {
                                 Ok(event) => Arc::new(event),
                                 Err(e) => {
                                     error!("Failed to parse event: '{e}'");
@@ -268,7 +300,11 @@ mod bpf_tests {
             // Create a metrics exporter, but don't start it
             let exporter = Exporter::new(bpf.take_metrics().unwrap());
 
-            let handle = bpf.start(run_rx, exporter.metrics.bpf_worker.clone());
+            let handle = bpf.start(
+                run_rx,
+                exporter.metrics.bpf_worker.clone(),
+                exporter.metrics.event_parser.clone(),
+            );
 
             tokio::time::sleep(Duration::from_millis(500)).await;
 
@@ -277,7 +313,7 @@ mod bpf_tests {
                 NamedTempFile::new_in(monitored_path).expect("Failed to create temporary file");
             println!("Created {file:?}");
 
-            let expected = Event::new(
+            let expected = Event::from_raw_parts(
                 file_activity_type_t::FILE_ACTIVITY_CREATION,
                 host_info::get_hostname(),
                 file.path().to_path_buf(),