From 2622fd2718700c25e2f2abec4d4494573dfd2bb3 Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Wed, 25 Mar 2026 04:39:07 +0000 Subject: [PATCH 01/21] pickle error: --- .../test_runner/basic_sanity_tests.py | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/tests/L2_testing/test_runner/basic_sanity_tests.py b/tests/L2_testing/test_runner/basic_sanity_tests.py index f98e60fe9..d107a57d6 100755 --- a/tests/L2_testing/test_runner/basic_sanity_tests.py +++ b/tests/L2_testing/test_runner/basic_sanity_tests.py @@ -85,42 +85,43 @@ def execute_test(): return test_utils.count_print_results(output_table) -# we need to do this asynchronous as if there is no such string we would end in endless loop -def read_asynchronous(proc, string_to_find, timeout): - """Reads asynchronous from process. Ends when found string or timeout occurred. +# Helper function for multiprocessing - must be at module level to be picklable +def _wait_for_string(proc, string_to_find): + """Waits indefinitely until string is found in process. Must be run with timeout multiprocess. Parameters: proc (process): process in which we want to read string_to_find (string): what we want to find in process - timeout (float): how long we should wait if string not found (seconds) Returns: - found (bool): True if found string_to_find inside proc. + None: Returns nothing if found, never ends if not found """ - # as this function should not be used outside asynchronous read, it is moved inside it - def wait_for_string(proc, string_to_find): - """Waits indefinitely until string is found in process. Must be run with timeout multiprocess. + while True: + # notice that all data are in stderr not in stdout, this is DobbyDaemon design + output = proc.stderr.readline() + if string_to_find in output: + test_utils.print_log("Found string \"%s\"" % string_to_find, test_utils.Severity.debug) + return - Parameters: - proc (process): process in which we want to read - string_to_find (string): what we want to find in process - Returns: - None: Returns nothing if found, never ends if not found +# we need to do this asynchronous as if there is no such string we would end in endless loop +def read_asynchronous(proc, string_to_find, timeout): + """Reads asynchronous from process. Ends when found string or timeout occurred. - """ + Parameters: + proc (process): process in which we want to read + string_to_find (string): what we want to find in process + timeout (float): how long we should wait if string not found (seconds) + + Returns: + found (bool): True if found string_to_find inside proc. - while True: - # notice that all data are in stderr not in stdout, this is DobbyDaemon design - output = proc.stderr.readline() - if string_to_find in output: - test_utils.print_log("Found string \"%s\"" % string_to_find, test_utils.Severity.debug) - return + """ found = False - reader = multiprocessing.Process(target=wait_for_string, args=(proc, string_to_find), kwargs={}) + reader = multiprocessing.Process(target=_wait_for_string, args=(proc, string_to_find), kwargs={}) test_utils.print_log("Starting multithread read", test_utils.Severity.debug) reader.start() reader.join(timeout) @@ -203,3 +204,4 @@ def stop_dobby_daemon(): if __name__ == "__main__": test_utils.parse_arguments(__file__, True) execute_test() + From a00cf91bdce41c230f768fc8a79b555dc71629a7 Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Wed, 25 Mar 2026 04:48:57 +0000 Subject: [PATCH 02/21] use threading --- tests/L2_testing/test_runner/basic_sanity_tests.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/L2_testing/test_runner/basic_sanity_tests.py b/tests/L2_testing/test_runner/basic_sanity_tests.py index d107a57d6..1fb18d39b 100755 --- a/tests/L2_testing/test_runner/basic_sanity_tests.py +++ b/tests/L2_testing/test_runner/basic_sanity_tests.py @@ -19,7 +19,7 @@ from subprocess import check_output import subprocess from time import sleep -import multiprocessing +import threading from os.path import basename tests = ( @@ -121,14 +121,13 @@ def read_asynchronous(proc, string_to_find, timeout): """ found = False - reader = multiprocessing.Process(target=_wait_for_string, args=(proc, string_to_find), kwargs={}) + reader = threading.Thread(target=_wait_for_string, args=(proc, string_to_find)) test_utils.print_log("Starting multithread read", test_utils.Severity.debug) reader.start() reader.join(timeout) # if thread still running if reader.is_alive(): test_utils.print_log("Reader still exists, closing", test_utils.Severity.debug) - reader.terminate() test_utils.print_log("Not found string \"%s\"" % string_to_find, test_utils.Severity.error) else: found = True From e7f4022db798823b1a85b8515332f4387c33bc98 Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Wed, 25 Mar 2026 05:47:45 +0000 Subject: [PATCH 03/21] Chnage to ubuntu20 --- .github/workflows/L2-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/L2-tests.yml index 7717c4f56..b417d1480 100755 --- a/.github/workflows/L2-tests.yml +++ b/.github/workflows/L2-tests.yml @@ -6,7 +6,7 @@ env: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: @@ -254,3 +254,4 @@ jobs: DobbyL2TestResults.json l2coverage if-no-files-found: warn + From 961353412eeb956632929a33952a6ae8c9b4bb5a Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Wed, 25 Mar 2026 06:09:14 +0000 Subject: [PATCH 04/21] run on container img --- .github/workflows/L2-tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) mode change 100755 => 100644 .github/workflows/L2-tests.yml diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/L2-tests.yml old mode 100755 new mode 100644 index b417d1480..5a0aa8968 --- a/.github/workflows/L2-tests.yml +++ b/.github/workflows/L2-tests.yml @@ -6,7 +6,10 @@ env: jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: + image: ubuntu:20.04 + options: --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:rw strategy: fail-fast: false matrix: From 88179937362a08dc5aa570b550bdd36abd5fedce Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Wed, 25 Mar 2026 06:13:21 +0000 Subject: [PATCH 05/21] change permissions --- .github/workflows/L2-tests.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .github/workflows/L2-tests.yml diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/L2-tests.yml old mode 100644 new mode 100755 From f8d2cac8577982c8c464fbd6ccb694eed00f2db1 Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Wed, 25 Mar 2026 06:15:59 +0000 Subject: [PATCH 06/21] remove sudo --- .github/workflows/L2-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/L2-tests.yml index 5a0aa8968..5349d7a36 100755 --- a/.github/workflows/L2-tests.yml +++ b/.github/workflows/L2-tests.yml @@ -32,9 +32,9 @@ jobs: steps: - name: Install packages run: > - sudo apt update + apt update && - sudo apt install -y build-essential cmake make git gcc pkgconf libtool libctemplate-dev libjsoncpp-dev libdbus-1-dev libsystemd-dev libyajl-dev libcap-dev go-md2man autoconf automake libseccomp-dev libboost-dev valgrind libcunit1-dev liblog4c-dev libfreetype6-dev libjpeg-dev xorg-dev python3 python3-pip libarchive-dev libcurl4 libcurl4-gnutls-dev libssl-dev libgpgme11-dev libtool-bin libarchive13 bison flex clang lcov figlet dbus libdbus-glib-1-dev dbus-user-session systemd libpam-systemd gnome-keyring iptables libprotobuf-c-dev libzstd-dev + apt install -y sudo build-essential cmake make git gcc pkgconf libtool libctemplate-dev libjsoncpp-dev libdbus-1-dev libsystemd-dev libyajl-dev libcap-dev go-md2man autoconf automake libseccomp-dev libboost-dev valgrind libcunit1-dev liblog4c-dev libfreetype6-dev libjpeg-dev xorg-dev python3 python3-pip libarchive-dev libcurl4 libcurl4-gnutls-dev libssl-dev libgpgme11-dev libtool-bin libarchive13 bison flex clang lcov figlet dbus libdbus-glib-1-dev dbus-user-session systemd libpam-systemd gnome-keyring iptables libprotobuf-c-dev libzstd-dev - name: Set gcc/with-coverage toolchain if: ${{ matrix.compiler == 'gcc' && matrix.coverage == 'with-coverage' }} From d661e5124720af75f6db716817a233e6b6bee782 Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Wed, 25 Mar 2026 06:27:13 +0000 Subject: [PATCH 07/21] making it noninteractive --- .github/workflows/L2-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/L2-tests.yml index 5349d7a36..8504ad8d2 100755 --- a/.github/workflows/L2-tests.yml +++ b/.github/workflows/L2-tests.yml @@ -31,6 +31,9 @@ jobs: name: Build in ${{ matrix.build_type }} Mode (${{ matrix.extra_flags }}) steps: - name: Install packages + env: + DEBIAN_FRONTEND: noninteractive + TZ: Etc/UTC run: > apt update && From 6a23f2db5757ba8a60194f7559d4cba971b9064d Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Wed, 25 Mar 2026 07:23:34 +0000 Subject: [PATCH 08/21] resolv.conf issues --- .github/workflows/L2-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/L2-tests.yml index 8504ad8d2..0773a3525 100755 --- a/.github/workflows/L2-tests.yml +++ b/.github/workflows/L2-tests.yml @@ -124,7 +124,7 @@ jobs: - name: build Dobby run: | - sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf + sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf || true cd Dobby sudo cp /usr/lib/x86_64-linux-gnu/dbus-1.0/include/dbus/dbus-arch-deps.h /usr/include/dbus-1.0/dbus sudo mkdir -p /usr/lib/plugins/dobby From 0c0024fb7f86c3a06a25c11687d2426cbdc9a37f Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Wed, 25 Mar 2026 07:31:24 +0000 Subject: [PATCH 09/21] Add recursive --- .github/workflows/L2-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/L2-tests.yml index 0773a3525..99ede8ac9 100755 --- a/.github/workflows/L2-tests.yml +++ b/.github/workflows/L2-tests.yml @@ -60,6 +60,7 @@ jobs: uses: actions/checkout@v4 with: path: Dobby + submodules: recursive - name: Install gmock run: | From 1903ec85f7ceaee2706d4d222f63af89b0f9a82d Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Wed, 25 Mar 2026 07:43:10 +0000 Subject: [PATCH 10/21] rt_dobby_schema fix --- .github/workflows/{L2-tests.yml => l2-tests.yml} | 10 ++++++++++ 1 file changed, 10 insertions(+) rename .github/workflows/{L2-tests.yml => l2-tests.yml} (94%) mode change 100755 => 100644 diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/l2-tests.yml old mode 100755 new mode 100644 similarity index 94% rename from .github/workflows/L2-tests.yml rename to .github/workflows/l2-tests.yml index 99ede8ac9..97e70dbee --- a/.github/workflows/L2-tests.yml +++ b/.github/workflows/l2-tests.yml @@ -62,6 +62,16 @@ jobs: path: Dobby submodules: recursive + - name: Initialize submodules + run: | + cd Dobby + git config --global --add safe.directory /__w/Dobby/Dobby/Dobby + git config --global --add safe.directory /__w/Dobby/Dobby/Dobby/libocispec + git config --global --add safe.directory /__w/Dobby/Dobby/Dobby/libocispec/runtime-spec + git config --global --add safe.directory /__w/Dobby/Dobby/Dobby/libocispec/image-spec + git config --global --add safe.directory /__w/Dobby/Dobby/Dobby/libocispec/yajl + git submodule update --init --recursive + - name: Install gmock run: | cd $GITHUB_WORKSPACE From d247dd6bf3acd37d959728ffed723596b94cb72c Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Wed, 25 Mar 2026 07:50:34 +0000 Subject: [PATCH 11/21] header fix --- .github/workflows/{l2-tests.yml => L2-tests.yml} | 2 ++ 1 file changed, 2 insertions(+) rename .github/workflows/{l2-tests.yml => L2-tests.yml} (98%) diff --git a/.github/workflows/l2-tests.yml b/.github/workflows/L2-tests.yml similarity index 98% rename from .github/workflows/l2-tests.yml rename to .github/workflows/L2-tests.yml index 97e70dbee..2e6a4c987 100644 --- a/.github/workflows/l2-tests.yml +++ b/.github/workflows/L2-tests.yml @@ -136,6 +136,8 @@ jobs: - name: build Dobby run: | sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf || true + # Configure git safe directories for CMake's libocispec generator + git config --global --add safe.directory '*' cd Dobby sudo cp /usr/lib/x86_64-linux-gnu/dbus-1.0/include/dbus/dbus-arch-deps.h /usr/include/dbus-1.0/dbus sudo mkdir -p /usr/lib/plugins/dobby From eab817ffa4ef0aa0f7cd3d9a83cc062567d1ae5a Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Wed, 25 Mar 2026 08:49:47 +0000 Subject: [PATCH 12/21] memswap removed 60 --- .github/workflows/L2-tests.yml | 25 +++---------------- .../OciConfigJson1.0.2-dobby.template | 4 +-- .../OciConfigJsonVM1.0.2-dobby.template | 4 +-- 3 files changed, 7 insertions(+), 26 deletions(-) diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/L2-tests.yml index 2e6a4c987..77b778692 100644 --- a/.github/workflows/L2-tests.yml +++ b/.github/workflows/L2-tests.yml @@ -7,9 +7,6 @@ env: jobs: build: runs-on: ubuntu-latest - container: - image: ubuntu:20.04 - options: --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:rw strategy: fail-fast: false matrix: @@ -31,13 +28,10 @@ jobs: name: Build in ${{ matrix.build_type }} Mode (${{ matrix.extra_flags }}) steps: - name: Install packages - env: - DEBIAN_FRONTEND: noninteractive - TZ: Etc/UTC run: > - apt update + sudo apt update && - apt install -y sudo build-essential cmake make git gcc pkgconf libtool libctemplate-dev libjsoncpp-dev libdbus-1-dev libsystemd-dev libyajl-dev libcap-dev go-md2man autoconf automake libseccomp-dev libboost-dev valgrind libcunit1-dev liblog4c-dev libfreetype6-dev libjpeg-dev xorg-dev python3 python3-pip libarchive-dev libcurl4 libcurl4-gnutls-dev libssl-dev libgpgme11-dev libtool-bin libarchive13 bison flex clang lcov figlet dbus libdbus-glib-1-dev dbus-user-session systemd libpam-systemd gnome-keyring iptables libprotobuf-c-dev libzstd-dev + sudo apt install -y build-essential cmake make git gcc pkgconf libtool libctemplate-dev libjsoncpp-dev libdbus-1-dev libsystemd-dev libyajl-dev libcap-dev go-md2man autoconf automake libseccomp-dev libboost-dev valgrind libcunit1-dev liblog4c-dev libfreetype6-dev libjpeg-dev xorg-dev python3 python3-pip libarchive-dev libcurl4 libcurl4-gnutls-dev libssl-dev libgpgme11-dev libtool-bin libarchive13 bison flex clang lcov figlet dbus libdbus-glib-1-dev dbus-user-session systemd libpam-systemd gnome-keyring iptables libprotobuf-c-dev libzstd-dev - name: Set gcc/with-coverage toolchain if: ${{ matrix.compiler == 'gcc' && matrix.coverage == 'with-coverage' }} @@ -60,17 +54,6 @@ jobs: uses: actions/checkout@v4 with: path: Dobby - submodules: recursive - - - name: Initialize submodules - run: | - cd Dobby - git config --global --add safe.directory /__w/Dobby/Dobby/Dobby - git config --global --add safe.directory /__w/Dobby/Dobby/Dobby/libocispec - git config --global --add safe.directory /__w/Dobby/Dobby/Dobby/libocispec/runtime-spec - git config --global --add safe.directory /__w/Dobby/Dobby/Dobby/libocispec/image-spec - git config --global --add safe.directory /__w/Dobby/Dobby/Dobby/libocispec/yajl - git submodule update --init --recursive - name: Install gmock run: | @@ -135,9 +118,7 @@ jobs: - name: build Dobby run: | - sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf || true - # Configure git safe directories for CMake's libocispec generator - git config --global --add safe.directory '*' + sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf cd Dobby sudo cp /usr/lib/x86_64-linux-gnu/dbus-1.0/include/dbus/dbus-arch-deps.h /usr/include/dbus-1.0/dbus sudo mkdir -p /usr/lib/plugins/dobby diff --git a/bundle/lib/source/templates/OciConfigJson1.0.2-dobby.template b/bundle/lib/source/templates/OciConfigJson1.0.2-dobby.template index 6cbdabd43..11fc4756d 100644 --- a/bundle/lib/source/templates/OciConfigJson1.0.2-dobby.template +++ b/bundle/lib/source/templates/OciConfigJson1.0.2-dobby.template @@ -328,8 +328,7 @@ static const char* ociJsonTemplate = R"JSON( ], "memory": { "limit": {{MEM_LIMIT}}, - "swap": {{MEM_LIMIT}}, - "swappiness": 60 + "swap": {{MEM_LIMIT}} }, "cpu": { {{#CPU_SHARES_ENABLED}} @@ -401,3 +400,4 @@ static const char* ociJsonTemplate = R"JSON( {{/ENABLE_RDK_PLUGINS}} } )JSON"; + diff --git a/bundle/lib/source/templates/OciConfigJsonVM1.0.2-dobby.template b/bundle/lib/source/templates/OciConfigJsonVM1.0.2-dobby.template index 21fe91d38..3b571cc9c 100644 --- a/bundle/lib/source/templates/OciConfigJsonVM1.0.2-dobby.template +++ b/bundle/lib/source/templates/OciConfigJsonVM1.0.2-dobby.template @@ -339,8 +339,7 @@ static const char* ociJsonTemplate = R"JSON( ], "memory": { "limit": {{MEM_LIMIT}}, - "swap": {{MEM_LIMIT}}, - "swappiness": 60 + "swap": {{MEM_LIMIT}} }, "cpu": { {{#CPU_SHARES_ENABLED}} @@ -412,3 +411,4 @@ static const char* ociJsonTemplate = R"JSON( {{/ENABLE_RDK_PLUGINS}} } )JSON"; + From 3511a779097d42eab8f9442005b7f3d44f140e22 Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Wed, 25 Mar 2026 13:33:43 +0000 Subject: [PATCH 13/21] Increase sleep --- tests/L2_testing/test_runner/test_utils.py | 3 ++- tests/L2_testing/test_runner/thunder_plugin.py | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) mode change 100755 => 100644 tests/L2_testing/test_runner/thunder_plugin.py diff --git a/tests/L2_testing/test_runner/test_utils.py b/tests/L2_testing/test_runner/test_utils.py index 9acf63181..7fa7f27bc 100755 --- a/tests/L2_testing/test_runner/test_utils.py +++ b/tests/L2_testing/test_runner/test_utils.py @@ -82,7 +82,7 @@ def __init__(self, log_to_stdout = False): # as this process is running infinitely we cannot use run_command_line as it waits for execution to end self.subproc = subprocess.Popen(cmd, **kvargs) - sleep(1) # give DobbyDaemon time to initialise + sleep(2) # give DobbyDaemon time to initialise def __enter__(self): return self.subproc @@ -507,3 +507,4 @@ def dobby_tool_command(command, container_id, params=None): process = run_command_line(full_command) return process + diff --git a/tests/L2_testing/test_runner/thunder_plugin.py b/tests/L2_testing/test_runner/thunder_plugin.py old mode 100755 new mode 100644 index 6ffc0a3e0..f0b860a7a --- a/tests/L2_testing/test_runner/thunder_plugin.py +++ b/tests/L2_testing/test_runner/thunder_plugin.py @@ -213,7 +213,7 @@ def start_wpeframework_vm(): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - sleep(2) + sleep(3) # Allow WPEFramework time to fully initialize return subproc @@ -252,6 +252,10 @@ def execute_test(): output_table.append(output) test_utils.print_single_result(output) + # Add delay after container state changes to allow operations to complete + if test.command in ["startContainer", "startContainerFromDobbySpec", "pauseContainer", "resumeContainer"]: + sleep(1) + stop_wpeframework(wpeframework) return test_utils.count_print_results(output_table) @@ -260,3 +264,4 @@ def execute_test(): if __name__ == "__main__": test_utils.parse_arguments(__file__, True) execute_test() + From 98c4b4bf424a6405834a17c7eb85936f74a64740 Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Fri, 27 Mar 2026 08:40:18 +0000 Subject: [PATCH 14/21] config.json issues --- bundle/lib/source/DobbyConfig.cpp | 21 +++++++-- pluginLauncher/tool/source/Main.cpp | 43 ++++++++++++------- .../test_runner/annotation_tests.py | 2 + .../L2_testing/test_runner/pid_limit_tests.py | 2 + .../test_runner/start_from_bundle.py | 9 ++-- 5 files changed, 56 insertions(+), 21 deletions(-) mode change 100644 => 100755 bundle/lib/source/DobbyConfig.cpp diff --git a/bundle/lib/source/DobbyConfig.cpp b/bundle/lib/source/DobbyConfig.cpp old mode 100644 new mode 100755 index 0fe8441ec..f9f7ee974 --- a/bundle/lib/source/DobbyConfig.cpp +++ b/bundle/lib/source/DobbyConfig.cpp @@ -659,8 +659,10 @@ void DobbyConfig::addPluginLauncherHooks(std::shared_ptr cfg, c cfg->hooks = (rt_dobby_schema_hooks*)calloc(1, sizeof(rt_dobby_schema_hooks)); } - // createRuntime, createContainer, poststart and poststop hook paths must - // resolve in the runtime namespace - config is in bundle + // createRuntime and poststop hook paths must resolve in the runtime namespace + // config is in bundle. For createContainer hook, we also use bundle path since + // according to OCI spec, hook path resolution should happen from runtime namespace + // even though execution happens in container namespace std::string configPath = bundlePath + "/config.json"; // populate createRuntime hook with DobbyPluginLauncher args @@ -670,6 +672,8 @@ void DobbyConfig::addPluginLauncherHooks(std::shared_ptr cfg, c cfg->hooks->create_runtime[cfg->hooks->create_runtime_len-1] = createRuntimeEntry; // populate createContainer hook with DobbyPluginLauncher args + // Note: createContainer runs in container namespace but path resolution should + // be from runtime namespace per OCI spec rt_defs_hook *createContainerEntry = (rt_defs_hook*)calloc(1, sizeof(rt_defs_hook)); setPluginHookEntry(createContainerEntry, "createContainer", configPath); cfg->hooks->create_container = (rt_defs_hook**)realloc(cfg->hooks->create_container, sizeof(rt_defs_hook*) * ++cfg->hooks->create_container_len); @@ -748,6 +752,16 @@ bool DobbyConfig::updateBundleConfig(const ContainerId& id, std::shared_ptrrdk_plugins && cfg->rdk_plugins->plugins_count) { + // Always bind mount the bundle directory into the container at the same path. + // This is needed because createContainer hooks run in the container's mount + // namespace but need to access the config.json file which exists on the host. + // By mounting the bundle directory, the host path becomes accessible. + if(!addMount(bundlePath, bundlePath, "bind", 0, + { "rbind", "ro", "nosuid", "nodev" })) + { + AI_LOG_WARN("Failed to add bundle mount for hooks, createContainer may fail"); + } + #ifdef USE_STARTCONTAINER_HOOK // bindmount DobbyPluginLauncher to container if(!addMount(PLUGINLAUNCHER_PATH, PLUGINLAUNCHER_PATH, "bind", 0, @@ -756,7 +770,7 @@ bool DobbyConfig::updateBundleConfig(const ContainerId& id, std::shared_ptr state = getContainerState(); + if (state) + { + gContainerId = std::string(state->id); + } + else + { + AI_LOG_WARN("Failed to get container state from stdin"); + return EXIT_FAILURE; + } + // Create a libocispec object for the container's config char *absPath = realpath(gConfigPath.c_str(), NULL); if (absPath == nullptr) { - AI_LOG_ERROR("Couldn't find config at %s", gConfigPath.c_str()); - return EXIT_FAILURE; + // The config path may not be accessible from the current namespace. + // Try using the bundle path from the container state as a fallback. + if (state->bundle != nullptr) + { + std::string fallbackPath = std::string(state->bundle) + "/config.json"; + AI_LOG_INFO("Config not found at '%s', trying bundle path '%s'", + gConfigPath.c_str(), fallbackPath.c_str()); + absPath = realpath(fallbackPath.c_str(), NULL); + } + + if (absPath == nullptr) + { + AI_LOG_ERROR("Couldn't find config at %s", gConfigPath.c_str()); + return EXIT_FAILURE; + } } const std::string fullConfigPath = std::string(absPath); free(absPath); @@ -478,19 +503,6 @@ int main(int argc, char *argv[]) return EXIT_FAILURE; } - // Get container id from state (using hostname may be incorrect if we - // launch multiple containers from same bundle) - std::shared_ptr state = getContainerState(); - if (state) - { - gContainerId = std::string(state->id); - } - else - { - AI_LOG_WARN("Failed to get container state from stdin"); - return false; - } - AI_LOG_MILESTONE("Running hook %s for container '%s'", gHookName.c_str(), gContainerId.c_str()); // Get the path of the container rootfs to give to plugins @@ -526,3 +538,4 @@ int main(int argc, char *argv[]) AI_LOG_WARN("Hook %s failed - plugin(s) ran with errors", gHookName.c_str()); return EXIT_FAILURE; } + diff --git a/tests/L2_testing/test_runner/annotation_tests.py b/tests/L2_testing/test_runner/annotation_tests.py index 008e1752f..a026e4661 100644 --- a/tests/L2_testing/test_runner/annotation_tests.py +++ b/tests/L2_testing/test_runner/annotation_tests.py @@ -63,6 +63,7 @@ def test_container(container_id, expected_output): if "started '" + container_id + "' container" not in status.stdout: return False, "Container did not launch successfully" + # Keep bundle around while container is running - Dobby uses bundle in-place return validate_annotation(container_id, expected_output) @@ -126,3 +127,4 @@ def validate_annotation(container_id, expected_output): if __name__ == "__main__": test_utils.parse_arguments(__file__, True) execute_test() + diff --git a/tests/L2_testing/test_runner/pid_limit_tests.py b/tests/L2_testing/test_runner/pid_limit_tests.py index 83689328b..6512707f5 100644 --- a/tests/L2_testing/test_runner/pid_limit_tests.py +++ b/tests/L2_testing/test_runner/pid_limit_tests.py @@ -69,6 +69,7 @@ def test_container(container_id, expected_output): if "started '" + container_id + "' container" not in status.stdout: return False, "Container did not launch successfully" + # Keep bundle around while container is running - Dobby uses bundle in-place return validate_pid_limit(container_id, expected_output) @@ -103,3 +104,4 @@ def validate_pid_limit(container_id, expected_output): if __name__ == "__main__": test_utils.parse_arguments(__file__, True) execute_test() + diff --git a/tests/L2_testing/test_runner/start_from_bundle.py b/tests/L2_testing/test_runner/start_from_bundle.py index e7b71cedb..befefa036 100755 --- a/tests/L2_testing/test_runner/start_from_bundle.py +++ b/tests/L2_testing/test_runner/start_from_bundle.py @@ -65,10 +65,12 @@ def test_container(container_id, expected_output): with test_utils.untar_bundle(container_id) as bundle_path: launch_result = test_utils.launch_container(container_id, bundle_path) - if launch_result: - return validate_output_file(container_id, expected_output) + # Keep bundle around until container has fully exited + # This is needed because Dobby uses the bundle in-place and hooks need access to config.json + if launch_result: + return validate_output_file(container_id, expected_output) - return False, "Container did not launch successfully" + return False, "Container did not launch successfully" def validate_output_file(container_id, expected_output): @@ -102,3 +104,4 @@ def validate_output_file(container_id, expected_output): if __name__ == "__main__": test_utils.parse_arguments(__file__, True) execute_test() + From 246754b7b156a7bb1b92d792d4048862aa653add Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Fri, 27 Mar 2026 09:04:48 +0000 Subject: [PATCH 15/21] container start issues --- bundle/lib/source/DobbyConfig.cpp | 48 +++++++++++++---------------- pluginLauncher/tool/source/Main.cpp | 28 ++++++++++++++--- 2 files changed, 45 insertions(+), 31 deletions(-) mode change 100755 => 100644 bundle/lib/source/DobbyConfig.cpp mode change 100644 => 100755 pluginLauncher/tool/source/Main.cpp diff --git a/bundle/lib/source/DobbyConfig.cpp b/bundle/lib/source/DobbyConfig.cpp old mode 100755 new mode 100644 index f9f7ee974..ef47baff1 --- a/bundle/lib/source/DobbyConfig.cpp +++ b/bundle/lib/source/DobbyConfig.cpp @@ -660,45 +660,48 @@ void DobbyConfig::addPluginLauncherHooks(std::shared_ptr cfg, c } // createRuntime and poststop hook paths must resolve in the runtime namespace - // config is in bundle. For createContainer hook, we also use bundle path since - // according to OCI spec, hook path resolution should happen from runtime namespace - // even though execution happens in container namespace - std::string configPath = bundlePath + "/config.json"; + // and execute in runtime namespace, so they can use the bundle path directly + std::string runtimeConfigPath = bundlePath + "/config.json"; + + // createContainer hook runs in container namespace, so it needs to use + // the fixed path where we mounted the config.json + std::string containerConfigPath = "/tmp/dobby_config.json"; // populate createRuntime hook with DobbyPluginLauncher args rt_defs_hook *createRuntimeEntry = (rt_defs_hook*)calloc(1, sizeof(rt_defs_hook)); - setPluginHookEntry(createRuntimeEntry, "createRuntime", configPath); + setPluginHookEntry(createRuntimeEntry, "createRuntime", runtimeConfigPath); cfg->hooks->create_runtime = (rt_defs_hook**)realloc(cfg->hooks->create_runtime, sizeof(rt_defs_hook*) * ++cfg->hooks->create_runtime_len); cfg->hooks->create_runtime[cfg->hooks->create_runtime_len-1] = createRuntimeEntry; // populate createContainer hook with DobbyPluginLauncher args - // Note: createContainer runs in container namespace but path resolution should - // be from runtime namespace per OCI spec + // Uses container namespace path since hook executes in container namespace rt_defs_hook *createContainerEntry = (rt_defs_hook*)calloc(1, sizeof(rt_defs_hook)); - setPluginHookEntry(createContainerEntry, "createContainer", configPath); + setPluginHookEntry(createContainerEntry, "createContainer", containerConfigPath); cfg->hooks->create_container = (rt_defs_hook**)realloc(cfg->hooks->create_container, sizeof(rt_defs_hook*) * ++cfg->hooks->create_container_len); cfg->hooks->create_container[cfg->hooks->create_container_len-1] = createContainerEntry; // populate poststart hook with DobbyPluginLauncher args + // poststart runs in runtime namespace, so use runtime path rt_defs_hook *poststartEntry = (rt_defs_hook*)calloc(1, sizeof(rt_defs_hook)); - setPluginHookEntry(poststartEntry, "poststart", configPath); + setPluginHookEntry(poststartEntry, "poststart", runtimeConfigPath); cfg->hooks->poststart = (rt_defs_hook**)realloc(cfg->hooks->poststart, sizeof(rt_defs_hook*) * ++cfg->hooks->poststart_len); cfg->hooks->poststart[cfg->hooks->poststart_len-1] = poststartEntry; // populate poststop hook with DobbyPluginLauncher args + // poststop runs in runtime namespace, so use runtime path rt_defs_hook *poststopEntry = (rt_defs_hook*)calloc(1, sizeof(rt_defs_hook)); - setPluginHookEntry(poststopEntry, "poststop", configPath); + setPluginHookEntry(poststopEntry, "poststop", runtimeConfigPath); cfg->hooks->poststop = (rt_defs_hook**)realloc(cfg->hooks->poststop, sizeof(rt_defs_hook*) * ++cfg->hooks->poststop_len); cfg->hooks->poststop[cfg->hooks->poststop_len-1] = poststopEntry; #ifdef USE_STARTCONTAINER_HOOK - // startContainer hook paths must resolve in the container namespace, - // config is in container rootdir - configPath = "/tmp/config.json"; + // startContainer hook runs in container namespace after pivot_root, + // use the path where config was mounted for startContainer + std::string startContainerConfigPath = "/tmp/config.json"; // populate startContainer hook with DobbyPluginLauncher args rt_defs_hook *startContainerEntry = (rt_defs_hook*)calloc(1, sizeof(rt_defs_hook)); - setPluginHookEntry(startContainerEntry, "startContainer", configPath); + setPluginHookEntry(startContainerEntry, "startContainer", startContainerConfigPath); cfg->hooks->start_container = (rt_defs_hook**)realloc(cfg->hooks->start_container, sizeof(rt_defs_hook*) * ++cfg->hooks->start_container_len); cfg->hooks->start_container[cfg->hooks->start_container_len-1] = startContainerEntry; #endif @@ -752,14 +755,14 @@ bool DobbyConfig::updateBundleConfig(const ContainerId& id, std::shared_ptrrdk_plugins && cfg->rdk_plugins->plugins_count) { - // Always bind mount the bundle directory into the container at the same path. + // Bind mount the config.json file to a fixed path inside the container. // This is needed because createContainer hooks run in the container's mount // namespace but need to access the config.json file which exists on the host. - // By mounting the bundle directory, the host path becomes accessible. - if(!addMount(bundlePath, bundlePath, "bind", 0, - { "rbind", "ro", "nosuid", "nodev" })) + // We use /tmp/dobby_config.json as a well-known location that exists in all containers. + if(!addMount(bundlePath + "/config.json", "/tmp/dobby_config.json", "bind", 0, + { "bind", "ro", "nosuid", "nodev" })) { - AI_LOG_WARN("Failed to add bundle mount for hooks, createContainer may fail"); + AI_LOG_WARN("Failed to add config mount for hooks, createContainer may fail"); } #ifdef USE_STARTCONTAINER_HOOK @@ -769,13 +772,6 @@ bool DobbyConfig::updateBundleConfig(const ContainerId& id, std::shared_ptr fallbackPaths; + + // Try container namespace fixed path first (for createContainer hook) + fallbackPaths.push_back("/tmp/dobby_config.json"); + + // Then try bundle path from state (for runtime namespace hooks) if (state->bundle != nullptr) { - std::string fallbackPath = std::string(state->bundle) + "/config.json"; - AI_LOG_INFO("Config not found at '%s', trying bundle path '%s'", + fallbackPaths.push_back(std::string(state->bundle) + "/config.json"); + } + + for (const auto& fallbackPath : fallbackPaths) + { + AI_LOG_INFO("Config not found at '%s', trying '%s'", gConfigPath.c_str(), fallbackPath.c_str()); absPath = realpath(fallbackPath.c_str(), NULL); + if (absPath != nullptr) + { + break; + } } if (absPath == nullptr) { - AI_LOG_ERROR("Couldn't find config at %s", gConfigPath.c_str()); + AI_LOG_ERROR("Couldn't find config at %s (also tried fallback paths)", gConfigPath.c_str()); return EXIT_FAILURE; } } From bc86f0585134b924fcb72cdd0768a0e149a70f92 Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Fri, 27 Mar 2026 11:58:08 +0000 Subject: [PATCH 16/21] Resolve commit path --- pluginLauncher/tool/source/Main.cpp | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 pluginLauncher/tool/source/Main.cpp diff --git a/pluginLauncher/tool/source/Main.cpp b/pluginLauncher/tool/source/Main.cpp old mode 100755 new mode 100644 From 2111b3326d6e9af25586a0a8719de865e78465c4 Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Fri, 27 Mar 2026 12:13:09 +0000 Subject: [PATCH 17/21] network isnterface issues --- .github/workflows/L2-tests.yml | 29 +++++++++++++++++++++++ daemon/process/settings/dobby.dev_vm.json | 3 ++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/L2-tests.yml index 77b778692..09248c3ca 100644 --- a/.github/workflows/L2-tests.yml +++ b/.github/workflows/L2-tests.yml @@ -116,6 +116,35 @@ jobs: ' > "/etc/dbus-1/system.d/org.rdk.dobby.conf" + - name: Setup network interfaces for testing + run: | + # Create dummy network interfaces that tests expect + sudo ip link add enp0s8 type dummy || true + sudo ip link add enp0s3 type dummy || true + sudo ip link set enp0s8 up || true + sudo ip link set enp0s3 up || true + # Create bridge for Dobby networking + sudo ip link add dobby0 type bridge || true + sudo ip link set dobby0 up || true + # Assign IP to dobby0 bridge for container networking + sudo ip addr add 100.64.11.1/24 dev dobby0 || true + # Enable IP forwarding for container network access + sudo sysctl -w net.ipv4.ip_forward=1 + # Setup NAT for container outbound traffic + sudo iptables -t nat -A POSTROUTING -s 100.64.11.0/24 -o eth0 -j MASQUERADE || true + sudo iptables -A FORWARD -i dobby0 -o eth0 -j ACCEPT || true + sudo iptables -A FORWARD -i eth0 -o dobby0 -m state --state RELATED,ESTABLISHED -j ACCEPT || true + + - name: Setup permissions and directories + run: | + # Create and set permissions for profiling/coverage directories + sudo mkdir -p /home/runner/work + sudo chmod -R 777 /home/runner/work + sudo chmod -R 777 $GITHUB_WORKSPACE || true + # Create tmp directory for container logs + sudo mkdir -p /tmp + sudo chmod 1777 /tmp + - name: build Dobby run: | sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf diff --git a/daemon/process/settings/dobby.dev_vm.json b/daemon/process/settings/dobby.dev_vm.json index 8ab3cb1f2..61efecc49 100644 --- a/daemon/process/settings/dobby.dev_vm.json +++ b/daemon/process/settings/dobby.dev_vm.json @@ -14,7 +14,7 @@ ], "network": { - "externalInterfaces": [ "enp0s8", "enp0s3", "docker0", "eth0" ], + "externalInterfaces": [ "eth0", "enp0s8", "enp0s3", "docker0" ], "addressRange": "100.64.11.0" }, @@ -32,3 +32,4 @@ + From 6b7a25f7ae70a54cbd35ada9ecdcffa981e1ed4d Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Mon, 30 Mar 2026 06:28:58 +0000 Subject: [PATCH 18/21] detect cgroupv2 --- bundle/lib/source/DobbyConfig.cpp | 4 +- bundle/lib/source/DobbyTemplate.cpp | 14 ++++- daemon/lib/source/DobbyEnv.cpp | 58 +++++++++++++------ rdkPlugins/GPU/source/GpuPlugin.cpp | 12 +++- .../IONMemory/source/IonMemoryPlugin.cpp | 12 +++- tests/L2_testing/test_runner/memcr_tests.py | 5 +- .../L2_testing/test_runner/pid_limit_tests.py | 41 +++++++++++-- tests/L2_testing/test_runner/test_utils.py | 55 ++++++++++++++++++ 8 files changed, 170 insertions(+), 31 deletions(-) mode change 100755 => 100644 tests/L2_testing/test_runner/test_utils.py diff --git a/bundle/lib/source/DobbyConfig.cpp b/bundle/lib/source/DobbyConfig.cpp index ef47baff1..30d099b0b 100644 --- a/bundle/lib/source/DobbyConfig.cpp +++ b/bundle/lib/source/DobbyConfig.cpp @@ -696,8 +696,8 @@ void DobbyConfig::addPluginLauncherHooks(std::shared_ptr cfg, c #ifdef USE_STARTCONTAINER_HOOK // startContainer hook runs in container namespace after pivot_root, - // use the path where config was mounted for startContainer - std::string startContainerConfigPath = "/tmp/config.json"; + // use the same path where config was mounted for createContainer + std::string startContainerConfigPath = "/tmp/dobby_config.json"; // populate startContainer hook with DobbyPluginLauncher args rt_defs_hook *startContainerEntry = (rt_defs_hook*)calloc(1, sizeof(rt_defs_hook)); diff --git a/bundle/lib/source/DobbyTemplate.cpp b/bundle/lib/source/DobbyTemplate.cpp index bfe80ca4c..b3a92b195 100644 --- a/bundle/lib/source/DobbyTemplate.cpp +++ b/bundle/lib/source/DobbyTemplate.cpp @@ -467,16 +467,25 @@ void DobbyTemplate::setTemplateCpuRtSched() long cpuRtRuntime = 0; long cpuRtPeriod = 0; + bool isCgroupV2 = false; while ((mnt = getmntent_r(procMounts, &mntBuf, buf, sizeof(buf))) != nullptr) { // skip entries that don't have a mount point, type or options if (!mnt->mnt_type || !mnt->mnt_dir || !mnt->mnt_opts) continue; - // skip non-cgroup mounts - if (strcmp(mnt->mnt_type, "cgroup") != 0) + // skip non-cgroup mounts (check for both cgroup v1 and v2) + if (strcmp(mnt->mnt_type, "cgroup") != 0 && strcmp(mnt->mnt_type, "cgroup2") != 0) continue; + // cgroupv2 doesn't support cpu.rt_runtime_us in the same way + if (strcmp(mnt->mnt_type, "cgroup2") == 0) + { + AI_LOG_INFO("cgroup v2 detected, CPU RT runtime defaults to disabled"); + isCgroupV2 = true; + break; + } + // check if a cpu cgroup mount char* mntopt = hasmntopt(mnt, "cpu"); if (!mntopt || (strncmp(mntopt, "cpu", 3) != 0)) @@ -700,3 +709,4 @@ bool DobbyTemplate::applyAt(int dirFd, const std::string& fileName, { return instance()->_applyAt(dirFd, fileName, dictionary, prettyPrint); } + diff --git a/daemon/lib/source/DobbyEnv.cpp b/daemon/lib/source/DobbyEnv.cpp index 5bec4286a..0a17c75e1 100644 --- a/daemon/lib/source/DobbyEnv.cpp +++ b/daemon/lib/source/DobbyEnv.cpp @@ -166,31 +166,54 @@ std::map DobbyEnv::getCgroupMountPoints() struct mntent* mnt; char buf[PATH_MAX + 256]; + // Check for cgroupv2 unified hierarchy first + bool isCgroupV2 = false; while ((mnt = getmntent_r(procMounts, &mntBuf, buf, sizeof(buf))) != nullptr) { - // skip entries that don't have a mountpount, type or options - if (!mnt->mnt_type || !mnt->mnt_dir || !mnt->mnt_opts) - continue; - - // skip non-cgroup mounts - if (strcmp(mnt->mnt_type, "cgroup") != 0) - continue; - - // check for the cgroup type - for (const std::pair cgroup : cgroupNames) + if (mnt->mnt_type && strcmp(mnt->mnt_type, "cgroup2") == 0) { - char* mntopt = hasmntopt(mnt, cgroup.first.c_str()); - if (!mntopt) + isCgroupV2 = true; + AI_LOG_INFO("detected cgroup v2 unified hierarchy @ '%s'", mnt->mnt_dir); + // For cgroupv2, all controllers are at the same mount point + std::string cgroupPath = mnt->mnt_dir; + for (const auto& cgroup : cgroupNames) + { + mounts[cgroup.second] = cgroupPath; + } + break; + } + } + + // Reset to scan for cgroupv1 if not v2 + if (!isCgroupV2) + { + rewind(procMounts); + while ((mnt = getmntent_r(procMounts, &mntBuf, buf, sizeof(buf))) != nullptr) + { + // skip entries that don't have a mountpount, type or options + if (!mnt->mnt_type || !mnt->mnt_dir || !mnt->mnt_opts) continue; - if (strcmp(mntopt, cgroup.first.c_str()) != 0) + // skip non-cgroup mounts + if (strcmp(mnt->mnt_type, "cgroup") != 0) continue; - AI_LOG_INFO("found cgroup '%s' mounted @ '%s'", - cgroup.first.c_str(), mnt->mnt_dir); + // check for the cgroup type + for (const std::pair cgroup : cgroupNames) + { + char* mntopt = hasmntopt(mnt, cgroup.first.c_str()); + if (!mntopt) + continue; - mounts[cgroup.second] = mnt->mnt_dir; - break; + if (strcmp(mntopt, cgroup.first.c_str()) != 0) + continue; + + AI_LOG_INFO("found cgroup '%s' mounted @ '%s'", + cgroup.first.c_str(), mnt->mnt_dir); + + mounts[cgroup.second] = mnt->mnt_dir; + break; + } } } @@ -200,3 +223,4 @@ std::map DobbyEnv::getCgroupMountPoints() return mounts; } + diff --git a/rdkPlugins/GPU/source/GpuPlugin.cpp b/rdkPlugins/GPU/source/GpuPlugin.cpp index 3edf4b041..164b3370f 100644 --- a/rdkPlugins/GPU/source/GpuPlugin.cpp +++ b/rdkPlugins/GPU/source/GpuPlugin.cpp @@ -194,10 +194,17 @@ std::string GpuPlugin::getGpuCgroupMountPoint() if (!mnt->mnt_dir || !mnt->mnt_type || !mnt->mnt_opts) continue; - // skip non-cgroup mounts - if (strcmp(mnt->mnt_type, "cgroup") != 0) + // skip non-cgroup mounts (check for both cgroup v1 and v2) + if (strcmp(mnt->mnt_type, "cgroup") != 0 && strcmp(mnt->mnt_type, "cgroup2") != 0) continue; + // cgroupv2 doesn't have gpu cgroup controller + if (strcmp(mnt->mnt_type, "cgroup2") == 0) + { + AI_LOG_WARN("cgroup v2 detected, GPU cgroup controller not supported"); + continue; + } + // check for the cgroup type char *mntopt = hasmntopt(mnt, "gpu"); if (!mntopt || strcmp(mntopt, "gpu") != 0) @@ -279,3 +286,4 @@ bool GpuPlugin::setupContainerGpuLimit(const std::string cgroupDirPath, } // End private methods + diff --git a/rdkPlugins/IONMemory/source/IonMemoryPlugin.cpp b/rdkPlugins/IONMemory/source/IonMemoryPlugin.cpp index 77d285fc1..4ea4c5650 100644 --- a/rdkPlugins/IONMemory/source/IonMemoryPlugin.cpp +++ b/rdkPlugins/IONMemory/source/IonMemoryPlugin.cpp @@ -208,10 +208,17 @@ std::string IonMemoryPlugin::findIonCGroupMountPoint() const if (!mnt->mnt_type || !mnt->mnt_dir || !mnt->mnt_opts) continue; - // skip non-cgroup mounts - if (strcmp(mnt->mnt_type, "cgroup") != 0) + // skip non-cgroup mounts (check for both cgroup v1 and v2) + if (strcmp(mnt->mnt_type, "cgroup") != 0 && strcmp(mnt->mnt_type, "cgroup2") != 0) continue; + // cgroupv2 doesn't have ion cgroup controller + if (strcmp(mnt->mnt_type, "cgroup2") == 0) + { + AI_LOG_WARN("cgroup v2 detected, ION cgroup controller not supported"); + continue; + } + // check if the ion cgroup char *mntopt = hasmntopt(mnt, "ion"); if (!mntopt || (strcmp(mntopt, "ion") != 0)) @@ -350,3 +357,4 @@ bool IonMemoryPlugin::setupContainerIonLimits(const std::string &cGroupDirPath, AI_LOG_FN_EXIT(); return true; } + diff --git a/tests/L2_testing/test_runner/memcr_tests.py b/tests/L2_testing/test_runner/memcr_tests.py index e72b49403..900a3a658 100644 --- a/tests/L2_testing/test_runner/memcr_tests.py +++ b/tests/L2_testing/test_runner/memcr_tests.py @@ -102,7 +102,9 @@ def get_container_pids(container_id): return [] info_json = json.loads(process.stdout) - return info_json.get("pids") + pids = info_json.get("pids") + # Handle case where pids is None + return pids if pids is not None else [] def get_checkpointed_pids(memcr_dump_dir = "/media/apps/memcr/"): @@ -280,3 +282,4 @@ def execute_test(): if __name__ == "__main__": test_utils.parse_arguments(__file__) execute_test() + diff --git a/tests/L2_testing/test_runner/pid_limit_tests.py b/tests/L2_testing/test_runner/pid_limit_tests.py index 6512707f5..cc89b64fe 100644 --- a/tests/L2_testing/test_runner/pid_limit_tests.py +++ b/tests/L2_testing/test_runner/pid_limit_tests.py @@ -17,6 +17,11 @@ import test_utils from pathlib import Path +import os + +def is_cgroupv2(): + """Check if the system is using cgroup v2 (unified hierarchy)""" + return os.path.exists('/sys/fs/cgroup/cgroup.controllers') tests = [ test_utils.Test("Pid limit default", @@ -87,18 +92,44 @@ def validate_pid_limit(container_id, expected_output): pid_limit = 0 - # check pids.max present in containers pid cgroup - path = Path("/sys/fs/cgroup/pids/" + container_id + "/pids.max") - if not path.is_file(): - return False, "%s not found" % path.absolute() + # Try multiple possible cgroup paths (cgroupv1 and cgroupv2) + possible_paths = [] + + if is_cgroupv2(): + # cgroupv2 unified hierarchy paths + possible_paths.extend([ + Path(f"/sys/fs/cgroup/dobby/{container_id}/pids.max"), + Path(f"/sys/fs/cgroup/user.slice/dobby/{container_id}/pids.max"), + Path(f"/sys/fs/cgroup/{container_id}/pids.max"), + ]) + + # cgroupv1 paths + possible_paths.extend([ + Path(f"/sys/fs/cgroup/pids/{container_id}/pids.max"), + Path(f"/sys/fs/cgroup/pids/dobby/{container_id}/pids.max"), + ]) + + path = None + for p in possible_paths: + if p.is_file(): + path = p + break + + if path is None: + tried_paths = ', '.join(str(p) for p in possible_paths) + return False, f"pids.max not found. Tried: {tried_paths}" with open(path, 'r') as fh: pid_limit = fh.readline().strip() + # cgroupv2 uses 'max' for unlimited + if pid_limit == 'max': + pid_limit = 'max' + if expected_output == pid_limit: return True, "Test passed" else: - return False, "Pid limit different then expected (expected: '%s', actual: '%s')" % (expected_output, pid_limit) + return False, "Pid limit different than expected (expected: '%s', actual: '%s')" % (expected_output, pid_limit) if __name__ == "__main__": diff --git a/tests/L2_testing/test_runner/test_utils.py b/tests/L2_testing/test_runner/test_utils.py old mode 100755 new mode 100644 index 7fa7f27bc..4c8bd9ab9 --- a/tests/L2_testing/test_runner/test_utils.py +++ b/tests/L2_testing/test_runner/test_utils.py @@ -16,6 +16,7 @@ # limitations under the License. from os import path +import os import subprocess from time import sleep from enum import IntEnum @@ -37,6 +38,57 @@ ) +def is_cgroupv2(): + """Check if the system is using cgroup v2 (unified hierarchy)""" + return os.path.exists('/sys/fs/cgroup/cgroup.controllers') + + +def patch_bundle_for_ci(bundle_path): + """Patch a bundle's config.json to work in CI environments (GitHub Actions). + + Fixes: + - Removes user namespace if cgroupv2 (causes conflicts) + - Removes swappiness settings (not supported in cgroupv2) + - Updates cgroup mount type for cgroupv2 + """ + config_path = os.path.join(bundle_path, "config.json") + if not os.path.exists(config_path): + print_log(f"Config not found at {config_path}, skipping patch", Severity.warning) + return + + try: + with open(config_path, 'r') as f: + config = json.load(f) + + modified = False + + # For cgroupv2 systems, we may need to adjust settings + if is_cgroupv2(): + # Remove swappiness if present (not supported in cgroupv2) + if 'linux' in config and 'resources' in config['linux']: + resources = config['linux']['resources'] + if 'memory' in resources and 'swappiness' in resources['memory']: + del resources['memory']['swappiness'] + modified = True + print_log("Removed swappiness (not supported in cgroupv2)", Severity.debug) + + # Update cgroup mount type if needed + if 'mounts' in config: + for mount in config['mounts']: + if mount.get('destination') == '/sys/fs/cgroup' and mount.get('type') == 'cgroup': + # cgroupv2 should use cgroup2 type, but 'cgroup' often works via auto-detect + # Add bind option to help with compatibility + pass # crun handles this automatically + + if modified: + with open(config_path, 'w') as f: + json.dump(config, f, indent=3) + print_log(f"Patched bundle config at {config_path}", Severity.debug) + + except Exception as e: + print_log(f"Error patching bundle: {e}", Severity.warning) + + class untar_bundle: """Context manager for working with tarball bundles""" def __init__(self, container_id): @@ -48,6 +100,9 @@ def __init__(self, container_id): get_bundle_path(""), "-zxvf", self.path + ".tar.gz"]) + + # Patch bundle for CI compatibility (cgroupv2, etc.) + patch_bundle_for_ci(self.path) def __enter__(self): return self.path From deb892b012d76f52531b3e15dbb29e53f747f073 Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Mon, 30 Mar 2026 06:47:46 +0000 Subject: [PATCH 19/21] add patch_bundle func --- tests/L2_testing/test_runner/test_utils.py | 56 ++++++++++++++++------ 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/tests/L2_testing/test_runner/test_utils.py b/tests/L2_testing/test_runner/test_utils.py index 4c8bd9ab9..312dbdf6e 100644 --- a/tests/L2_testing/test_runner/test_utils.py +++ b/tests/L2_testing/test_runner/test_utils.py @@ -46,10 +46,12 @@ def is_cgroupv2(): def patch_bundle_for_ci(bundle_path): """Patch a bundle's config.json to work in CI environments (GitHub Actions). - Fixes: - - Removes user namespace if cgroupv2 (causes conflicts) + Fixes for cgroupv2: - Removes swappiness settings (not supported in cgroupv2) - - Updates cgroup mount type for cgroupv2 + - Removes kernel memory limit (not supported in cgroupv2) + - Removes kmemTCPLimit (not supported in cgroupv2) + - Removes realtimeRuntime/realtimePeriod if not supported + - Updates cgroup mount to bind mount for cgroupv2 """ config_path = os.path.join(bundle_path, "config.json") if not os.path.exists(config_path): @@ -62,23 +64,49 @@ def patch_bundle_for_ci(bundle_path): modified = False - # For cgroupv2 systems, we may need to adjust settings + # For cgroupv2 systems, we need to remove unsupported settings if is_cgroupv2(): - # Remove swappiness if present (not supported in cgroupv2) if 'linux' in config and 'resources' in config['linux']: resources = config['linux']['resources'] - if 'memory' in resources and 'swappiness' in resources['memory']: - del resources['memory']['swappiness'] - modified = True - print_log("Removed swappiness (not supported in cgroupv2)", Severity.debug) + + # Remove memory settings not supported in cgroupv2 + if 'memory' in resources: + memory = resources['memory'] + unsupported_memory = ['swappiness', 'kernel', 'kernelTCP', 'disableOOMKiller'] + for setting in unsupported_memory: + if setting in memory: + del memory[setting] + modified = True + print_log(f"Removed memory.{setting} (not supported in cgroupv2)", Severity.debug) + + # Clean up empty memory section + if not memory: + del resources['memory'] + modified = True + + # Remove CPU realtime settings if not supported + if 'cpu' in resources: + cpu = resources['cpu'] + # Check if RT scheduling is available + if not os.path.exists('/sys/fs/cgroup/cpu.rt_runtime_us'): + unsupported_cpu = ['realtimeRuntime', 'realtimePeriod'] + for setting in unsupported_cpu: + if setting in cpu: + del cpu[setting] + modified = True + print_log(f"Removed cpu.{setting} (RT not supported)", Severity.debug) - # Update cgroup mount type if needed + # Fix cgroup mount for cgroupv2 - use bind mount instead if 'mounts' in config: for mount in config['mounts']: - if mount.get('destination') == '/sys/fs/cgroup' and mount.get('type') == 'cgroup': - # cgroupv2 should use cgroup2 type, but 'cgroup' often works via auto-detect - # Add bind option to help with compatibility - pass # crun handles this automatically + if mount.get('destination') == '/sys/fs/cgroup': + if mount.get('type') == 'cgroup': + # Change to bind mount for cgroupv2 compatibility + mount['type'] = 'bind' + mount['source'] = '/sys/fs/cgroup' + mount['options'] = ['rbind', 'nosuid', 'noexec', 'nodev', 'ro'] + modified = True + print_log("Changed cgroup mount to bind mount for cgroupv2", Severity.debug) if modified: with open(config_path, 'w') as f: From db3f0fd7ac9e8b3fd969d97467ebcf4729a4d788 Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Mon, 30 Mar 2026 07:08:13 +0000 Subject: [PATCH 20/21] dns config fix --- .github/workflows/L2-tests.yml | 39 +++++++++++++++++++++--- rdkPlugins/Networking/source/Netlink.cpp | 33 ++++++++++++++++++-- 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/L2-tests.yml index 09248c3ca..c153a182e 100644 --- a/.github/workflows/L2-tests.yml +++ b/.github/workflows/L2-tests.yml @@ -123,17 +123,36 @@ jobs: sudo ip link add enp0s3 type dummy || true sudo ip link set enp0s8 up || true sudo ip link set enp0s3 up || true + # Create bridge for Dobby networking sudo ip link add dobby0 type bridge || true sudo ip link set dobby0 up || true # Assign IP to dobby0 bridge for container networking sudo ip addr add 100.64.11.1/24 dev dobby0 || true + # Enable IP forwarding for container network access sudo sysctl -w net.ipv4.ip_forward=1 - # Setup NAT for container outbound traffic - sudo iptables -t nat -A POSTROUTING -s 100.64.11.0/24 -o eth0 -j MASQUERADE || true - sudo iptables -A FORWARD -i dobby0 -o eth0 -j ACCEPT || true - sudo iptables -A FORWARD -i eth0 -o dobby0 -m state --state RELATED,ESTABLISHED -j ACCEPT || true + + # Detect the real default-route interface (not hardcoded eth0) + EXT_IF=$(ip -o -4 route show to default | awk '{print $5}' | head -n1) + echo "Detected external interface: $EXT_IF" + + # Setup NAT for container outbound traffic using detected interface + sudo iptables -t nat -A POSTROUTING -s 100.64.11.0/24 -o "$EXT_IF" -j MASQUERADE || true + sudo iptables -A FORWARD -i dobby0 -o "$EXT_IF" -j ACCEPT || true + sudo iptables -A FORWARD -i "$EXT_IF" -o dobby0 -m state --state RELATED,ESTABLISHED -j ACCEPT || true + + # Force a resolv.conf that works well in CI/container contexts + # (avoid systemd stub resolver issues) + sudo rm -f /etc/resolv.conf + printf "nameserver 1.1.1.1\nnameserver 8.8.8.8\n" | sudo tee /etc/resolv.conf + + # Verify network setup + echo "=== Network configuration ===" + ip addr show dobby0 + cat /etc/resolv.conf + echo "=== iptables NAT ===" + sudo iptables -t nat -L POSTROUTING -n -v | head -5 - name: Setup permissions and directories run: | @@ -147,7 +166,8 @@ jobs: - name: build Dobby run: | - sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf + # Don't override resolv.conf - we already set it up in network setup step + # sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf cd Dobby sudo cp /usr/lib/x86_64-linux-gnu/dbus-1.0/include/dbus/dbus-arch-deps.h /usr/include/dbus-1.0/dbus sudo mkdir -p /usr/lib/plugins/dobby @@ -252,6 +272,15 @@ jobs: - name: Run the l2 test working-directory: Dobby/tests/L2_testing/test_runner/ run: | + # Redirect GCC coverage output to a writable path + export GCOV_PREFIX=/tmp/gcov + export GCOV_PREFIX_STRIP=3 + mkdir -p /tmp/gcov + + # Verify DNS is working before running tests + echo "Testing DNS resolution..." + nslookup example.com || echo "DNS lookup warning (may affect some tests)" + python3 runner.py -p 3 -v 5 cp $GITHUB_WORKSPACE/Dobby/tests/L2_testing/test_runner/DobbyL2TestResults.json $GITHUB_WORKSPACE diff --git a/rdkPlugins/Networking/source/Netlink.cpp b/rdkPlugins/Networking/source/Netlink.cpp index 3c8dec07a..35688c859 100644 --- a/rdkPlugins/Networking/source/Netlink.cpp +++ b/rdkPlugins/Networking/source/Netlink.cpp @@ -598,6 +598,20 @@ bool Netlink::setLinkAddress(const NlLink& link, const in_addr_t address, { AI_LOG_FN_ENTRY(); + // Validate the link has a valid interface index + if (!link) + { + AI_LOG_ERROR_EXIT("invalid link object (null)"); + return false; + } + + int ifindex = rtnl_link_get_ifindex(link); + if (ifindex <= 0) + { + AI_LOG_ERROR_EXIT("invalid interface index %d for link", ifindex); + return false; + } + // create the link route address NlRouteAddress addr(address, netmask); if (!addr) @@ -606,7 +620,7 @@ bool Netlink::setLinkAddress(const NlLink& link, const in_addr_t address, return false; } - AI_LOG_INFO("setting link address to '%s'", addr.toString().c_str()); + AI_LOG_INFO("setting link address to '%s' on ifindex %d", addr.toString().c_str(), ifindex); // set the link index rtnl_addr_set_link(addr, link); @@ -643,6 +657,20 @@ bool Netlink::setLinkAddress(const NlLink& link, const struct in6_addr address, { AI_LOG_FN_ENTRY(); + // Validate the link has a valid interface index + if (!link) + { + AI_LOG_ERROR_EXIT("invalid link object (null)"); + return false; + } + + int ifindex = rtnl_link_get_ifindex(link); + if (ifindex <= 0) + { + AI_LOG_ERROR_EXIT("invalid interface index %d for link", ifindex); + return false; + } + // create the link route address NlRouteAddress addr(address, netmask); if (!addr) @@ -651,7 +679,7 @@ bool Netlink::setLinkAddress(const NlLink& link, const struct in6_addr address, return false; } - AI_LOG_INFO("setting link address to '%s'", addr.toString().c_str()); + AI_LOG_INFO("setting link address to '%s' on ifindex %d", addr.toString().c_str(), ifindex); // set the link index rtnl_addr_set_link(addr, link); @@ -2109,3 +2137,4 @@ bool Netlink::delArpEntry(const std::string &iface, const in_addr_t address) AI_LOG_FN_EXIT(); return true; } + From b6d69d28eba96a933e95d97fb7349d33e83617a8 Mon Sep 17 00:00:00 2001 From: Jeyasona Date: Mon, 30 Mar 2026 08:25:55 +0000 Subject: [PATCH 21/21] gcov fix --- .github/workflows/L2-tests.yml | 28 ++++++++++++++++++---- tests/L2_testing/test_runner/test_utils.py | 14 +++++++---- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/L2-tests.yml index c153a182e..5bb0ed150 100644 --- a/.github/workflows/L2-tests.yml +++ b/.github/workflows/L2-tests.yml @@ -3,6 +3,9 @@ on: [push, pull_request] env: MEMCR_REF: "b58f2b8e26cab6b67eceaa36fd6ce5a6d04dcd28" + # GCOV settings for coverage - must be at workflow level so all processes inherit + GCOV_PREFIX: /tmp/gcov + GCOV_PREFIX_STRIP: "3" jobs: build: @@ -163,6 +166,19 @@ jobs: # Create tmp directory for container logs sudo mkdir -p /tmp sudo chmod 1777 /tmp + + # Setup GCOV output directory - CRITICAL for coverage builds + # The hook processes (DobbyPluginLauncher) need this to be writable + sudo mkdir -p /tmp/gcov + sudo chmod -R 777 /tmp/gcov + + # Make GCOV env vars available to ALL processes including those started by sudo/systemd + echo "GCOV_PREFIX=/tmp/gcov" | sudo tee -a /etc/environment + echo "GCOV_PREFIX_STRIP=3" | sudo tee -a /etc/environment + + # Also export to current shell for subsequent steps + echo "GCOV_PREFIX=/tmp/gcov" >> $GITHUB_ENV + echo "GCOV_PREFIX_STRIP=3" >> $GITHUB_ENV - name: build Dobby run: | @@ -272,15 +288,19 @@ jobs: - name: Run the l2 test working-directory: Dobby/tests/L2_testing/test_runner/ run: | - # Redirect GCC coverage output to a writable path - export GCOV_PREFIX=/tmp/gcov - export GCOV_PREFIX_STRIP=3 - mkdir -p /tmp/gcov + # Verify GCOV env is set (should be inherited from job-level env) + echo "GCOV_PREFIX=$GCOV_PREFIX" + echo "GCOV_PREFIX_STRIP=$GCOV_PREFIX_STRIP" + + # Ensure directory exists and is writable + sudo mkdir -p /tmp/gcov + sudo chmod -R 777 /tmp/gcov # Verify DNS is working before running tests echo "Testing DNS resolution..." nslookup example.com || echo "DNS lookup warning (may affect some tests)" + # Run tests - GCOV_PREFIX is inherited from job env python3 runner.py -p 3 -v 5 cp $GITHUB_WORKSPACE/Dobby/tests/L2_testing/test_runner/DobbyL2TestResults.json $GITHUB_WORKSPACE diff --git a/tests/L2_testing/test_runner/test_utils.py b/tests/L2_testing/test_runner/test_utils.py index 312dbdf6e..032ef68fb 100644 --- a/tests/L2_testing/test_runner/test_utils.py +++ b/tests/L2_testing/test_runner/test_utils.py @@ -156,12 +156,18 @@ def __init__(self, log_to_stdout = False): print_log("Starting Dobby Daemon (logging to Journal)...", Severity.debug) + # Build environment with GCOV settings to ensure coverage data goes to writable location + # This is critical for CI where hooks like DobbyPluginLauncher need to write .gcda files + daemon_env = os.environ.copy() + daemon_env["GCOV_PREFIX"] = "/tmp/gcov" + daemon_env["GCOV_PREFIX_STRIP"] = "3" + if log_to_stdout: - cmd = ["sudo", "DobbyDaemon", "--nofork"] - kvargs = {"universal_newlines": True} + cmd = ["sudo", "-E", "DobbyDaemon", "--nofork"] + kvargs = {"universal_newlines": True, "env": daemon_env} else: - cmd = ["sudo", "DobbyDaemon", "--nofork", "--journald", "--noconsole"] - kvargs = {"universal_newlines": True, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE} + cmd = ["sudo", "-E", "DobbyDaemon", "--nofork", "--journald", "--noconsole"] + kvargs = {"universal_newlines": True, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "env": daemon_env} # as this process is running infinitely we cannot use run_command_line as it waits for execution to end self.subproc = subprocess.Popen(cmd, **kvargs)