MINIFICPP-2745 Upgrade OPC UA library to v1.5.2

Author: lordgamez

cmake/BundledOpen62541.cmake

@@ -34,7 +34,7 @@ function(use_bundled_open62541 SOURCE_DIR BINARY_DIR)
 
     set(OPEN62541_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS}
             "-DCMAKE_INSTALL_PREFIX=${OPEN62541_BYPRODUCT_DIR}"
-            -DOPEN62541_VERSION=v1.3.3
+            -DOPEN62541_VERSION=v1.5.2
             -DUA_ENABLE_ENCRYPTION=ON
             -DUA_FORCE_WERROR=OFF
             -DUA_ENABLE_DEBUG_SANITIZER=OFF)
@@ -44,8 +44,8 @@ function(use_bundled_open62541 SOURCE_DIR BINARY_DIR)
     # Build project
     ExternalProject_Add(
             open62541-external
-            URL "https://github.com/open62541/open62541/archive/refs/tags/v1.4.10.tar.gz"
-            URL_HASH "SHA256=1a2e762e50bb6dae8d80029dfb66fdbc432876a004e62d618f7cf1bb5b4f495f"
+            URL "https://github.com/open62541/open62541/archive/refs/tags/v1.5.2.tar.gz"
+            URL_HASH "SHA256=61110a51c4f00a70a6e47882ea92550b667eede62e66c64431fdc37ff660361c"
             SOURCE_DIR "${BINARY_DIR}/thirdparty/open62541-src"
             PATCH_COMMAND ${PC}
             LIST_SEPARATOR % # This is needed for passing semicolon-separated lists

extensions/opc/src/opc.cpp

@@ -33,6 +33,8 @@
 #include "open62541/client_highlevel.h"
 #include "open62541/client_config_default.h"
 
+extern "C" int mp_vsnprintf(char* s, size_t count, const char* format, va_list arg);
+
 namespace org::apache::nifi::minifi::opc {
 
 /*
@@ -586,7 +588,7 @@ std::string OPCDateTime2String(UA_DateTime raw_date) {
 
 void logFunc(void *context, UA_LogLevel level, UA_LogCategory /*category*/, const char *msg, va_list args) {
   std::array<char, 1024> buffer{};
-  (void)vsnprintf(buffer.data(), buffer.size(), msg, args);
+  (void)mp_vsnprintf(buffer.data(), buffer.size(), msg, args);
   auto loggerPtr = reinterpret_cast<core::logging::Logger*>(context);
   loggerPtr->log_string(MapOPCLogLevel(level), buffer.data());
 }

extensions/opc/tests/OpcUaTestServer.h

@@ -24,6 +24,8 @@
 #include "unit/TestUtils.h"
 #include "unit/Catch.h"
 
+extern "C" int mp_vsnprintf(char* s, size_t count, const char* format, va_list arg);
+
 using namespace std::literals::chrono_literals;
 
 namespace org::apache::nifi::minifi::test {
@@ -37,7 +39,7 @@ class OpcUaTestServer {
     UA_ServerConfig_setMinimal(config, port, nullptr);
     config->logging->log = [] (void *log_context, UA_LogLevel level, UA_LogCategory /*category*/, const char *msg, va_list args) {
       char buffer[1024];
-      vsnprintf(buffer, sizeof(buffer), msg, args);
+      mp_vsnprintf(buffer, sizeof(buffer), msg, args);
 
       std::string level_str;
       switch (level) {

extensions/opc/tests/features/opcua.feature

@@ -115,10 +115,8 @@ Feature: Putting and fetching data to OPC UA server
     And a PutFile processor with the "Directory" property set to "/tmp/output" in the "fetch-opc-ua-node" flow
     And PutFile's success relationship is auto-terminated in the "fetch-opc-ua-node" flow
     And PutFile is EVENT_DRIVEN in the "fetch-opc-ua-node" flow
-    And a host resource file "opcua_client_cert.der" is bound to the "/tmp/resources/opcua/opcua_client_cert.der" path in the MiNiFi container "create-opc-ua-node"
-    And a host resource file "opcua_client_key.der" is bound to the "/tmp/resources/opcua/opcua_client_key.der" path in the MiNiFi container "create-opc-ua-node"
-    And a host resource file "opcua_client_cert.der" is bound to the "/tmp/resources/opcua/opcua_client_cert.der" path in the MiNiFi container "fetch-opc-ua-node"
-    And a host resource file "opcua_client_key.der" is bound to the "/tmp/resources/opcua/opcua_client_key.der" path in the MiNiFi container "fetch-opc-ua-node"
+    And the OPC UA server certificate files are placed in the "/tmp/resources/opcua/" directory in the MiNiFi container "create-opc-ua-node"
+    And the OPC UA server certificate files are placed in the "/tmp/resources/opcua/" directory in the MiNiFi container "fetch-opc-ua-node"
     And these processor properties are set in the "create-opc-ua-node" flow
       | processor name    | property name                   | property value                                    |
       | PutOPCProcessor   | Parent node ID                  | 85                                                |
@@ -132,7 +130,7 @@ Feature: Putting and fetching data to OPC UA server
       | PutOPCProcessor   | Certificate path                | /tmp/resources/opcua/opcua_client_cert.der        |
       | PutOPCProcessor   | Key path                        | /tmp/resources/opcua/opcua_client_key.der         |
       | PutOPCProcessor   | Trusted server certificate path | /tmp/resources/opcua/opcua_client_cert.der        |
-      | PutOPCProcessor   | Application URI                 | urn:open62541.server.application                  |
+      | PutOPCProcessor   | Application URI                 | urn:open62541.unconfigured.application                  |
     And these processor properties are set in the "fetch-opc-ua-node" flow
       | processor name    | property name                   | property value                                    |
       | FetchOPCProcessor | Node ID                         | 9999                                              |
@@ -143,7 +141,7 @@ Feature: Putting and fetching data to OPC UA server
       | FetchOPCProcessor | Certificate path                | /tmp/resources/opcua/opcua_client_cert.der        |
       | FetchOPCProcessor | Key path                        | /tmp/resources/opcua/opcua_client_key.der         |
       | FetchOPCProcessor | Trusted server certificate path | /tmp/resources/opcua/opcua_client_cert.der        |
-      | FetchOPCProcessor | Application URI                 | urn:open62541.server.application                  |
+      | FetchOPCProcessor | Application URI                 | urn:open62541.unconfigured.application                  |
 
     And in the "create-opc-ua-node" flow the "success" relationship of the GetFile processor is connected to the PutOPCProcessor
     And in the "fetch-opc-ua-node" flow the "success" relationship of the FetchOPCProcessor processor is connected to the PutFile
@@ -153,7 +151,7 @@ Feature: Putting and fetching data to OPC UA server
     When all instances start up
 
     Then in the "fetch-opc-ua-node" container at least one file with the content "Test" is placed in the "/tmp/output" directory in less than 60 seconds
-    And the OPC UA server logs contain the following message: "SecureChannel opened with SecurityPolicy http://opcfoundation.org/UA/SecurityPolicy#Aes128_Sha256_RsaOaep" in less than 5 seconds
+    And the OPC UA server logs contain the following message: "Channel opened with SecurityMode SignAndEncrypt for SecurityPolicy http://opcfoundation.org/UA/SecurityPolicy#Aes256_Sha256_RsaPss" in less than 5 seconds
 
   Scenario: Create and fetch data from an OPC UA node through username and password authenticated connection
     Given a GetFile processor with the "Input Directory" property set to "/tmp/input" in the "create-opc-ua-node" flow

extensions/opc/tests/features/resources/opcua_client_cert.der

@@ -20,8 +20,10 @@
 
 
 class OPCUAServerContainer(Container):
+    OPC_SERVER_IMAGE = "lordgamez/open62541:1.5.2"
+
     def __init__(self, test_context: MinifiTestContext, command: Optional[List[str]] = None):
-        super().__init__("lordgamez/open62541:1.4.10", f"opcua-server-{test_context.scenario_id}", test_context.network, command=command)
+        super().__init__(self.OPC_SERVER_IMAGE, f"opcua-server-{test_context.scenario_id}", test_context.network, command=command)
 
     def deploy(self):
         super().deploy()

extensions/opc/tests/features/resources/opcua_client_key.der

@@ -13,9 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
+import os
+import tempfile
+import docker
+import io
+import tarfile
 import humanfriendly
-from behave import step, then
 
+from behave import given, step, then
 from minifi_test_framework.steps import checking_steps        # noqa: F401
 from minifi_test_framework.steps import configuration_steps   # noqa: F401
 from minifi_test_framework.steps import core_steps            # noqa: F401
@@ -45,3 +51,36 @@ def step_impl(context, log_message, duration):
         timeout_seconds=timeout_seconds,
         bail_condition=lambda: opcua_container.exited,
         context=context)
+
+
+def _copy_file_from_docker_image(image_name: str, file_path: str, output_path: str):
+    docker_client: docker.DockerClient = docker.from_env()
+    container = docker_client.containers.create(image_name)
+
+    try:
+        bits, _ = container.get_archive(file_path)
+        tar_stream = io.BytesIO(b"".join(bits))
+        with tarfile.open(fileobj=tar_stream) as tar:
+            member = tar.getmembers()[0]
+            file_content = tar.extractfile(member).read()
+
+        with open(output_path, "wb") as f:
+            f.write(file_content)
+
+        return True
+    except Exception as e:
+        logging.error(f"Error copying file {file_path} from Docker image {image_name}: {e}")
+        return False
+    finally:
+        container.remove(force=True)
+
+
+@given('the OPC UA server certificate files are placed in the "{directory}" directory in the MiNiFi container "{container_name}"')
+def step_impl(context: MinifiTestContext, directory: str, container_name: str):
+    if not hasattr(context, "opcua_cert_temp_dir"):
+        context.opcua_cert_temp_dir = tempfile.TemporaryDirectory()
+        _copy_file_from_docker_image(OPCUAServerContainer.OPC_SERVER_IMAGE, "/opt/open62541/pki/created/server_cert.der", os.path.join(context.opcua_cert_temp_dir.name, "server_cert.der"))
+        _copy_file_from_docker_image(OPCUAServerContainer.OPC_SERVER_IMAGE, "/opt/open62541/pki/created/server_key.der", os.path.join(context.opcua_cert_temp_dir.name, "server_key.der"))
+
+    context.get_or_create_minifi_container(container_name).add_host_file(os.path.join(context.opcua_cert_temp_dir.name, "server_cert.der"), os.path.join(directory, "opcua_client_cert.der"))
+    context.get_or_create_minifi_container(container_name).add_host_file(os.path.join(context.opcua_cert_temp_dir.name, "server_key.der"), os.path.join(directory, "opcua_client_key.der"))

extensions/opc/tests/features/steps/opc_ua_server_container.py

@@ -1,42 +1,53 @@
 diff --git a/CMakeLists.txt b/CMakeLists.txt
-index d0e9d75e2..bbf1b4c9a 100644
+index e9baae1ec..954909854 100644
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -11,7 +11,7 @@ endif()
+@@ -18,7 +18,7 @@ endif()
 
- string(TOLOWER "${CMAKE_BUILD_TYPE}" BUILD_TYPE_LOWER_CASE)
+ # set(CMAKE_VERBOSE_MAKEFILE ON)
 
 -set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/tools/cmake")
 +list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/tools/cmake")
- if(${CMAKE_VERSION} VERSION_LESS 3.12)
-     set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${PROJECT_SOURCE_DIR}/tools/cmake3.12")
- endif()
-@@ -642,17 +642,17 @@ if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_C_COMPILER_ID STREQUAL "Clang")
 
-         # IPO requires too much memory for unit tests
-         # GCC docu recommends to compile all files with the same options, therefore ignore it completely
--        if(NOT UA_BUILD_UNIT_TESTS AND NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION)
--            # needed to check if IPO is supported (check needs cmake > 3.9)
--            if("${CMAKE_VERSION}" VERSION_GREATER 3.9)
--                cmake_policy(SET CMP0069 NEW) # needed as long as required cmake < 3.9
--                include(CheckIPOSupported)
--                check_ipo_supported(RESULT CC_HAS_IPO) # Inter Procedural Optimization / Link Time Optimization (should be same as -flto)
--                if(CC_HAS_IPO)
--                    set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON)
--                endif()
--            endif()
--        endif()
-+	# if(NOT UA_BUILD_UNIT_TESTS AND NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION)
-+	#     # needed to check if IPO is supported (check needs cmake > 3.9)
-+	#     if("${CMAKE_VERSION}" VERSION_GREATER 3.9)
-+	#         cmake_policy(SET CMP0069 NEW) # needed as long as required cmake < 3.9
-+	#         include(CheckIPOSupported)
-+	#         check_ipo_supported(RESULT CC_HAS_IPO) # Inter Procedural Optimization / Link Time Optimization (should be same as -flto)
-+	#         if(CC_HAS_IPO)
-+	#             set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON)
-+	#         endif()
-+	#     endif()
-+	# endif()
+ find_package(Python3 REQUIRED)
+ find_package(Git)
+@@ -669,7 +669,6 @@ if((CMAKE_COMPILER_IS_GNUCC OR CMAKE_C_COMPILER_ID STREQUAL "Clang") AND
+     # Threading support
+     if(UA_MULTITHREADING GREATER_EQUAL 100 AND NOT WIN32)
+         check_add_cc_flag("-pthread")
+-        check_add_cc_flag("-ftls-model=initial-exec") # Enable thread-local variables in the .so shared object
+     endif()
+ 
+     # Force 32bit build
+@@ -691,17 +690,6 @@ if((CMAKE_COMPILER_IS_GNUCC OR CMAKE_C_COMPILER_ID STREQUAL "Clang") AND
+         endif()
      endif()
 
-     if(UA_ENABLE_AMALGAMATION)
+-    # Inter Procedural Optimization / Link Time Optimization (should be same as -flto)
+-    # IPO requires too much memory for unit tests
+-    # GCC docu recommends to compile all files with the same options, therefore ignore it completely
+-    if(NOT UA_BUILD_UNIT_TESTS AND NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION)
+-        include(CheckIPOSupported)
+-        check_ipo_supported(RESULT CC_HAS_IPO)
+-        if(CC_HAS_IPO)
+-            set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON)
+-        endif()
+-    endif()
+-
+     # Linker
+     set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") # cmake sets -rdynamic by default
+     if(APPLE)
+diff --git a/arch/posix/eventloop_posix.h b/arch/posix/eventloop_posix.h
+index a5f51383e..0d8b78418 100644
+--- a/arch/posix/eventloop_posix.h
++++ b/arch/posix/eventloop_posix.h
+@@ -277,7 +277,9 @@ typedef int SOCKET;
+ 
+ #ifndef __ANDROID__
+ #ifndef __APPLE__
++#ifdef __GLIBC__
+ #include <bits/stdio_lim.h>
++#endif
+ #endif /* !__APPLE__ */
+ #endif /* !__ANDROID__ */
+