Feature/1445 async shutdown pending events warning

.github/workflows/build-packages.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        BUILD_ARCH: ["x86-64", "arm64"]
+        BUILD_ARCH: ["x86-64"]
     env:
       BUILD_ARCH: ${{ matrix.BUILD_ARCH }}
     steps:

.github/workflows/release.yml

@@ -16,21 +16,10 @@ env:
 
 jobs:
 
-  release-started:
-    runs-on: ubuntu-latest
-    if: startsWith(github.ref, 'refs/tags')
-    steps:
-      - uses: elastic/oblt-actions/slack/send@v1
-        with:
-          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
-          channel-id: "#apm-agent-php"
-          message: |
-            :runner: [${{ github.repository }}] Release *${{ github.ref_name }}* has been triggered : (<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here> for further details)
-
   build:
     uses: ./.github/workflows/build.yml
     with:
-      build_arch: all
+      build_arch: x86_64
 
   build-packages:
     permissions:
@@ -40,17 +29,15 @@ jobs:
       - build
     uses: ./.github/workflows/build-packages.yml
 
-  sign:
-    runs-on: ubuntu-latest
+  release:
     needs:
       - build-packages
-    env:
-      BUCKET_NAME: "apm-agent-php"
+    runs-on: ubuntu-latest
     permissions:
-      artifact-metadata: write
-      attestations: write
       contents: write
-      id-token: write
+    env:
+      GITHUB_TOKEN: ${{ github.token }}
+      TAG_NAME: ${{ github.ref_name }}
     steps:
       - uses: actions/checkout@v6
 
@@ -59,6 +46,7 @@ jobs:
         with:
           pattern: packages-*
           path: ${{ env.BUILD_PACKAGES }}
+
       - name: Moving packages out of folders
         run: |
           pushd ${{ env.BUILD_PACKAGES }}
@@ -67,122 +55,12 @@ jobs:
           popd
           ls -R ${{ env.BUILD_PACKAGES }}
 
-      - name: generate build provenance
-        uses: actions/attest@v4
-        with:
-          subject-path: "${{ github.workspace }}/${{ env.BUILD_PACKAGES }}/*"
-
-      ## NOTE: The name of the zip should match the name of the folder to be zipped.
-      - name: Prepare packages to be signed
-        run: zip -r packages.zip packages/
-        working-directory: build
-
-      - uses: elastic/oblt-actions/google/auth@v1
-        with:
-          project-number: '911195782929'
-
-      - id: 'upload-file'
-        uses: 'google-github-actions/upload-cloud-storage@v3'
-        with:
-          path: "${{ env.BUILD_PACKAGES }}.zip"
-          destination: "${{ env.BUCKET_NAME }}/${{ github.run_id }}"
-
-      - id: buildkite-run
-        uses: elastic/oblt-actions/buildkite/run@v1
-        with:
-          token: ${{ secrets.BUILDKITE_TOKEN }}
-          pipeline: observability-robots-php-release
-          wait-for: true
-          env-vars: |
-            BUNDLE_URL=https://storage.googleapis.com/${{ env.BUCKET_NAME }}/${{ steps.upload-file.outputs.uploaded }}
-
-      - uses: elastic/oblt-actions/buildkite/download-artifact@v1
-        with:
-          build-number: ${{ steps.buildkite-run.outputs.number }}
-          path: signed-artifacts.zip
-          pipeline: ${{ steps.buildkite-run.outputs.pipeline }}
-          token: ${{ secrets.BUILDKITE_TOKEN }}
-
-      # this artifact name is used also in some other places,
-      # such as ./.github/workflows/test-packages.yml.
-      # Therefore v4 cannot be used at the moment.
-      - uses: actions/upload-artifact@v7
-        with:
-          name: signed-artifacts
-          path: signed-artifacts.zip
-
-  generate-test-packages-matrix:
-    if: startsWith(github.ref, 'refs/tags')
-    uses: ./.github/workflows/generate-matrix.yml
-
-  test-packages:
-    if: startsWith(github.ref, 'refs/tags')
-    needs:
-      - sign
-      - generate-test-packages-matrix
-    permissions:
-      contents: read
-      packages: read
-    uses: ./.github/workflows/test-packages.yml
-    with:
-      include: ${{ needs.generate-test-packages-matrix.outputs.include }}
-      max-parallel: 40
-      package-name: 'signed-artifacts'
-
-  release:
-    needs:
-      - test-packages
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    env:
-      GITHUB_TOKEN: ${{ github.token }}
-      TAG_NAME: ${{ github.ref_name }}
-    steps:
-
-      - uses: actions/checkout@v6
-      - uses: actions/download-artifact@v8
-        with:
-          name: signed-artifacts
-          path: ${{ env.BUILD_PACKAGES }}
-
-      - name: Unzip signed packages
-        run: unzip ${PACKAGE_FILE} && rm ${PACKAGE_FILE}
-        working-directory: ${{ env.BUILD_PACKAGES }}
-        env:
-          PACKAGE_FILE: "signed-artifacts.zip"
-
-      - name: Create draft release
-        if: startsWith(github.ref, 'refs/tags')
-        run: make -f .ci/Makefile draft-release
-
-      - name: Verify draft release
-        if: startsWith(github.ref, 'refs/tags')
-        run: ORIGINAL_PACKAGES_LOCATION=${{ env.BUILD_PACKAGES }} make -f .ci/Makefile download-verify
-
-      - name: Publish release
-        if: startsWith(github.ref, 'refs/tags')
-        run: make -f .ci/Makefile github-release-ready
-
-  notify:
-    if: always()
-    needs:
-      - build
-      - build-packages
-      - generate-test-packages-matrix
-      - release
-      - sign
-      - test-packages
-    runs-on: ubuntu-latest
-    steps:
-      - id: check
-        uses: elastic/oblt-actions/check-dependent-jobs@v1
-        with:
-          jobs: ${{ toJSON(needs) }}
-      - if: startsWith(github.ref, 'refs/tags')
-        uses: elastic/oblt-actions/slack/notify-result@v1
-        with:
-          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
-          channel-id: "#apm-agent-php"
-          status: ${{ steps.check.outputs.status }}
-          message: "[${{ github.repository }}] Release (<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}>)"
+      - name: Create GitHub release
+        run: |
+          gh release create "${TAG_NAME}" \
+            --title "${TAG_NAME}" \
+            --notes "Ecentria fork release. Changes from upstream v1.16.0:
+          - Add configurable max send queue size (#1441)
+          - Fix span ID collisions: use random_bytes() instead of mt_rand() (#1443)" \
+            --repo ${{ github.repository }} \
+            ${{ env.BUILD_PACKAGES }}/*.*

agent/native/ext/ConfigManager.cpp

@@ -117,6 +117,7 @@ typedef struct DurationOptionAdditionalMetadata DurationOptionAdditionalMetadata
 struct SizeOptionAdditionalMetadata
 {
     SizeUnits defaultUnits = sizeUnits_byte;
+    Int64 minValidValueInBytes = INT64_MIN;
 };
 typedef struct SizeOptionAdditionalMetadata SizeOptionAdditionalMetadata;
 
@@ -427,7 +428,14 @@ static ResultCode parseSizeValue( const OptionMetadata* optMeta, String rawValue
     ResultCode parseResultCode = parseSize( stringToView( rawValue )
                                                 , optMeta->additionalData.sizeData.defaultUnits
                                                 , /* out */ &parsedValue->u.sizeValue );
-    if ( parseResultCode == resultSuccess ) parsedValue->type = parsedOptionValueType_size;
+    if ( parseResultCode == resultSuccess )
+    {
+        if ( sizeToBytes( parsedValue->u.sizeValue ) < optMeta->additionalData.sizeData.minValidValueInBytes )
+        {
+            return resultParsingFailed;
+        }
+        parsedValue->type = parsedOptionValueType_size;
+    }
     return parseResultCode;
 }
 
@@ -667,7 +675,7 @@ static OptionMetadata buildDurationOptionMetadata(
     };
 }
 
-[[maybe_unused]] static OptionMetadata buildSizeOptionMetadata(
+static OptionMetadata buildSizeOptionMetadata(
         String name
         , StringView iniName
         , bool isSecret
@@ -676,6 +684,7 @@ static OptionMetadata buildDurationOptionMetadata(
         , SetConfigSnapshotFieldFunc setFieldFunc
         , GetConfigSnapshotFieldFunc getFieldFunc
         , SizeUnits defaultUnits
+        , Int64 minValidValueInBytes
 )
 {
     return (OptionMetadata)
@@ -692,7 +701,7 @@ static OptionMetadata buildDurationOptionMetadata(
         .setField = setFieldFunc,
         .getField = getFieldFunc,
         .parsedValueToZval = &parsedSizeValueToZval,
-        .additionalData = (OptionAdditionalMetadata){ .sizeData = (SizeOptionAdditionalMetadata){ .defaultUnits = defaultUnits } }
+        .additionalData = (OptionAdditionalMetadata){ .sizeData = (SizeOptionAdditionalMetadata){ .defaultUnits = defaultUnits, .minValidValueInBytes = minValidValueInBytes } }
     };
 }
 
@@ -827,6 +836,7 @@ ELASTIC_APM_DEFINE_ENUM_FIELD_ACCESS_FUNCS( LogLevel, logLevelSyslog )
 #   ifdef PHP_WIN32
 ELASTIC_APM_DEFINE_ENUM_FIELD_ACCESS_FUNCS( LogLevel, logLevelWinSysDebug )
 #   endif
+ELASTIC_APM_DEFINE_FIELD_ACCESS_FUNCS( sizeValue, maxSendQueueSize )
 #   if ( ELASTIC_APM_MEMORY_TRACKING_ENABLED_01 != 0 )
 ELASTIC_APM_DEFINE_ENUM_FIELD_ACCESS_FUNCS( MemoryTrackingLevel, memoryTrackingLevel )
 #   endif
@@ -882,6 +892,9 @@ ELASTIC_APM_DEFINE_FIELD_ACCESS_FUNCS( stringValue, debugDiagnosticsFile )
 #define ELASTIC_APM_INIT_DURATION_METADATA( fieldName, optName, defaultValue, defaultUnits, isNegativeValid ) \
     ELASTIC_APM_INIT_METADATA_EX( buildDurationOptionMetadata, fieldName, optName, /* isSecret */ false, /* isDynamic */ false, defaultValue, defaultUnits, isNegativeValid )
 
+#define ELASTIC_APM_INIT_SIZE_METADATA( fieldName, optName, defaultValue, defaultUnits, minValidValueInBytes ) \
+    ELASTIC_APM_INIT_METADATA_EX( buildSizeOptionMetadata, fieldName, optName, /* isSecret */ false, /* isDynamic */ false, defaultValue, defaultUnits, minValidValueInBytes )
+
 #define ELASTIC_APM_INIT_SECRET_METADATA( buildFunc, fieldName, optName, defaultValue ) \
     ELASTIC_APM_INIT_METADATA_EX( buildFunc, fieldName, optName, /* isSecret */ true, /* isDynamic */ false, defaultValue )
 
@@ -1115,6 +1128,13 @@ static void initOptionsMetadata( OptionMetadata* optsMeta )
             ELASTIC_APM_CFG_OPT_NAME_LOG_LEVEL_WIN_SYS_DEBUG );
     #endif
 
+    ELASTIC_APM_INIT_SIZE_METADATA(
+            maxSendQueueSize,
+            ELASTIC_APM_CFG_OPT_NAME_MAX_SEND_QUEUE_SIZE,
+            /* defaultValue: */ makeSize( 10, sizeUnits_mebibyte ),
+            /* defaultUnits: */ sizeUnits_mebibyte,
+            /* minValidValueInBytes: */ 1 );
+
     #if ( ELASTIC_APM_MEMORY_TRACKING_ENABLED_01 != 0 )
     ELASTIC_APM_ENUM_INIT_METADATA(
             /* fieldName: */ memoryTrackingLevel,

agent/native/ext/ConfigManager.h

@@ -96,6 +96,7 @@ enum OptionId
     #ifdef PHP_WIN32
     optionId_logLevelWinSysDebug,
     #endif
+    optionId_maxSendQueueSize,
     #if ( ELASTIC_APM_MEMORY_TRACKING_ENABLED_01 != 0 )
     optionId_memoryTrackingLevel,
     #endif
@@ -303,6 +304,8 @@ const ConfigSnapshot* getGlobalCurrentConfigSnapshot();
 #define ELASTIC_APM_CFG_OPT_NAME_LOG_LEVEL_WIN_SYS_DEBUG "log_level_win_sys_debug"
 #   endif
 
+#define ELASTIC_APM_CFG_OPT_NAME_MAX_SEND_QUEUE_SIZE "max_send_queue_size"
+
 /**
  * Internal configuration option (not included in public documentation)
  */

agent/native/ext/ConfigSnapshot.h

@@ -25,6 +25,7 @@
 #include "LogLevel.h"
 #include "OptionalBool.h"
 #include "time_util.h" // Duration
+#include "util.h" // Size
 #include "elastic_apm_assert_enabled.h"
 
 struct ConfigSnapshot
@@ -68,6 +69,7 @@ struct ConfigSnapshot
         #ifdef PHP_WIN32
     LogLevel logLevelWinSysDebug = logLevel_off;
         #endif
+    Size maxSendQueueSize;
         #if ( ELASTIC_APM_MEMORY_TRACKING_ENABLED_01 != 0 )
     MemoryTrackingLevel memoryTrackingLevel = memoryTrackingLevel_off;
         #endif

agent/native/ext/backend_comm.cpp

@@ -28,7 +28,6 @@
 #include "Tracer.h"
 #include "ConfigSnapshot.h"
 #include "util.h"
-#include "util_for_PHP.h"
 #include "basic_macros.h"
 #include "backend_comm_backoff.h"
 
@@ -660,15 +659,14 @@ static void freeDataToSendQueue( DataToSendQueue* dataQueue )
     }
 }
 
-#define ELASTIC_APM_MAX_QUEUE_SIZE_IN_BYTES (10 * 1024 * 1024)
-
 struct BackgroundBackendComm
 {
     Mutex* mutex;
     ConditionVariable* condVar;
     Thread* thread;
     DataToSendQueue dataToSendQueue;
     size_t dataToSendTotalSize;
+    UInt64 maxQueueSizeInBytes;
     size_t nextEventsBatchId;
     bool shouldExit;
     TimeSpec shouldExitBy;
@@ -799,6 +797,15 @@ ResultCode backgroundBackendCommThreadFunc_shouldBreakLoop(
 
         if ( compareAbsTimeSpecs( &sharedStateSnapshot->shouldExitBy, &now ) < 0 )
         {
+            StringView serializedEvents = stringBufferToView( sharedStateSnapshot->firstDataToSendNode->serializedEvents );
+            ELASTIC_APM_LOG_WARNING(
+                    "Async shutdown drain timed out with queued events still pending - remaining queued events will be dropped"
+                    "; total size of queued events: %" PRIu64
+                    "; first pending batch ID: %" PRIu64
+                    "; first pending batch size: %" PRIu64
+                    , (UInt64) sharedStateSnapshot->dataToSendTotalSize
+                    , (UInt64) sharedStateSnapshot->firstDataToSendNode->id
+                    , (UInt64) serializedEvents.length );
             *shouldBreakLoop = true;
             goto success;
         }
@@ -1051,13 +1058,17 @@ ResultCode newBackgroundBackendComm( const ConfigSnapshot* config, BackgroundBac
 
     ResultCode resultCode;
     BackgroundBackendComm* backgroundBackendComm = NULL;
+    Int64 maxQueueSizeInBytes = sizeToBytes( config->maxSendQueueSize );
+
+    ELASTIC_APM_ASSERT( maxQueueSizeInBytes > 0, "maxQueueSizeInBytes: %" PRId64, maxQueueSizeInBytes );
 
     ELASTIC_APM_MALLOC_INSTANCE_IF_FAILED_GOTO( BackgroundBackendComm, /* out */ backgroundBackendComm );
     backgroundBackendComm->condVar = NULL;
     backgroundBackendComm->mutex = NULL;
     backgroundBackendComm->thread = NULL;
     initDataToSendQueue( &( backgroundBackendComm->dataToSendQueue ) );
     backgroundBackendComm->dataToSendTotalSize = 0;
+    backgroundBackendComm->maxQueueSizeInBytes = (UInt64) maxQueueSizeInBytes;
     backgroundBackendComm->nextEventsBatchId = 1;
     backgroundBackendComm->shouldExit = false;
     ELASTIC_APM_CALL_IF_FAILED_GOTO( newMutex( &( backgroundBackendComm->mutex ), /* dbgDesc */ "Background backend communications" ) );
@@ -1179,15 +1190,21 @@ ResultCode enqueueEventsToSendToApmServer( StringView userAgentHttpHeader, Strin
     bool shouldUnlockMutex = false;
     UInt64 id;
     BackgroundBackendComm* backgroundBackendComm = g_backgroundBackendComm;
+    UInt64 totalSizeAfterEnqueue;
 
     ELASTIC_APM_CALL_IF_FAILED_GOTO( lockMutex( backgroundBackendComm->mutex, &shouldUnlockMutex, __FUNCTION__ ) );
 
-    if ( backgroundBackendComm->dataToSendTotalSize >= ELASTIC_APM_MAX_QUEUE_SIZE_IN_BYTES )
+    totalSizeAfterEnqueue = (UInt64) backgroundBackendComm->dataToSendTotalSize + (UInt64) serializedEvents.length;
+    if ( totalSizeAfterEnqueue > backgroundBackendComm->maxQueueSizeInBytes )
     {
         ELASTIC_APM_LOG_ERROR(
-                "Already queued events are above max queue size - dropping these events"
+                "Queueing these events would exceed max queue size - dropping these events"
                 "; size of already queued events: %" PRIu64
-                , (UInt64) backgroundBackendComm->dataToSendTotalSize );
+                "; size of events to queue: %" PRIu64
+                "; max queue size: %" PRIu64
+                , (UInt64) backgroundBackendComm->dataToSendTotalSize
+                , (UInt64) serializedEvents.length
+                , backgroundBackendComm->maxQueueSizeInBytes );
         ELASTIC_APM_SET_RESULT_CODE_AND_GOTO_FAILURE();
     }