From 91d7f2816a240fa0e5e82cf519ca267f4745b9ea Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Thu, 21 Aug 2025 10:55:53 +0100 Subject: [PATCH 01/13] . --- .github/workflows/deploy.yaml | 192 ++++++++++++++++++++-------------- docker-compose.yaml | 5 +- fluentbit/Dockerfile | 2 +- fluentbit/fluent-bit.conf | 55 +++++----- 4 files changed, 142 insertions(+), 112 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 73a4b8d..0b0556b 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -1,6 +1,6 @@ name: Deploy on: - push: + pull_request: branches: [ main ] workflow_dispatch: jobs: @@ -20,7 +20,111 @@ jobs: run: docker build --tag $DOCKER_IMAGE . - name: Push to container registry run: docker push $DOCKER_IMAGE - run-apibox: + # run-apibox: + # needs: deploy + # runs-on: ubuntu-latest + # env: + # NAME: server-logging + # DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest + # steps: + # - name: executing remote ssh commands + # uses: appleboy/ssh-action@master + # with: + # host: ${{ secrets.APIBOX_URL }} + # username: ${{ secrets.APIBOX_USER }} + # key: ${{ secrets.APIBOX_SSH_KEY }} + # port: ${{ secrets.APIBOX_PORT }} + # envs: NAME,DOCKER_IMAGE + # script_stop: true + # script: | + # docker pull ${DOCKER_IMAGE} + # docker stop ${NAME} && docker rm ${NAME} || true + # docker run \ + # --name ${NAME} \ + # --detach \ + # --restart on-failure \ + # --env OPENSEARCH_URL=vpc-kiba-logs-iz46qlwv7gq2xvvtrtlduv4lbq.eu-west-1.es.amazonaws.com \ + # --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ + # ${DOCKER_IMAGE} + # run-certbox: + # needs: deploy + # runs-on: ubuntu-latest + # env: + # NAME: server-logging + # DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest + # steps: + # - name: executing remote ssh commands + # uses: appleboy/ssh-action@master + # with: + # host: ${{ secrets.CERTBOX_URL }} + # username: ${{ secrets.CERTBOX_USER }} + # key: ${{ secrets.CERTBOX_SSH_KEY }} + # port: ${{ secrets.CERTBOX_PORT }} + # envs: NAME,DOCKER_IMAGE + # script_stop: true + # script: | + # docker pull ${DOCKER_IMAGE} + # docker stop ${NAME} && docker rm ${NAME} || true + # docker run \ + # --name ${NAME} \ + # --detach \ + # --restart on-failure \ + # --env OPENSEARCH_URL=vpc-kiba-logs-iz46qlwv7gq2xvvtrtlduv4lbq.eu-west-1.es.amazonaws.com \ + # --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ + # ${DOCKER_IMAGE} + # run-workerbox: + # needs: deploy + # runs-on: ubuntu-latest + # env: + # NAME: server-logging + # DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest + # steps: + # - name: executing remote ssh commands + # uses: appleboy/ssh-action@master + # with: + # host: ${{ secrets.WORKERBOX_URL }} + # username: ${{ secrets.WORKERBOX_USER }} + # key: ${{ secrets.WORKERBOX_SSH_KEY }} + # port: ${{ secrets.WORKERBOX_PORT }} + # envs: NAME,DOCKER_IMAGE + # script_stop: true + # script: | + # docker pull ${DOCKER_IMAGE} + # docker stop ${NAME} && docker rm ${NAME} || true + # docker run \ + # --name ${NAME} \ + # --detach \ + # --restart on-failure \ + # --env OPENSEARCH_URL=vpc-kiba-logs-iz46qlwv7gq2xvvtrtlduv4lbq.eu-west-1.es.amazonaws.com \ + # --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ + # ${DOCKER_IMAGE} + # run-mdtpbox: + # needs: deploy + # runs-on: ubuntu-latest + # env: + # NAME: server-logging + # DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest + # steps: + # - name: executing remote ssh commands + # uses: appleboy/ssh-action@master + # with: + # host: ${{ secrets.MDTPBOX_URL }} + # username: ${{ secrets.MDTPBOX_USER }} + # key: ${{ secrets.MDTPBOX_SSH_KEY }} + # port: ${{ secrets.MDTPBOX_PORT }} + # envs: NAME,DOCKER_IMAGE + # script_stop: true + # script: | + # docker pull ${DOCKER_IMAGE} + # docker stop ${NAME} && docker rm ${NAME} || true + # docker run \ + # --name ${NAME} \ + # --detach \ + # --restart on-failure \ + # --env OPENSEARCH_URL=vpc-kiba-logs-iz46qlwv7gq2xvvtrtlduv4lbq.eu-west-1.es.amazonaws.com \ + # --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ + # ${DOCKER_IMAGE} + run-ys-appbox: needs: deploy runs-on: ubuntu-latest env: @@ -30,85 +134,10 @@ jobs: - name: executing remote ssh commands uses: appleboy/ssh-action@master with: - host: ${{ secrets.APIBOX_URL }} - username: ${{ secrets.APIBOX_USER }} - key: ${{ secrets.APIBOX_SSH_KEY }} - port: ${{ secrets.APIBOX_PORT }} - envs: NAME,DOCKER_IMAGE - script_stop: true - script: | - docker pull ${DOCKER_IMAGE} - docker stop ${NAME} && docker rm ${NAME} || true - docker run \ - --name ${NAME} \ - --detach \ - --restart on-failure \ - --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ - ${DOCKER_IMAGE} - run-certbox: - needs: deploy - runs-on: ubuntu-latest - env: - NAME: server-logging - DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest - steps: - - name: executing remote ssh commands - uses: appleboy/ssh-action@master - with: - host: ${{ secrets.CERTBOX_URL }} - username: ${{ secrets.CERTBOX_USER }} - key: ${{ secrets.CERTBOX_SSH_KEY }} - port: ${{ secrets.CERTBOX_PORT }} - envs: NAME,DOCKER_IMAGE - script_stop: true - script: | - docker pull ${DOCKER_IMAGE} - docker stop ${NAME} && docker rm ${NAME} || true - docker run \ - --name ${NAME} \ - --detach \ - --restart on-failure \ - --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ - ${DOCKER_IMAGE} - run-workerbox: - needs: deploy - runs-on: ubuntu-latest - env: - NAME: server-logging - DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest - steps: - - name: executing remote ssh commands - uses: appleboy/ssh-action@master - with: - host: ${{ secrets.WORKERBOX_URL }} - username: ${{ secrets.WORKERBOX_USER }} - key: ${{ secrets.WORKERBOX_SSH_KEY }} - port: ${{ secrets.WORKERBOX_PORT }} - envs: NAME,DOCKER_IMAGE - script_stop: true - script: | - docker pull ${DOCKER_IMAGE} - docker stop ${NAME} && docker rm ${NAME} || true - docker run \ - --name ${NAME} \ - --detach \ - --restart on-failure \ - --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ - ${DOCKER_IMAGE} - run-mdtpbox: - needs: deploy - runs-on: ubuntu-latest - env: - NAME: server-logging - DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest - steps: - - name: executing remote ssh commands - uses: appleboy/ssh-action@master - with: - host: ${{ secrets.MDTPBOX_URL }} - username: ${{ secrets.MDTPBOX_USER }} - key: ${{ secrets.MDTPBOX_SSH_KEY }} - port: ${{ secrets.MDTPBOX_PORT }} + host: ${{ secrets.YS_APPBOX_URL }} + username: ${{ secrets.YS_APPBOX_USER }} + key: ${{ secrets.YS_APPBOX_SSH_KEY }} + port: ${{ secrets.YS_APPBOX_PORT }} envs: NAME,DOCKER_IMAGE script_stop: true script: | @@ -118,5 +147,6 @@ jobs: --name ${NAME} \ --detach \ --restart on-failure \ + --env OPENSEARCH_URL=vpc-yieldseeker-logs-vzis2bk6an4gq654xu6fynrnby.eu-west-1.es.amazonaws.com \ --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ ${DOCKER_IMAGE} diff --git a/docker-compose.yaml b/docker-compose.yaml index f00a867..7d559ad 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,13 +1,12 @@ -version: "3.7" - services: fluent-bit: build: ./fluentbit depends_on: - elasticsearch elasticsearch: - image: elasticsearch:7.6.2 + image: elasticsearch:9.1.2 ports: - "9200:9200" environment: - discovery.type=single-node + - xpack.security.enabled=false diff --git a/fluentbit/Dockerfile b/fluentbit/Dockerfile index d7dea5d..29dbb9b 100644 --- a/fluentbit/Dockerfile +++ b/fluentbit/Dockerfile @@ -1,4 +1,4 @@ -FROM fluent/fluent-bit:2.0.9 +FROM fluent/fluent-bit:4.0.8 COPY *.conf /fluent-bit/etc/ COPY *.lua /fluent-bit/etc/ diff --git a/fluentbit/fluent-bit.conf b/fluentbit/fluent-bit.conf index a8b24fe..c1abe8e 100644 --- a/fluentbit/fluent-bit.conf +++ b/fluentbit/fluent-bit.conf @@ -2,41 +2,42 @@ Log_Level info Parsers_File parsers.conf -# # Testing only : python input -# [INPUT] -# Name dummy -# Dummy {"log": "{\"date\":\"2022-04-21T17:53:48.739537\",\"path\":\"/Users/krishan/Projects/nftoftheday/.env/lib/python3.9/site-packages/core/logging.py\",\"function\":\"api\",\"line\":142,\"message\":\"\",\"level\":\"INFO\",\"logger\":\"api\",\"format\":\"KIBA_API_1\",\"name\":\"notd-api\",\"version\":\"local\",\"environment\":\"dev\",\"requestId\":null,\"apiAction\":\"MESSAGE\",\"apiPath\":\"CMD\",\"apiQuery\":\"a=1&b=2\",\"apiResponse\":\"\",\"apiDuration\":\"\"}"} -# Tag docker_logs - -# # Testing only : nginx input -# [INPUT] -# Name dummy -# Dummy {"log": "{\"time_iso8601\":\"2022-05-19T11:05:14+00:00\",\"request_id\":\"878d3011fbf2fa76aebc4fd40a3d1e89\",\"request_method\":\"GET\",\"uri\":\"/index.html\",\"query_string\":\"\",\"status\":\"200\",\"request_time\":\"0.000\",\"request_length\":\"73\",\"remote_addr\":\"172.17.0.1\",\"remote_user\":\"\",\"remote_port\":\"64588\",\"scheme\":\"http\",\"http_host\":\"localhost\",\"http_referrer\":\"\",\"http_user_agent\":\"curl/7.79.1\",\"bytes_sent\":\"868\",\"format\":\"KIBA_NGINX_1\"}"} -# Tag docker_logs - -# # Testing only : local es output -# [OUTPUT] -# Name es -# Match * -# Host elasticsearch +# Testing only : python input +[INPUT] + Name dummy + Dummy {"log": "{\"date\":\"2022-04-21T17:53:48.739537\",\"path\":\"/Users/krishan/Projects/nftoftheday/.env/lib/python3.9/site-packages/core/logging.py\",\"function\":\"api\",\"line\":142,\"message\":\"\",\"level\":\"INFO\",\"logger\":\"api\",\"format\":\"KIBA_API_1\",\"name\":\"notd-api\",\"version\":\"local\",\"environment\":\"dev\",\"requestId\":null,\"apiAction\":\"MESSAGE\",\"apiPath\":\"CMD\",\"apiQuery\":\"a=1&b=2\",\"apiResponse\":\"\",\"apiDuration\":\"\"}"} + Tag docker_logs -# Read docker logs files +# Testing only : nginx input [INPUT] - Name tail - Path /var/lib/docker/containers/*/*.log - Parser docker_log_parser + Name dummy + Dummy {"log": "{\"time_iso8601\":\"2022-05-19T11:05:14+00:00\",\"request_id\":\"878d3011fbf2fa76aebc4fd40a3d1e89\",\"request_method\":\"GET\",\"uri\":\"/index.html\",\"query_string\":\"\",\"status\":\"200\",\"request_time\":\"0.000\",\"request_length\":\"73\",\"remote_addr\":\"172.17.0.1\",\"remote_user\":\"\",\"remote_port\":\"64588\",\"scheme\":\"http\",\"http_host\":\"localhost\",\"http_referrer\":\"\",\"http_user_agent\":\"curl/7.79.1\",\"bytes_sent\":\"868\",\"format\":\"KIBA_NGINX_1\"}"} Tag docker_logs -# Send to opensearch instance on AWS +# Testing only : local es output [OUTPUT] - Name opensearch + Name es Match * - Host vpc-kiba-logs-iz46qlwv7gq2xvvtrtlduv4lbq.eu-west-1.es.amazonaws.com - Port 443 - tls On - Logstash_Format On + Host elasticsearch Suppress_Type_Name On +# # Read docker logs files +# [INPUT] +# Name tail +# Path /var/lib/docker/containers/*/*.log +# Parser docker_log_parser +# Tag docker_logs + +# # Send to opensearch instance on AWS +# [OUTPUT] +# Name opensearch +# Match * +# Host ${OPENSEARCH_URL} +# Port 443 +# tls On +# Logstash_Format On +# Suppress_Type_Name On + # Extract log fields (docker wraps logs in {"log": "", ...}) [FILTER] Name parser From 00f21c1e2f5a7b4405dcf0eea5ac0380117427b3 Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Thu, 21 Aug 2025 10:56:38 +0100 Subject: [PATCH 02/13] . --- fluentbit/fluent-bit.conf | 56 +++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/fluentbit/fluent-bit.conf b/fluentbit/fluent-bit.conf index c1abe8e..d25f829 100644 --- a/fluentbit/fluent-bit.conf +++ b/fluentbit/fluent-bit.conf @@ -2,42 +2,42 @@ Log_Level info Parsers_File parsers.conf -# Testing only : python input -[INPUT] - Name dummy - Dummy {"log": "{\"date\":\"2022-04-21T17:53:48.739537\",\"path\":\"/Users/krishan/Projects/nftoftheday/.env/lib/python3.9/site-packages/core/logging.py\",\"function\":\"api\",\"line\":142,\"message\":\"\",\"level\":\"INFO\",\"logger\":\"api\",\"format\":\"KIBA_API_1\",\"name\":\"notd-api\",\"version\":\"local\",\"environment\":\"dev\",\"requestId\":null,\"apiAction\":\"MESSAGE\",\"apiPath\":\"CMD\",\"apiQuery\":\"a=1&b=2\",\"apiResponse\":\"\",\"apiDuration\":\"\"}"} - Tag docker_logs - -# Testing only : nginx input -[INPUT] - Name dummy - Dummy {"log": "{\"time_iso8601\":\"2022-05-19T11:05:14+00:00\",\"request_id\":\"878d3011fbf2fa76aebc4fd40a3d1e89\",\"request_method\":\"GET\",\"uri\":\"/index.html\",\"query_string\":\"\",\"status\":\"200\",\"request_time\":\"0.000\",\"request_length\":\"73\",\"remote_addr\":\"172.17.0.1\",\"remote_user\":\"\",\"remote_port\":\"64588\",\"scheme\":\"http\",\"http_host\":\"localhost\",\"http_referrer\":\"\",\"http_user_agent\":\"curl/7.79.1\",\"bytes_sent\":\"868\",\"format\":\"KIBA_NGINX_1\"}"} - Tag docker_logs - -# Testing only : local es output -[OUTPUT] - Name es - Match * - Host elasticsearch - Suppress_Type_Name On +# # Testing only : python input +# [INPUT] +# Name dummy +# Dummy {"log": "{\"date\":\"2022-04-21T17:53:48.739537\",\"path\":\"/Users/krishan/Projects/nftoftheday/.env/lib/python3.9/site-packages/core/logging.py\",\"function\":\"api\",\"line\":142,\"message\":\"\",\"level\":\"INFO\",\"logger\":\"api\",\"format\":\"KIBA_API_1\",\"name\":\"notd-api\",\"version\":\"local\",\"environment\":\"dev\",\"requestId\":null,\"apiAction\":\"MESSAGE\",\"apiPath\":\"CMD\",\"apiQuery\":\"a=1&b=2\",\"apiResponse\":\"\",\"apiDuration\":\"\"}"} +# Tag docker_logs -# # Read docker logs files +# # Testing only : nginx input # [INPUT] -# Name tail -# Path /var/lib/docker/containers/*/*.log -# Parser docker_log_parser +# Name dummy +# Dummy {"log": "{\"time_iso8601\":\"2022-05-19T11:05:14+00:00\",\"request_id\":\"878d3011fbf2fa76aebc4fd40a3d1e89\",\"request_method\":\"GET\",\"uri\":\"/index.html\",\"query_string\":\"\",\"status\":\"200\",\"request_time\":\"0.000\",\"request_length\":\"73\",\"remote_addr\":\"172.17.0.1\",\"remote_user\":\"\",\"remote_port\":\"64588\",\"scheme\":\"http\",\"http_host\":\"localhost\",\"http_referrer\":\"\",\"http_user_agent\":\"curl/7.79.1\",\"bytes_sent\":\"868\",\"format\":\"KIBA_NGINX_1\"}"} # Tag docker_logs -# # Send to opensearch instance on AWS +# # Testing only : local es output # [OUTPUT] -# Name opensearch +# Name es # Match * -# Host ${OPENSEARCH_URL} -# Port 443 -# tls On -# Logstash_Format On +# Host elasticsearch # Suppress_Type_Name On +# Read docker logs files +[INPUT] + Name tail + Path /var/lib/docker/containers/*/*.log + Parser docker_log_parser + Tag docker_logs + +# Send to opensearch instance on AWS +[OUTPUT] + Name opensearch + Match * + Host ${OPENSEARCH_URL} + Port 443 + tls On + Logstash_Format On + Suppress_Type_Name On + # Extract log fields (docker wraps logs in {"log": "", ...}) [FILTER] Name parser From 3492cf400466874f299802c7a2956c6f47631f8b Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Thu, 21 Aug 2025 11:07:02 +0100 Subject: [PATCH 03/13] . --- .github/workflows/deploy.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 0b0556b..2e9d7fa 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -35,8 +35,8 @@ jobs: # key: ${{ secrets.APIBOX_SSH_KEY }} # port: ${{ secrets.APIBOX_PORT }} # envs: NAME,DOCKER_IMAGE - # script_stop: true # script: | + # set -e # docker pull ${DOCKER_IMAGE} # docker stop ${NAME} && docker rm ${NAME} || true # docker run \ @@ -61,8 +61,8 @@ jobs: # key: ${{ secrets.CERTBOX_SSH_KEY }} # port: ${{ secrets.CERTBOX_PORT }} # envs: NAME,DOCKER_IMAGE - # script_stop: true # script: | + # set -e # docker pull ${DOCKER_IMAGE} # docker stop ${NAME} && docker rm ${NAME} || true # docker run \ @@ -87,8 +87,8 @@ jobs: # key: ${{ secrets.WORKERBOX_SSH_KEY }} # port: ${{ secrets.WORKERBOX_PORT }} # envs: NAME,DOCKER_IMAGE - # script_stop: true # script: | + # set -e # docker pull ${DOCKER_IMAGE} # docker stop ${NAME} && docker rm ${NAME} || true # docker run \ @@ -113,8 +113,8 @@ jobs: # key: ${{ secrets.MDTPBOX_SSH_KEY }} # port: ${{ secrets.MDTPBOX_PORT }} # envs: NAME,DOCKER_IMAGE - # script_stop: true # script: | + # set -e # docker pull ${DOCKER_IMAGE} # docker stop ${NAME} && docker rm ${NAME} || true # docker run \ @@ -139,8 +139,8 @@ jobs: key: ${{ secrets.YS_APPBOX_SSH_KEY }} port: ${{ secrets.YS_APPBOX_PORT }} envs: NAME,DOCKER_IMAGE - script_stop: true script: | + set -e docker pull ${DOCKER_IMAGE} docker stop ${NAME} && docker rm ${NAME} || true docker run \ From 7be0e631e97e2e6b4f9bd1cfc0504e6e6e62da49 Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Thu, 21 Aug 2025 11:40:14 +0100 Subject: [PATCH 04/13] . --- fluentbit/fluent-bit.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fluentbit/fluent-bit.conf b/fluentbit/fluent-bit.conf index d25f829..43c0f12 100644 --- a/fluentbit/fluent-bit.conf +++ b/fluentbit/fluent-bit.conf @@ -27,6 +27,8 @@ Path /var/lib/docker/containers/*/*.log Parser docker_log_parser Tag docker_logs + Docker_Mode On + Docker_Mode_Flush 5 # Send to opensearch instance on AWS [OUTPUT] From cacdff884fc3993bf7b82c7b3092dd80288637e4 Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Thu, 21 Aug 2025 11:59:59 +0100 Subject: [PATCH 05/13] . --- fluentbit/fluent-bit.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/fluentbit/fluent-bit.conf b/fluentbit/fluent-bit.conf index 43c0f12..951ece4 100644 --- a/fluentbit/fluent-bit.conf +++ b/fluentbit/fluent-bit.conf @@ -29,6 +29,7 @@ Tag docker_logs Docker_Mode On Docker_Mode_Flush 5 + Docker_Mode_Parser On # Send to opensearch instance on AWS [OUTPUT] From ef5209de1f4b2a7729bd07b3600b7f0ca383d243 Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Thu, 21 Aug 2025 12:14:29 +0100 Subject: [PATCH 06/13] . --- fluentbit/Dockerfile | 4 ++- fluentbit/fluent-bit.conf | 75 --------------------------------------- fluentbit/fluent-bit.yaml | 67 ++++++++++++++++++++++++++++++++++ fluentbit/parsers.conf | 13 ------- 4 files changed, 70 insertions(+), 89 deletions(-) delete mode 100644 fluentbit/fluent-bit.conf create mode 100644 fluentbit/fluent-bit.yaml delete mode 100644 fluentbit/parsers.conf diff --git a/fluentbit/Dockerfile b/fluentbit/Dockerfile index 29dbb9b..d68c844 100644 --- a/fluentbit/Dockerfile +++ b/fluentbit/Dockerfile @@ -1,4 +1,6 @@ FROM fluent/fluent-bit:4.0.8 -COPY *.conf /fluent-bit/etc/ +COPY *.yaml /fluent-bit/etc/ COPY *.lua /fluent-bit/etc/ + +CMD ["/fluent-bit/bin/fluent-bit", "--config=/fluent-bit/etc/fluent-bit.yaml"] diff --git a/fluentbit/fluent-bit.conf b/fluentbit/fluent-bit.conf deleted file mode 100644 index 951ece4..0000000 --- a/fluentbit/fluent-bit.conf +++ /dev/null @@ -1,75 +0,0 @@ -[SERVICE] - Log_Level info - Parsers_File parsers.conf - -# # Testing only : python input -# [INPUT] -# Name dummy -# Dummy {"log": "{\"date\":\"2022-04-21T17:53:48.739537\",\"path\":\"/Users/krishan/Projects/nftoftheday/.env/lib/python3.9/site-packages/core/logging.py\",\"function\":\"api\",\"line\":142,\"message\":\"\",\"level\":\"INFO\",\"logger\":\"api\",\"format\":\"KIBA_API_1\",\"name\":\"notd-api\",\"version\":\"local\",\"environment\":\"dev\",\"requestId\":null,\"apiAction\":\"MESSAGE\",\"apiPath\":\"CMD\",\"apiQuery\":\"a=1&b=2\",\"apiResponse\":\"\",\"apiDuration\":\"\"}"} -# Tag docker_logs - -# # Testing only : nginx input -# [INPUT] -# Name dummy -# Dummy {"log": "{\"time_iso8601\":\"2022-05-19T11:05:14+00:00\",\"request_id\":\"878d3011fbf2fa76aebc4fd40a3d1e89\",\"request_method\":\"GET\",\"uri\":\"/index.html\",\"query_string\":\"\",\"status\":\"200\",\"request_time\":\"0.000\",\"request_length\":\"73\",\"remote_addr\":\"172.17.0.1\",\"remote_user\":\"\",\"remote_port\":\"64588\",\"scheme\":\"http\",\"http_host\":\"localhost\",\"http_referrer\":\"\",\"http_user_agent\":\"curl/7.79.1\",\"bytes_sent\":\"868\",\"format\":\"KIBA_NGINX_1\"}"} -# Tag docker_logs - -# # Testing only : local es output -# [OUTPUT] -# Name es -# Match * -# Host elasticsearch -# Suppress_Type_Name On - -# Read docker logs files -[INPUT] - Name tail - Path /var/lib/docker/containers/*/*.log - Parser docker_log_parser - Tag docker_logs - Docker_Mode On - Docker_Mode_Flush 5 - Docker_Mode_Parser On - -# Send to opensearch instance on AWS -[OUTPUT] - Name opensearch - Match * - Host ${OPENSEARCH_URL} - Port 443 - tls On - Logstash_Format On - Suppress_Type_Name On - -# Extract log fields (docker wraps logs in {"log": "", ...}) -[FILTER] - Name parser - Match docker_logs - Parser kiba_log_parser - Key_Name log - Preserve_Key Off - Reserve_Data On - -[FILTER] - Name rewrite_tag - Match docker_logs - Rule $format ^(KIBA_NGINX_[A-Z0-9_]+)$ "docker_logs.kiba.nginx" false - -[FILTER] - Name rewrite_tag - Match docker_logs - Rule $format ^(KIBA_[A-Z0-9_]+)$ "docker_logs.kiba" false - -# Configure nginx logs -[FILTER] - Name lua - Match docker_logs.kiba.nginx - Script nginx.lua - Call parse - -# Add tag to all records (for debugging log processing) -[FILTER] - Name lua - Match * - Script util.lua - Call append_tag diff --git a/fluentbit/fluent-bit.yaml b/fluentbit/fluent-bit.yaml new file mode 100644 index 0000000..47d82a2 --- /dev/null +++ b/fluentbit/fluent-bit.yaml @@ -0,0 +1,67 @@ +service: + log_level: info +parsers: + - name: docker_log_parser + format: json + time_key: time + time_format: "%Y-%m-%dT%H:%M:%S.%L" + time_keep: false + - name: kiba_log_parser + format: json + time_key: date + time_format: "%Y-%m-%dT%H:%M:%S.%L" + time_keep: false +pipeline: + inputs: + - name: tail + path: /var/lib/docker/containers/*/*.log + parser: docker_log_parser + tag: docker_logs + multiline.parser: docker + outputs: + - name: opensearch + match: "*" + host: ${OPENSEARCH_URL} + port: 443 + tls: "On" + logstash_format: "On" + suppress_type_name: "On" + # # Testing only + # inputs: + # - name: dummy + # dummy: | + # {"log": "{\"date\":\"2022-04-21T17:53:48.739537\",\"path\":\"/Users/krishan/Projects/nftoftheday/.env/lib/python3.9/site-packages/core/logging.py\",\"function\":\"api\",\"line\":142,\"message\":\"\",\"level\":\"INFO\",\"logger\":\"api\",\"format\":\"KIBA_API_1\",\"name\":\"notd-api\",\"version\":\"local\",\"environment\":\"dev\",\"requestId\":null,\"apiAction\":\"MESSAGE\",\"apiPath\":\"CMD\",\"apiQuery\":\"a=1&b=2\",\"apiResponse\":\"\",\"apiDuration\":\"\"}"} + # tag: docker_logs + # - name: dummy + # dummy: | + # {"log": "{\"time_iso8601\":\"2022-05-19T11:05:14+00:00\",\"request_id\":\"878d3011fbf2fa76aebc4fd40a3d1e89\",\"request_method\":\"GET\",\"uri\":\"/index.html\",\"query_string\":\"\",\"status\":\"200\",\"request_time\":\"0.000\",\"request_length\":\"73\",\"remote_addr\":\"172.17.0.1\",\"remote_user\":\"\",\"remote_port\":\"64588\",\"scheme\":\"http\",\"http_host\":\"localhost\",\"http_referrer\":\"\",\"http_user_agent\":\"curl/7.79.1\",\"bytes_sent\":\"868\",\"format\":\"KIBA_NGINX_1\"}"} + # tag: docker_logs + # outputs: + # - name: es + # match: "*" + # host: elasticsearch + # suppress_type_name: "On" + filters: + # Extract log fields (docker wraps logs in {"log": "", ...}) + - name: parser + match: docker_logs + parser: kiba_log_parser + key_name: log + preserve_key: "Off" + reserve_data: "On" + - name: rewrite_tag + match: docker_logs + rule: $format ^(KIBA_NGINX_[A-Z0-9_]+)$ "docker_logs.kiba.nginx" false + - name: rewrite_tag + match: docker_logs + rule: $format ^(KIBA_[A-Z0-9_]+)$ "docker_logs.kiba" false + # Configure nginx logs + - name: lua + match: docker_logs.kiba.nginx + script: nginx.lua + call: parse + # Add tag to all records (for debugging log processing) + - name: lua + match: "*" + script: util.lua + call: append_tag diff --git a/fluentbit/parsers.conf b/fluentbit/parsers.conf deleted file mode 100644 index 38f9992..0000000 --- a/fluentbit/parsers.conf +++ /dev/null @@ -1,13 +0,0 @@ -[PARSER] - Name docker_log_parser - Format json - Time_Key time - Time_Format %Y-%m-%dT%H:%M:%S.%L - Time_Keep Off - -[PARSER] - Name kiba_log_parser - Format json - Time_Key date - Time_Format %Y-%m-%dT%H:%M:%S.%L - Time_Keep Off From d2a4ea095f3b9a4152f559fdf40ffcb173657c67 Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Thu, 21 Aug 2025 12:16:30 +0100 Subject: [PATCH 07/13] . --- fluentbit/fluent-bit.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/fluentbit/fluent-bit.yaml b/fluentbit/fluent-bit.yaml index 47d82a2..790a92b 100644 --- a/fluentbit/fluent-bit.yaml +++ b/fluentbit/fluent-bit.yaml @@ -18,6 +18,7 @@ pipeline: parser: docker_log_parser tag: docker_logs multiline.parser: docker + path_key: container_log_path outputs: - name: opensearch match: "*" @@ -42,6 +43,35 @@ pipeline: # host: elasticsearch # suppress_type_name: "On" filters: + # Extract container ID from file path + - name: modify + match: docker_logs + add: container_id_full ${container_log_path} + - name: modify + match: docker_logs + condition: key_exists container_log_path + copy: container_log_path container_id_full + - name: modify + match: docker_logs + condition: key_exists container_id_full + replace: container_id_full "/var/lib/docker/containers/" "" + - name: modify + match: docker_logs + condition: key_exists container_id_full + replace: container_id_full "/.+" "" + # Extract short container ID (first 12 chars) + - name: modify + match: docker_logs + condition: key_exists container_id_full + copy: container_id_full container_id + - name: modify + match: docker_logs + condition: key_exists container_id + replace: container_id "^(.{12}).*" "$1" + # Remove the raw path key as we don't need it anymore + - name: modify + match: docker_logs + remove: container_log_path # Extract log fields (docker wraps logs in {"log": "", ...}) - name: parser match: docker_logs From 8a1ea967a79dbfc1ab6c1a76d63edadd4c13d775 Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Thu, 21 Aug 2025 12:18:59 +0100 Subject: [PATCH 08/13] . --- fluentbit/fluent-bit.yaml | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/fluentbit/fluent-bit.yaml b/fluentbit/fluent-bit.yaml index 790a92b..f509fe1 100644 --- a/fluentbit/fluent-bit.yaml +++ b/fluentbit/fluent-bit.yaml @@ -44,34 +44,23 @@ pipeline: # suppress_type_name: "On" filters: # Extract container ID from file path - - name: modify - match: docker_logs - add: container_id_full ${container_log_path} - name: modify match: docker_logs condition: key_exists container_log_path - copy: container_log_path container_id_full + replace: container_log_path "/var/lib/docker/containers/" "" - name: modify match: docker_logs - condition: key_exists container_id_full - replace: container_id_full "/var/lib/docker/containers/" "" - - name: modify - match: docker_logs - condition: key_exists container_id_full - replace: container_id_full "/.+" "" - # Extract short container ID (first 12 chars) - - name: modify - match: docker_logs - condition: key_exists container_id_full - copy: container_id_full container_id + condition: key_exists container_log_path + replace: container_log_path "/.+" "" + # Rename to container_id (short version - first 12 chars) - name: modify match: docker_logs - condition: key_exists container_id - replace: container_id "^(.{12}).*" "$1" - # Remove the raw path key as we don't need it anymore + condition: key_exists container_log_path + replace: container_log_path "^(.{12}).*" "$1" - name: modify match: docker_logs - remove: container_log_path + condition: key_exists container_log_path + rename: container_log_path container_id # Extract log fields (docker wraps logs in {"log": "", ...}) - name: parser match: docker_logs From 3c52e037370dc9e2414bba248df1cc20d4ad3e59 Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Thu, 21 Aug 2025 12:21:01 +0100 Subject: [PATCH 09/13] . --- fluentbit/fluent-bit.yaml | 21 ++++----------------- fluentbit/util.lua | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/fluentbit/fluent-bit.yaml b/fluentbit/fluent-bit.yaml index f509fe1..9bdb014 100644 --- a/fluentbit/fluent-bit.yaml +++ b/fluentbit/fluent-bit.yaml @@ -43,24 +43,11 @@ pipeline: # host: elasticsearch # suppress_type_name: "On" filters: - # Extract container ID from file path - - name: modify - match: docker_logs - condition: key_exists container_log_path - replace: container_log_path "/var/lib/docker/containers/" "" - - name: modify - match: docker_logs - condition: key_exists container_log_path - replace: container_log_path "/.+" "" - # Rename to container_id (short version - first 12 chars) - - name: modify - match: docker_logs - condition: key_exists container_log_path - replace: container_log_path "^(.{12}).*" "$1" - - name: modify + # Extract container ID from file path using Lua + - name: lua match: docker_logs - condition: key_exists container_log_path - rename: container_log_path container_id + script: util.lua + call: extract_container_id # Extract log fields (docker wraps logs in {"log": "", ...}) - name: parser match: docker_logs diff --git a/fluentbit/util.lua b/fluentbit/util.lua index b96e901..62f4c19 100644 --- a/fluentbit/util.lua +++ b/fluentbit/util.lua @@ -3,3 +3,19 @@ function append_tag(tag, timestamp, record) newRecord["tag"] = tag return 2, timestamp, newRecord end + +function extract_container_id(tag, timestamp, record) + newRecord = record + if record["container_log_path"] then + -- Extract container ID from path like /var/lib/docker/containers/abc123.../abc123...-json.log + local path = record["container_log_path"] + local container_id = string.match(path, "/var/lib/docker/containers/([^/]+)/") + if container_id then + -- Keep the full container ID + newRecord["container_id"] = container_id + end + -- Remove the path key as we don't need it anymore + newRecord["container_log_path"] = nil + end + return 2, timestamp, newRecord +end From 56eb98a65f019072f8e037e2979858583b76efe8 Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Thu, 21 Aug 2025 12:24:20 +0100 Subject: [PATCH 10/13] . --- fluentbit/fluent-bit.yaml | 2 +- fluentbit/util.lua | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fluentbit/fluent-bit.yaml b/fluentbit/fluent-bit.yaml index 9bdb014..7182560 100644 --- a/fluentbit/fluent-bit.yaml +++ b/fluentbit/fluent-bit.yaml @@ -18,7 +18,7 @@ pipeline: parser: docker_log_parser tag: docker_logs multiline.parser: docker - path_key: container_log_path + path_key: containerLogPath outputs: - name: opensearch match: "*" diff --git a/fluentbit/util.lua b/fluentbit/util.lua index 62f4c19..06a1e57 100644 --- a/fluentbit/util.lua +++ b/fluentbit/util.lua @@ -6,13 +6,13 @@ end function extract_container_id(tag, timestamp, record) newRecord = record - if record["container_log_path"] then + if record["containerLogPath"] then -- Extract container ID from path like /var/lib/docker/containers/abc123.../abc123...-json.log - local path = record["container_log_path"] + local path = record["containerLogPath"] local container_id = string.match(path, "/var/lib/docker/containers/([^/]+)/") if container_id then -- Keep the full container ID - newRecord["container_id"] = container_id + newRecord["containerId"] = container_id end -- Remove the path key as we don't need it anymore newRecord["container_log_path"] = nil From a73fd3405fd93f7bab66c3b061f271c98af07cb9 Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Thu, 21 Aug 2025 12:24:33 +0100 Subject: [PATCH 11/13] . --- fluentbit/util.lua | 3 --- 1 file changed, 3 deletions(-) diff --git a/fluentbit/util.lua b/fluentbit/util.lua index 06a1e57..241eae1 100644 --- a/fluentbit/util.lua +++ b/fluentbit/util.lua @@ -7,14 +7,11 @@ end function extract_container_id(tag, timestamp, record) newRecord = record if record["containerLogPath"] then - -- Extract container ID from path like /var/lib/docker/containers/abc123.../abc123...-json.log local path = record["containerLogPath"] local container_id = string.match(path, "/var/lib/docker/containers/([^/]+)/") if container_id then - -- Keep the full container ID newRecord["containerId"] = container_id end - -- Remove the path key as we don't need it anymore newRecord["container_log_path"] = nil end return 2, timestamp, newRecord From f8bd1cce9e8a359083dd196fa9fd22123b06e5d0 Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Thu, 21 Aug 2025 12:26:53 +0100 Subject: [PATCH 12/13] . --- fluentbit/util.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluentbit/util.lua b/fluentbit/util.lua index 241eae1..1283984 100644 --- a/fluentbit/util.lua +++ b/fluentbit/util.lua @@ -12,7 +12,7 @@ function extract_container_id(tag, timestamp, record) if container_id then newRecord["containerId"] = container_id end - newRecord["container_log_path"] = nil + newRecord["containerLogPath"] = nil end return 2, timestamp, newRecord end From 5397dbd267f76c5cf4a803ceaaba8c8f425875e9 Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Thu, 21 Aug 2025 13:38:56 +0100 Subject: [PATCH 13/13] . --- .github/workflows/deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 2e9d7fa..edaf19d 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -1,6 +1,6 @@ name: Deploy on: - pull_request: + push: branches: [ main ] workflow_dispatch: jobs: