Fiddle: update versions and add ys-appbox

.github/workflows/deploy.yaml

@@ -20,7 +20,111 @@ jobs:
         run: docker build --tag $DOCKER_IMAGE .
       - name: Push to container registry
         run: docker push $DOCKER_IMAGE
-  run-apibox:
+  # run-apibox:
+  #   needs: deploy
+  #   runs-on: ubuntu-latest
+  #   env:
+  #     NAME: server-logging
+  #     DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest
+  #   steps:
+  #     - name: executing remote ssh commands
+  #       uses: appleboy/ssh-action@master
+  #       with:
+  #         host: ${{ secrets.APIBOX_URL }}
+  #         username: ${{ secrets.APIBOX_USER }}
+  #         key: ${{ secrets.APIBOX_SSH_KEY }}
+  #         port: ${{ secrets.APIBOX_PORT }}
+  #         envs: NAME,DOCKER_IMAGE
+  #         script: |
+  #           set -e
+  #           docker pull ${DOCKER_IMAGE}
+  #           docker stop ${NAME} && docker rm ${NAME} || true
+  #           docker run \
+  #               --name ${NAME} \
+  #               --detach \
+  #               --restart on-failure \
+  #               --env OPENSEARCH_URL=vpc-kiba-logs-iz46qlwv7gq2xvvtrtlduv4lbq.eu-west-1.es.amazonaws.com \
+  #               --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \
+  #               ${DOCKER_IMAGE}
+  # run-certbox:
+  #   needs: deploy
+  #   runs-on: ubuntu-latest
+  #   env:
+  #     NAME: server-logging
+  #     DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest
+  #   steps:
+  #     - name: executing remote ssh commands
+  #       uses: appleboy/ssh-action@master
+  #       with:
+  #         host: ${{ secrets.CERTBOX_URL }}
+  #         username: ${{ secrets.CERTBOX_USER }}
+  #         key: ${{ secrets.CERTBOX_SSH_KEY }}
+  #         port: ${{ secrets.CERTBOX_PORT }}
+  #         envs: NAME,DOCKER_IMAGE
+  #         script: |
+  #           set -e
+  #           docker pull ${DOCKER_IMAGE}
+  #           docker stop ${NAME} && docker rm ${NAME} || true
+  #           docker run \
+  #               --name ${NAME} \
+  #               --detach \
+  #               --restart on-failure \
+  #               --env OPENSEARCH_URL=vpc-kiba-logs-iz46qlwv7gq2xvvtrtlduv4lbq.eu-west-1.es.amazonaws.com \
+  #               --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \
+  #               ${DOCKER_IMAGE}
+  # run-workerbox:
+  #   needs: deploy
+  #   runs-on: ubuntu-latest
+  #   env:
+  #     NAME: server-logging
+  #     DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest
+  #   steps:
+  #     - name: executing remote ssh commands
+  #       uses: appleboy/ssh-action@master
+  #       with:
+  #         host: ${{ secrets.WORKERBOX_URL }}
+  #         username: ${{ secrets.WORKERBOX_USER }}
+  #         key: ${{ secrets.WORKERBOX_SSH_KEY }}
+  #         port: ${{ secrets.WORKERBOX_PORT }}
+  #         envs: NAME,DOCKER_IMAGE
+  #         script: |
+  #           set -e
+  #           docker pull ${DOCKER_IMAGE}
+  #           docker stop ${NAME} && docker rm ${NAME} || true
+  #           docker run \
+  #               --name ${NAME} \
+  #               --detach \
+  #               --restart on-failure \
+  #               --env OPENSEARCH_URL=vpc-kiba-logs-iz46qlwv7gq2xvvtrtlduv4lbq.eu-west-1.es.amazonaws.com \
+  #               --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \
+  #               ${DOCKER_IMAGE}
+  # run-mdtpbox:
+  #   needs: deploy
+  #   runs-on: ubuntu-latest
+  #   env:
+  #     NAME: server-logging
+  #     DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest
+  #   steps:
+  #     - name: executing remote ssh commands
+  #       uses: appleboy/ssh-action@master
+  #       with:
+  #         host: ${{ secrets.MDTPBOX_URL }}
+  #         username: ${{ secrets.MDTPBOX_USER }}
+  #         key: ${{ secrets.MDTPBOX_SSH_KEY }}
+  #         port: ${{ secrets.MDTPBOX_PORT }}
+  #         envs: NAME,DOCKER_IMAGE
+  #         script: |
+  #           set -e
+  #           docker pull ${DOCKER_IMAGE}
+  #           docker stop ${NAME} && docker rm ${NAME} || true
+  #           docker run \
+  #               --name ${NAME} \
+  #               --detach \
+  #               --restart on-failure \
+  #               --env OPENSEARCH_URL=vpc-kiba-logs-iz46qlwv7gq2xvvtrtlduv4lbq.eu-west-1.es.amazonaws.com \
+  #               --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \
+  #               ${DOCKER_IMAGE}
+  run-ys-appbox:
     needs: deploy
     runs-on: ubuntu-latest
     env:
@@ -30,93 +134,19 @@ jobs:
       - name: executing remote ssh commands
         uses: appleboy/ssh-action@master
         with:
-          host: ${{ secrets.APIBOX_URL }}
-          username: ${{ secrets.APIBOX_USER }}
-          key: ${{ secrets.APIBOX_SSH_KEY }}
-          port: ${{ secrets.APIBOX_PORT }}
+          host: ${{ secrets.YS_APPBOX_URL }}
+          username: ${{ secrets.YS_APPBOX_USER }}
+          key: ${{ secrets.YS_APPBOX_SSH_KEY }}
+          port: ${{ secrets.YS_APPBOX_PORT }}
           envs: NAME,DOCKER_IMAGE
-          script_stop: true
-          script: |
-            docker pull ${DOCKER_IMAGE}
-            docker stop ${NAME} && docker rm ${NAME} || true
-            docker run \
-                --name ${NAME} \
-                --detach \
-                --restart on-failure \
-                --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \
-                ${DOCKER_IMAGE}
-  run-certbox:
-    needs: deploy
-    runs-on: ubuntu-latest
-    env:
-      NAME: server-logging
-      DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest
-    steps:
-      - name: executing remote ssh commands
-        uses: appleboy/ssh-action@master
-        with:
-          host: ${{ secrets.CERTBOX_URL }}
-          username: ${{ secrets.CERTBOX_USER }}
-          key: ${{ secrets.CERTBOX_SSH_KEY }}
-          port: ${{ secrets.CERTBOX_PORT }}
-          envs: NAME,DOCKER_IMAGE
-          script_stop: true
-          script: |
-            docker pull ${DOCKER_IMAGE}
-            docker stop ${NAME} && docker rm ${NAME} || true
-            docker run \
-                --name ${NAME} \
-                --detach \
-                --restart on-failure \
-                --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \
-                ${DOCKER_IMAGE}
-  run-workerbox:
-    needs: deploy
-    runs-on: ubuntu-latest
-    env:
-      NAME: server-logging
-      DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest
-    steps:
-      - name: executing remote ssh commands
-        uses: appleboy/ssh-action@master
-        with:
-          host: ${{ secrets.WORKERBOX_URL }}
-          username: ${{ secrets.WORKERBOX_USER }}
-          key: ${{ secrets.WORKERBOX_SSH_KEY }}
-          port: ${{ secrets.WORKERBOX_PORT }}
-          envs: NAME,DOCKER_IMAGE
-          script_stop: true
-          script: |
-            docker pull ${DOCKER_IMAGE}
-            docker stop ${NAME} && docker rm ${NAME} || true
-            docker run \
-                --name ${NAME} \
-                --detach \
-                --restart on-failure \
-                --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \
-                ${DOCKER_IMAGE}
-  run-mdtpbox:
-    needs: deploy
-    runs-on: ubuntu-latest
-    env:
-      NAME: server-logging
-      DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest
-    steps:
-      - name: executing remote ssh commands
-        uses: appleboy/ssh-action@master
-        with:
-          host: ${{ secrets.MDTPBOX_URL }}
-          username: ${{ secrets.MDTPBOX_USER }}
-          key: ${{ secrets.MDTPBOX_SSH_KEY }}
-          port: ${{ secrets.MDTPBOX_PORT }}
-          envs: NAME,DOCKER_IMAGE
-          script_stop: true
           script: |
+            set -e
             docker pull ${DOCKER_IMAGE}
             docker stop ${NAME} && docker rm ${NAME} || true
             docker run \
                 --name ${NAME} \
                 --detach \
                 --restart on-failure \
+                --env OPENSEARCH_URL=vpc-yieldseeker-logs-vzis2bk6an4gq654xu6fynrnby.eu-west-1.es.amazonaws.com \
                 --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \
                 ${DOCKER_IMAGE}

docker-compose.yaml

@@ -1,13 +1,12 @@
-version: "3.7"
-
 services:
   fluent-bit:
     build: ./fluentbit
     depends_on:
       - elasticsearch
   elasticsearch:
-    image: elasticsearch:7.6.2
+    image: elasticsearch:9.1.2
     ports:
       - "9200:9200"
     environment:
       - discovery.type=single-node
+      - xpack.security.enabled=false

fluentbit/Dockerfile

@@ -1,4 +1,6 @@
-FROM fluent/fluent-bit:2.0.9
+FROM fluent/fluent-bit:4.0.8
 
-COPY *.conf /fluent-bit/etc/
+COPY *.yaml /fluent-bit/etc/
 COPY *.lua /fluent-bit/etc/
+
+CMD ["/fluent-bit/bin/fluent-bit", "--config=/fluent-bit/etc/fluent-bit.yaml"]

fluentbit/fluent-bit.yaml

@@ -0,0 +1,73 @@
+service:
+  log_level: info
+parsers:
+  - name: docker_log_parser
+    format: json
+    time_key: time
+    time_format: "%Y-%m-%dT%H:%M:%S.%L"
+    time_keep: false
+  - name: kiba_log_parser
+    format: json
+    time_key: date
+    time_format: "%Y-%m-%dT%H:%M:%S.%L"
+    time_keep: false
+pipeline:
+  inputs:
+    - name: tail
+      path: /var/lib/docker/containers/*/*.log
+      parser: docker_log_parser
+      tag: docker_logs
+      multiline.parser: docker
+      path_key: containerLogPath
+  outputs:
+    - name: opensearch
+      match: "*"
+      host: ${OPENSEARCH_URL}
+      port: 443
+      tls: "On"
+      logstash_format: "On"
+      suppress_type_name: "On"
+  # # Testing only
+  # inputs:
+  #   - name: dummy
+  #     dummy: |
+  #       {"log": "{\"date\":\"2022-04-21T17:53:48.739537\",\"path\":\"/Users/krishan/Projects/nftoftheday/.env/lib/python3.9/site-packages/core/logging.py\",\"function\":\"api\",\"line\":142,\"message\":\"\",\"level\":\"INFO\",\"logger\":\"api\",\"format\":\"KIBA_API_1\",\"name\":\"notd-api\",\"version\":\"local\",\"environment\":\"dev\",\"requestId\":null,\"apiAction\":\"MESSAGE\",\"apiPath\":\"CMD\",\"apiQuery\":\"a=1&b=2\",\"apiResponse\":\"\",\"apiDuration\":\"\"}"}
+  #     tag: docker_logs
+  #   - name: dummy
+  #     dummy: |
+  #       {"log": "{\"time_iso8601\":\"2022-05-19T11:05:14+00:00\",\"request_id\":\"878d3011fbf2fa76aebc4fd40a3d1e89\",\"request_method\":\"GET\",\"uri\":\"/index.html\",\"query_string\":\"\",\"status\":\"200\",\"request_time\":\"0.000\",\"request_length\":\"73\",\"remote_addr\":\"172.17.0.1\",\"remote_user\":\"\",\"remote_port\":\"64588\",\"scheme\":\"http\",\"http_host\":\"localhost\",\"http_referrer\":\"\",\"http_user_agent\":\"curl/7.79.1\",\"bytes_sent\":\"868\",\"format\":\"KIBA_NGINX_1\"}"}
+  #     tag: docker_logs
+  # outputs:
+  #   - name: es
+  #     match: "*"
+  #     host: elasticsearch
+  #     suppress_type_name: "On"
+  filters:
+    # Extract container ID from file path using Lua
+    - name: lua
+      match: docker_logs
+      script: util.lua
+      call: extract_container_id
+    # Extract log fields (docker wraps logs in {"log": "<msg>", ...})
+    - name: parser
+      match: docker_logs
+      parser: kiba_log_parser
+      key_name: log
+      preserve_key: "Off"
+      reserve_data: "On"
+    - name: rewrite_tag
+      match: docker_logs
+      rule: $format ^(KIBA_NGINX_[A-Z0-9_]+)$ "docker_logs.kiba.nginx" false
+    - name: rewrite_tag
+      match: docker_logs
+      rule: $format ^(KIBA_[A-Z0-9_]+)$ "docker_logs.kiba" false
+    # Configure nginx logs
+    - name: lua
+      match: docker_logs.kiba.nginx
+      script: nginx.lua
+      call: parse
+    # Add tag to all records (for debugging log processing)
+    - name: lua
+      match: "*"
+      script: util.lua
+      call: append_tag