diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 73a4b8d..edaf19d 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -20,7 +20,111 @@ jobs: run: docker build --tag $DOCKER_IMAGE . - name: Push to container registry run: docker push $DOCKER_IMAGE - run-apibox: + # run-apibox: + # needs: deploy + # runs-on: ubuntu-latest + # env: + # NAME: server-logging + # DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest + # steps: + # - name: executing remote ssh commands + # uses: appleboy/ssh-action@master + # with: + # host: ${{ secrets.APIBOX_URL }} + # username: ${{ secrets.APIBOX_USER }} + # key: ${{ secrets.APIBOX_SSH_KEY }} + # port: ${{ secrets.APIBOX_PORT }} + # envs: NAME,DOCKER_IMAGE + # script: | + # set -e + # docker pull ${DOCKER_IMAGE} + # docker stop ${NAME} && docker rm ${NAME} || true + # docker run \ + # --name ${NAME} \ + # --detach \ + # --restart on-failure \ + # --env OPENSEARCH_URL=vpc-kiba-logs-iz46qlwv7gq2xvvtrtlduv4lbq.eu-west-1.es.amazonaws.com \ + # --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ + # ${DOCKER_IMAGE} + # run-certbox: + # needs: deploy + # runs-on: ubuntu-latest + # env: + # NAME: server-logging + # DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest + # steps: + # - name: executing remote ssh commands + # uses: appleboy/ssh-action@master + # with: + # host: ${{ secrets.CERTBOX_URL }} + # username: ${{ secrets.CERTBOX_USER }} + # key: ${{ secrets.CERTBOX_SSH_KEY }} + # port: ${{ secrets.CERTBOX_PORT }} + # envs: NAME,DOCKER_IMAGE + # script: | + # set -e + # docker pull ${DOCKER_IMAGE} + # docker stop ${NAME} && docker rm ${NAME} || true + # docker run \ + # --name ${NAME} \ + # --detach \ + # --restart on-failure \ + # --env OPENSEARCH_URL=vpc-kiba-logs-iz46qlwv7gq2xvvtrtlduv4lbq.eu-west-1.es.amazonaws.com \ + # --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ + # ${DOCKER_IMAGE} + # run-workerbox: + # needs: deploy + # runs-on: ubuntu-latest + # env: + # NAME: server-logging + # DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest + # steps: + # - name: executing remote ssh commands + # uses: appleboy/ssh-action@master + # with: + # host: ${{ secrets.WORKERBOX_URL }} + # username: ${{ secrets.WORKERBOX_USER }} + # key: ${{ secrets.WORKERBOX_SSH_KEY }} + # port: ${{ secrets.WORKERBOX_PORT }} + # envs: NAME,DOCKER_IMAGE + # script: | + # set -e + # docker pull ${DOCKER_IMAGE} + # docker stop ${NAME} && docker rm ${NAME} || true + # docker run \ + # --name ${NAME} \ + # --detach \ + # --restart on-failure \ + # --env OPENSEARCH_URL=vpc-kiba-logs-iz46qlwv7gq2xvvtrtlduv4lbq.eu-west-1.es.amazonaws.com \ + # --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ + # ${DOCKER_IMAGE} + # run-mdtpbox: + # needs: deploy + # runs-on: ubuntu-latest + # env: + # NAME: server-logging + # DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest + # steps: + # - name: executing remote ssh commands + # uses: appleboy/ssh-action@master + # with: + # host: ${{ secrets.MDTPBOX_URL }} + # username: ${{ secrets.MDTPBOX_USER }} + # key: ${{ secrets.MDTPBOX_SSH_KEY }} + # port: ${{ secrets.MDTPBOX_PORT }} + # envs: NAME,DOCKER_IMAGE + # script: | + # set -e + # docker pull ${DOCKER_IMAGE} + # docker stop ${NAME} && docker rm ${NAME} || true + # docker run \ + # --name ${NAME} \ + # --detach \ + # --restart on-failure \ + # --env OPENSEARCH_URL=vpc-kiba-logs-iz46qlwv7gq2xvvtrtlduv4lbq.eu-west-1.es.amazonaws.com \ + # --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ + # ${DOCKER_IMAGE} + run-ys-appbox: needs: deploy runs-on: ubuntu-latest env: @@ -30,93 +134,19 @@ jobs: - name: executing remote ssh commands uses: appleboy/ssh-action@master with: - host: ${{ secrets.APIBOX_URL }} - username: ${{ secrets.APIBOX_USER }} - key: ${{ secrets.APIBOX_SSH_KEY }} - port: ${{ secrets.APIBOX_PORT }} + host: ${{ secrets.YS_APPBOX_URL }} + username: ${{ secrets.YS_APPBOX_USER }} + key: ${{ secrets.YS_APPBOX_SSH_KEY }} + port: ${{ secrets.YS_APPBOX_PORT }} envs: NAME,DOCKER_IMAGE - script_stop: true - script: | - docker pull ${DOCKER_IMAGE} - docker stop ${NAME} && docker rm ${NAME} || true - docker run \ - --name ${NAME} \ - --detach \ - --restart on-failure \ - --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ - ${DOCKER_IMAGE} - run-certbox: - needs: deploy - runs-on: ubuntu-latest - env: - NAME: server-logging - DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest - steps: - - name: executing remote ssh commands - uses: appleboy/ssh-action@master - with: - host: ${{ secrets.CERTBOX_URL }} - username: ${{ secrets.CERTBOX_USER }} - key: ${{ secrets.CERTBOX_SSH_KEY }} - port: ${{ secrets.CERTBOX_PORT }} - envs: NAME,DOCKER_IMAGE - script_stop: true - script: | - docker pull ${DOCKER_IMAGE} - docker stop ${NAME} && docker rm ${NAME} || true - docker run \ - --name ${NAME} \ - --detach \ - --restart on-failure \ - --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ - ${DOCKER_IMAGE} - run-workerbox: - needs: deploy - runs-on: ubuntu-latest - env: - NAME: server-logging - DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest - steps: - - name: executing remote ssh commands - uses: appleboy/ssh-action@master - with: - host: ${{ secrets.WORKERBOX_URL }} - username: ${{ secrets.WORKERBOX_USER }} - key: ${{ secrets.WORKERBOX_SSH_KEY }} - port: ${{ secrets.WORKERBOX_PORT }} - envs: NAME,DOCKER_IMAGE - script_stop: true - script: | - docker pull ${DOCKER_IMAGE} - docker stop ${NAME} && docker rm ${NAME} || true - docker run \ - --name ${NAME} \ - --detach \ - --restart on-failure \ - --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ - ${DOCKER_IMAGE} - run-mdtpbox: - needs: deploy - runs-on: ubuntu-latest - env: - NAME: server-logging - DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest - steps: - - name: executing remote ssh commands - uses: appleboy/ssh-action@master - with: - host: ${{ secrets.MDTPBOX_URL }} - username: ${{ secrets.MDTPBOX_USER }} - key: ${{ secrets.MDTPBOX_SSH_KEY }} - port: ${{ secrets.MDTPBOX_PORT }} - envs: NAME,DOCKER_IMAGE - script_stop: true script: | + set -e docker pull ${DOCKER_IMAGE} docker stop ${NAME} && docker rm ${NAME} || true docker run \ --name ${NAME} \ --detach \ --restart on-failure \ + --env OPENSEARCH_URL=vpc-yieldseeker-logs-vzis2bk6an4gq654xu6fynrnby.eu-west-1.es.amazonaws.com \ --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ ${DOCKER_IMAGE} diff --git a/docker-compose.yaml b/docker-compose.yaml index f00a867..7d559ad 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,13 +1,12 @@ -version: "3.7" - services: fluent-bit: build: ./fluentbit depends_on: - elasticsearch elasticsearch: - image: elasticsearch:7.6.2 + image: elasticsearch:9.1.2 ports: - "9200:9200" environment: - discovery.type=single-node + - xpack.security.enabled=false diff --git a/fluentbit/Dockerfile b/fluentbit/Dockerfile index d7dea5d..d68c844 100644 --- a/fluentbit/Dockerfile +++ b/fluentbit/Dockerfile @@ -1,4 +1,6 @@ -FROM fluent/fluent-bit:2.0.9 +FROM fluent/fluent-bit:4.0.8 -COPY *.conf /fluent-bit/etc/ +COPY *.yaml /fluent-bit/etc/ COPY *.lua /fluent-bit/etc/ + +CMD ["/fluent-bit/bin/fluent-bit", "--config=/fluent-bit/etc/fluent-bit.yaml"] diff --git a/fluentbit/fluent-bit.conf b/fluentbit/fluent-bit.conf deleted file mode 100644 index a8b24fe..0000000 --- a/fluentbit/fluent-bit.conf +++ /dev/null @@ -1,71 +0,0 @@ -[SERVICE] - Log_Level info - Parsers_File parsers.conf - -# # Testing only : python input -# [INPUT] -# Name dummy -# Dummy {"log": "{\"date\":\"2022-04-21T17:53:48.739537\",\"path\":\"/Users/krishan/Projects/nftoftheday/.env/lib/python3.9/site-packages/core/logging.py\",\"function\":\"api\",\"line\":142,\"message\":\"\",\"level\":\"INFO\",\"logger\":\"api\",\"format\":\"KIBA_API_1\",\"name\":\"notd-api\",\"version\":\"local\",\"environment\":\"dev\",\"requestId\":null,\"apiAction\":\"MESSAGE\",\"apiPath\":\"CMD\",\"apiQuery\":\"a=1&b=2\",\"apiResponse\":\"\",\"apiDuration\":\"\"}"} -# Tag docker_logs - -# # Testing only : nginx input -# [INPUT] -# Name dummy -# Dummy {"log": "{\"time_iso8601\":\"2022-05-19T11:05:14+00:00\",\"request_id\":\"878d3011fbf2fa76aebc4fd40a3d1e89\",\"request_method\":\"GET\",\"uri\":\"/index.html\",\"query_string\":\"\",\"status\":\"200\",\"request_time\":\"0.000\",\"request_length\":\"73\",\"remote_addr\":\"172.17.0.1\",\"remote_user\":\"\",\"remote_port\":\"64588\",\"scheme\":\"http\",\"http_host\":\"localhost\",\"http_referrer\":\"\",\"http_user_agent\":\"curl/7.79.1\",\"bytes_sent\":\"868\",\"format\":\"KIBA_NGINX_1\"}"} -# Tag docker_logs - -# # Testing only : local es output -# [OUTPUT] -# Name es -# Match * -# Host elasticsearch - -# Read docker logs files -[INPUT] - Name tail - Path /var/lib/docker/containers/*/*.log - Parser docker_log_parser - Tag docker_logs - -# Send to opensearch instance on AWS -[OUTPUT] - Name opensearch - Match * - Host vpc-kiba-logs-iz46qlwv7gq2xvvtrtlduv4lbq.eu-west-1.es.amazonaws.com - Port 443 - tls On - Logstash_Format On - Suppress_Type_Name On - -# Extract log fields (docker wraps logs in {"log": "", ...}) -[FILTER] - Name parser - Match docker_logs - Parser kiba_log_parser - Key_Name log - Preserve_Key Off - Reserve_Data On - -[FILTER] - Name rewrite_tag - Match docker_logs - Rule $format ^(KIBA_NGINX_[A-Z0-9_]+)$ "docker_logs.kiba.nginx" false - -[FILTER] - Name rewrite_tag - Match docker_logs - Rule $format ^(KIBA_[A-Z0-9_]+)$ "docker_logs.kiba" false - -# Configure nginx logs -[FILTER] - Name lua - Match docker_logs.kiba.nginx - Script nginx.lua - Call parse - -# Add tag to all records (for debugging log processing) -[FILTER] - Name lua - Match * - Script util.lua - Call append_tag diff --git a/fluentbit/fluent-bit.yaml b/fluentbit/fluent-bit.yaml new file mode 100644 index 0000000..7182560 --- /dev/null +++ b/fluentbit/fluent-bit.yaml @@ -0,0 +1,73 @@ +service: + log_level: info +parsers: + - name: docker_log_parser + format: json + time_key: time + time_format: "%Y-%m-%dT%H:%M:%S.%L" + time_keep: false + - name: kiba_log_parser + format: json + time_key: date + time_format: "%Y-%m-%dT%H:%M:%S.%L" + time_keep: false +pipeline: + inputs: + - name: tail + path: /var/lib/docker/containers/*/*.log + parser: docker_log_parser + tag: docker_logs + multiline.parser: docker + path_key: containerLogPath + outputs: + - name: opensearch + match: "*" + host: ${OPENSEARCH_URL} + port: 443 + tls: "On" + logstash_format: "On" + suppress_type_name: "On" + # # Testing only + # inputs: + # - name: dummy + # dummy: | + # {"log": "{\"date\":\"2022-04-21T17:53:48.739537\",\"path\":\"/Users/krishan/Projects/nftoftheday/.env/lib/python3.9/site-packages/core/logging.py\",\"function\":\"api\",\"line\":142,\"message\":\"\",\"level\":\"INFO\",\"logger\":\"api\",\"format\":\"KIBA_API_1\",\"name\":\"notd-api\",\"version\":\"local\",\"environment\":\"dev\",\"requestId\":null,\"apiAction\":\"MESSAGE\",\"apiPath\":\"CMD\",\"apiQuery\":\"a=1&b=2\",\"apiResponse\":\"\",\"apiDuration\":\"\"}"} + # tag: docker_logs + # - name: dummy + # dummy: | + # {"log": "{\"time_iso8601\":\"2022-05-19T11:05:14+00:00\",\"request_id\":\"878d3011fbf2fa76aebc4fd40a3d1e89\",\"request_method\":\"GET\",\"uri\":\"/index.html\",\"query_string\":\"\",\"status\":\"200\",\"request_time\":\"0.000\",\"request_length\":\"73\",\"remote_addr\":\"172.17.0.1\",\"remote_user\":\"\",\"remote_port\":\"64588\",\"scheme\":\"http\",\"http_host\":\"localhost\",\"http_referrer\":\"\",\"http_user_agent\":\"curl/7.79.1\",\"bytes_sent\":\"868\",\"format\":\"KIBA_NGINX_1\"}"} + # tag: docker_logs + # outputs: + # - name: es + # match: "*" + # host: elasticsearch + # suppress_type_name: "On" + filters: + # Extract container ID from file path using Lua + - name: lua + match: docker_logs + script: util.lua + call: extract_container_id + # Extract log fields (docker wraps logs in {"log": "", ...}) + - name: parser + match: docker_logs + parser: kiba_log_parser + key_name: log + preserve_key: "Off" + reserve_data: "On" + - name: rewrite_tag + match: docker_logs + rule: $format ^(KIBA_NGINX_[A-Z0-9_]+)$ "docker_logs.kiba.nginx" false + - name: rewrite_tag + match: docker_logs + rule: $format ^(KIBA_[A-Z0-9_]+)$ "docker_logs.kiba" false + # Configure nginx logs + - name: lua + match: docker_logs.kiba.nginx + script: nginx.lua + call: parse + # Add tag to all records (for debugging log processing) + - name: lua + match: "*" + script: util.lua + call: append_tag diff --git a/fluentbit/parsers.conf b/fluentbit/parsers.conf deleted file mode 100644 index 38f9992..0000000 --- a/fluentbit/parsers.conf +++ /dev/null @@ -1,13 +0,0 @@ -[PARSER] - Name docker_log_parser - Format json - Time_Key time - Time_Format %Y-%m-%dT%H:%M:%S.%L - Time_Keep Off - -[PARSER] - Name kiba_log_parser - Format json - Time_Key date - Time_Format %Y-%m-%dT%H:%M:%S.%L - Time_Keep Off diff --git a/fluentbit/util.lua b/fluentbit/util.lua index b96e901..1283984 100644 --- a/fluentbit/util.lua +++ b/fluentbit/util.lua @@ -3,3 +3,16 @@ function append_tag(tag, timestamp, record) newRecord["tag"] = tag return 2, timestamp, newRecord end + +function extract_container_id(tag, timestamp, record) + newRecord = record + if record["containerLogPath"] then + local path = record["containerLogPath"] + local container_id = string.match(path, "/var/lib/docker/containers/([^/]+)/") + if container_id then + newRecord["containerId"] = container_id + end + newRecord["containerLogPath"] = nil + end + return 2, timestamp, newRecord +end