From 8e017a6ef3f24cfe1a63dda8bdad456133a4b94a Mon Sep 17 00:00:00 2001 From: Geraint Edwards Date: Thu, 7 Mar 2024 08:44:22 +0000 Subject: [PATCH 01/10] add some web services (babbage-web, zebedee-reader, es-web) to homepage-publishing --- v2/manifests/core-ons/babbage.yml | 35 ++++++++++++++++++++++ v2/manifests/core-ons/zebedee-reader.yml | 4 +-- v2/manifests/core-ons/zebedee.yml | 1 + v2/manifests/deps/elasticsearch.yml | 16 ++++++++++ v2/stacks/homepage-publishing/core-ons.yml | 17 ++++++++++- v2/stacks/homepage-publishing/deps.yml | 4 +++ 6 files changed, 74 insertions(+), 3 deletions(-) diff --git a/v2/manifests/core-ons/babbage.yml b/v2/manifests/core-ons/babbage.yml index 4067208e..df8b5aed 100644 --- a/v2/manifests/core-ons/babbage.yml +++ b/v2/manifests/core-ons/babbage.yml @@ -34,3 +34,38 @@ services: interval: ${HEALTHCHECK_INTERVAL:-30s} timeout: 10s retries: 10 + babbage-web: + build: + context: ${DP_REPO_DIR:-../../../..}/babbage + dockerfile: Dockerfile.local + expose: + - "8080" + ports: + - "8088:8080" + restart: unless-stopped + environment: + JAVA_OPTS: " -Xmx512m -Xdebug -Xrunjdwp:transport=dt_socket,address=8010,server=y,suspend=n" + RESTOLINO_STATIC: "target/web" + RESTOLINO_CLASSES: "target/classes" + PACKAGE_PREFIX: "com.github.onsdigital" + DP_COLOURED_LOGGING: "true" + DP_LOGGING_FORMAT: "pretty_json" + DEV_ENVIRONMENT: "Y" + IS_PUBLISHING: "N" + RELOAD_TEMPLATES: "Y" + TEMPLATES_DIR_KEY: "target/web/templates/handlebars" + ENABLE_COVID19_FEATURE: "false" + ENABLE_CACHE: "Y" + ENABLE_METRICS: "Y" + CONTENT_SERVICE_URL: ${ZEBEDEE_URL:-http://zebedee-reader:8082} + HIGHCHARTS_EXPORT_SERVER: ${HIGHCHARTS_EXPORT_SERVER:-http://highcharts:9999/} + ELASTIC_SEARCH_SERVER: ${ELASTIC_SEARCH_SERVER:-elasticsearch-web} + ELASTIC_SEARCH_PORT: ${ELASTIC_SEARCH_PORT:-9300} + MAP_RENDERER_HOST: ${MAP_RENDERER_HOST:-http://localhost:23500} + TABLE_RENDERER_HOST: ${TABLE_RENDERER_HOST:-http://localhost:23300} + OTEL_JAVAAGENT_ENABLED: ${OTEL_JAVAAGENT_ENABLED:-"false"} + healthcheck: + test: [ "CMD", "curl", "-sSf", "http://localhost:8088/health" ] + interval: ${HEALTHCHECK_INTERVAL:-30s} + timeout: 10s + retries: 10 diff --git a/v2/manifests/core-ons/zebedee-reader.yml b/v2/manifests/core-ons/zebedee-reader.yml index 60840afe..31d70eb9 100644 --- a/v2/manifests/core-ons/zebedee-reader.yml +++ b/v2/manifests/core-ons/zebedee-reader.yml @@ -8,7 +8,7 @@ services: expose: - "8082" ports: - - 8082:8082 + - 18082:8082 volumes: - ${zebedee_root:?please define a valid zebedee_root in your local system}:/zebedee_root restart: unless-stopped @@ -22,7 +22,7 @@ services: zebedee_root: "/zebedee_root" OTEL_JAVAAGENT_ENABLED: ${OTEL_JAVAAGENT_ENABLED:-"false"} healthcheck: - test: ["CMD", "curl", "-sSf", "http://localhost:8082/health"] + test: ["CMD", "curl", "-sSf", "http://localhost:18082/health"] interval: ${HEALTHCHECK_INTERVAL:-30s} timeout: 10s retries: 10 diff --git a/v2/manifests/core-ons/zebedee.yml b/v2/manifests/core-ons/zebedee.yml index befd4f49..5c96471c 100644 --- a/v2/manifests/core-ons/zebedee.yml +++ b/v2/manifests/core-ons/zebedee.yml @@ -30,6 +30,7 @@ services: db_audit_username: "postgres" db_audit_password: "postgres" publish_url: ${PUBLISH_URL:-http://the-train:8084} + website_url: ${website_url:-http://babbage-web:8080} ENABLE_CENTRALISED_KEYRING: "true" KEYRING_SECRET_KEY: "38c03PzhNuSrYV8J0537XQ==" KEYRING_INIT_VECTOR: "RkL9MmjfRcPB86alO82gHQ==" diff --git a/v2/manifests/deps/elasticsearch.yml b/v2/manifests/deps/elasticsearch.yml index b38c940e..c73d9c3a 100644 --- a/v2/manifests/deps/elasticsearch.yml +++ b/v2/manifests/deps/elasticsearch.yml @@ -14,6 +14,22 @@ services: timeout: 10s retries: 10 + elasticsearch-web: + image: elasticsearch:2.4.2 + expose: + - "9200" + - "9300" + ports: + - "9308:9300" + - "9208:9200" + environment: + - "discovery.type=single-node" + healthcheck: + test: ["CMD", "curl", "-sSf", "http://localhost:9208/_cluster/health?wait_for_status=yellow"] + interval: 60s + timeout: 10s + retries: 10 + sitewideelasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:7.10.0 expose: diff --git a/v2/stacks/homepage-publishing/core-ons.yml b/v2/stacks/homepage-publishing/core-ons.yml index 23ebcaf1..a993d7dc 100644 --- a/v2/stacks/homepage-publishing/core-ons.yml +++ b/v2/stacks/homepage-publishing/core-ons.yml @@ -25,8 +25,16 @@ services: depends_on: dp-api-router: condition: service_healthy + babbage-web: + extends: + file: ${PATH_MANIFESTS}/core-ons/babbage.yml + service: babbage-web environment: - IS_PUBLISHING: "Y" + depends_on: + babbage: + condition: service_healthy + zebedee-reader: + condition: service_healthy the-train: extends: file: ${PATH_MANIFESTS}/core-ons/the-train.yml @@ -62,6 +70,13 @@ services: depends_on: elasticsearch: condition: service_started + zebedee-reader: + extends: + file: ${PATH_MANIFESTS}/core-ons/zebedee-reader.yml + service: zebedee-reader + depends_on: + elasticsearch-web: + condition: service_started florence: extends: file: ${PATH_MANIFESTS}/core-ons/florence.yml diff --git a/v2/stacks/homepage-publishing/deps.yml b/v2/stacks/homepage-publishing/deps.yml index 856dc58f..f7ab5e93 100644 --- a/v2/stacks/homepage-publishing/deps.yml +++ b/v2/stacks/homepage-publishing/deps.yml @@ -42,3 +42,7 @@ services: extends: file: ${PATH_MANIFESTS}/deps/elasticsearch.yml service: elasticsearch + elasticsearch-web: + extends: + file: ${PATH_MANIFESTS}/deps/elasticsearch.yml + service: elasticsearch-web From 2b260be196d575559fa2ac1ecadcca558aaf06ca Mon Sep 17 00:00:00 2001 From: Geraint Edwards Date: Thu, 7 Mar 2024 09:06:54 +0000 Subject: [PATCH 02/10] zebedee-reader uses elasticsearch-web --- v2/manifests/core-ons/zebedee-reader.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/manifests/core-ons/zebedee-reader.yml b/v2/manifests/core-ons/zebedee-reader.yml index 31d70eb9..e866ad0f 100644 --- a/v2/manifests/core-ons/zebedee-reader.yml +++ b/v2/manifests/core-ons/zebedee-reader.yml @@ -21,6 +21,7 @@ services: PACKAGE_PREFIX: "com.github.onsdigital.zebedee.reader.api" zebedee_root: "/zebedee_root" OTEL_JAVAAGENT_ENABLED: ${OTEL_JAVAAGENT_ENABLED:-"false"} + ELASTIC_SEARCH_SERVER: "elasticsearch-web" healthcheck: test: ["CMD", "curl", "-sSf", "http://localhost:18082/health"] interval: ${HEALTHCHECK_INTERVAL:-30s} From e96c6cc4c791ed4879676b9338a28a49de68c3ac Mon Sep 17 00:00:00 2001 From: Geraint Edwards Date: Thu, 7 Mar 2024 14:15:17 +0000 Subject: [PATCH 03/10] fix health for babbage-web and zebedee-reader --- v2/manifests/core-ons/zebedee-reader.yml | 2 +- v2/manifests/deps/elasticsearch.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/manifests/core-ons/zebedee-reader.yml b/v2/manifests/core-ons/zebedee-reader.yml index e866ad0f..210a89a6 100644 --- a/v2/manifests/core-ons/zebedee-reader.yml +++ b/v2/manifests/core-ons/zebedee-reader.yml @@ -23,7 +23,7 @@ services: OTEL_JAVAAGENT_ENABLED: ${OTEL_JAVAAGENT_ENABLED:-"false"} ELASTIC_SEARCH_SERVER: "elasticsearch-web" healthcheck: - test: ["CMD", "curl", "-sSf", "http://localhost:18082/health"] + test: ["CMD", "curl", "-sSf", "http://localhost:8082/health"] interval: ${HEALTHCHECK_INTERVAL:-30s} timeout: 10s retries: 10 diff --git a/v2/manifests/deps/elasticsearch.yml b/v2/manifests/deps/elasticsearch.yml index c73d9c3a..33cd1054 100644 --- a/v2/manifests/deps/elasticsearch.yml +++ b/v2/manifests/deps/elasticsearch.yml @@ -25,7 +25,7 @@ services: environment: - "discovery.type=single-node" healthcheck: - test: ["CMD", "curl", "-sSf", "http://localhost:9208/_cluster/health?wait_for_status=yellow"] + test: ["CMD", "curl", "-sSf", "http://localhost:9200/_cluster/health?wait_for_status=yellow"] interval: 60s timeout: 10s retries: 10 From 82239c82532a403dae6070169eaf0bb0c4807551 Mon Sep 17 00:00:00 2001 From: Geraint Edwards Date: Thu, 7 Mar 2024 14:17:00 +0000 Subject: [PATCH 04/10] fix health port for babbage-web --- v2/manifests/core-ons/babbage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/manifests/core-ons/babbage.yml b/v2/manifests/core-ons/babbage.yml index df8b5aed..5ab993ef 100644 --- a/v2/manifests/core-ons/babbage.yml +++ b/v2/manifests/core-ons/babbage.yml @@ -65,7 +65,7 @@ services: TABLE_RENDERER_HOST: ${TABLE_RENDERER_HOST:-http://localhost:23300} OTEL_JAVAAGENT_ENABLED: ${OTEL_JAVAAGENT_ENABLED:-"false"} healthcheck: - test: [ "CMD", "curl", "-sSf", "http://localhost:8088/health" ] + test: [ "CMD", "curl", "-sSf", "http://localhost:8080/health" ] interval: ${HEALTHCHECK_INTERVAL:-30s} timeout: 10s retries: 10 From 123f29632afb667bc169208e3beb573d62e3063e Mon Sep 17 00:00:00 2001 From: Geraint Edwards Date: Thu, 7 Mar 2024 14:44:46 +0000 Subject: [PATCH 05/10] add files-api and localstack(s3) to homepage-publishing --- v2/stacks/homepage-publishing/core-ons.yml | 9 +++++++++ v2/stacks/homepage-publishing/deps.yml | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/v2/stacks/homepage-publishing/core-ons.yml b/v2/stacks/homepage-publishing/core-ons.yml index a993d7dc..3bf8b49e 100644 --- a/v2/stacks/homepage-publishing/core-ons.yml +++ b/v2/stacks/homepage-publishing/core-ons.yml @@ -56,6 +56,15 @@ services: depends_on: zebedee: condition: service_healthy + dp-files-api: + extends: + file: ${PATH_MANIFESTS}/core-ons/dp-files-api.yml + service: dp-files-api + environment: + KAFKA_ADDR: 'kafka-1:9092,kafka-2:9093,kafka-3:9094' + depends_on: + dp-api-router: + condition: service_healthy dp-topic-api: extends: file: ${PATH_MANIFESTS}/core-ons/dp-topic-api.yml diff --git a/v2/stacks/homepage-publishing/deps.yml b/v2/stacks/homepage-publishing/deps.yml index f7ab5e93..d0d9258a 100644 --- a/v2/stacks/homepage-publishing/deps.yml +++ b/v2/stacks/homepage-publishing/deps.yml @@ -46,3 +46,7 @@ services: extends: file: ${PATH_MANIFESTS}/deps/elasticsearch.yml service: elasticsearch-web + localstack: + extends: + file: ${PATH_MANIFESTS}/deps/localstack.yml + service: localstack From 2f77f3e8ed2b03ab21e5f62f9ef8dc477ee3a55e Mon Sep 17 00:00:00 2001 From: Geraint Edwards Date: Thu, 7 Mar 2024 16:28:02 +0000 Subject: [PATCH 06/10] add dataset-api, sitewideelasticsearch; fix make images; babbage uses elasticsearch --- v2/manifests/core-ons/babbage.yml | 2 +- v2/stacks/common.mk | 3 ++- v2/stacks/homepage-publishing/core-ons.yml | 12 ++++++++++++ v2/stacks/homepage-publishing/deps.yml | 4 ++++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/v2/manifests/core-ons/babbage.yml b/v2/manifests/core-ons/babbage.yml index 5ab993ef..ff810ac8 100644 --- a/v2/manifests/core-ons/babbage.yml +++ b/v2/manifests/core-ons/babbage.yml @@ -24,7 +24,7 @@ services: ENABLE_COVID19_FEATURE: "false" CONTENT_SERVICE_URL: ${ZEBEDEE_URL:-http://zebedee:8082} HIGHCHARTS_EXPORT_SERVER: ${HIGHCHARTS_EXPORT_SERVER:-http://highcharts:9999/} - ELASTIC_SEARCH_SERVER: ${ELASTIC_SEARCH_SERVER:-localhost} + ELASTIC_SEARCH_SERVER: ${ELASTIC_SEARCH_SERVER:-elasticsearch} ELASTIC_SEARCH_PORT: ${ELASTIC_SEARCH_PORT:-9300} MAP_RENDERER_HOST: ${MAP_RENDERER_HOST:-http://localhost:23500} TABLE_RENDERER_HOST: ${TABLE_RENDERER_HOST:-http://localhost:23300} diff --git a/v2/stacks/common.mk b/v2/stacks/common.mk index 6859d234..6f8853ed 100644 --- a/v2/stacks/common.mk +++ b/v2/stacks/common.mk @@ -56,7 +56,8 @@ ps-docker: .PHONY: image-id image-id: $(LOCAL_ENV_FILE) - @COMPOSE_ENV_FILES=$(COMPOSE_ENV_FILES) docker images --format='{{ .ID }} {{ .Repository }}' | awk '/$(SERVICE)$$/{print $$1}' + @COMPOSE_PROJECT_NAME=$(shell make config | yq .name); \ + COMPOSE_ENV_FILES=$(COMPOSE_ENV_FILES) docker images --format='{{ .ID }} {{ .Repository }}' | awk "/$$COMPOSE_PROJECT_NAME-"'$(SERVICE)$$/{print $$1}' .PHONY: clean-image clean-image: diff --git a/v2/stacks/homepage-publishing/core-ons.yml b/v2/stacks/homepage-publishing/core-ons.yml index 3bf8b49e..a345595b 100644 --- a/v2/stacks/homepage-publishing/core-ons.yml +++ b/v2/stacks/homepage-publishing/core-ons.yml @@ -107,3 +107,15 @@ services: depends_on: dp-image-api: condition: service_healthy + dp-dataset-api: + extends: + file: ${PATH_MANIFESTS}/core-ons/dp-dataset-api.yml + service: dp-dataset-api + environment: + DISABLE_GRAPH_DB_DEPENDENCY: false + extra_hosts: + - "host.docker.internal:host-gateway" # localhost mapping for linux + - "docker.for.mac.host.internal:host-gateway" # localhost mapping for mac (docker v17.12 to v18.02) + depends_on: + zebedee: + condition: service_healthy diff --git a/v2/stacks/homepage-publishing/deps.yml b/v2/stacks/homepage-publishing/deps.yml index d0d9258a..2f09be82 100644 --- a/v2/stacks/homepage-publishing/deps.yml +++ b/v2/stacks/homepage-publishing/deps.yml @@ -50,3 +50,7 @@ services: extends: file: ${PATH_MANIFESTS}/deps/localstack.yml service: localstack + sitewideelasticsearch: + extends: + file: ${PATH_MANIFESTS}/deps/elasticsearch.yml + service: sitewideelasticsearch From cb872fa9189df5fa4d92522d169d01600381c58b Mon Sep 17 00:00:00 2001 From: Geraint Edwards Date: Fri, 8 Mar 2024 10:49:56 +0000 Subject: [PATCH 07/10] fix for non-app services not to use stack name prefix --- v2/scripts/clone.sh | 2 +- v2/stacks/common.mk | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/v2/scripts/clone.sh b/v2/scripts/clone.sh index fd27eb4a..7225c95e 100755 --- a/v2/scripts/clone.sh +++ b/v2/scripts/clone.sh @@ -77,7 +77,7 @@ for repo_url in ${repos[@]}; do error "failed to '$git_arg' repo: $repo_pp ($branch_pp $repo_url_pp)" errors+=( $repo ) else - info "successfully '$git_arg' on repo: $repo_pp ($branch_pp $repo_url_pp)" + info "successful '$git_arg' on repo: $repo_pp ($branch_pp $repo_url_pp)" fi else info "repo already cloned, skipping: $repo_pp ($branch_pp $repo_url_pp)" diff --git a/v2/stacks/common.mk b/v2/stacks/common.mk index 6f8853ed..5c53f00c 100644 --- a/v2/stacks/common.mk +++ b/v2/stacks/common.mk @@ -54,10 +54,21 @@ ps-docker: @do=$@; do=$${do%-docker}; \ COMPOSE_ENV_FILES=$(COMPOSE_ENV_FILES) docker $$do $(SERVICE) $(ENV_FILE_ARGS) +.PHONY: get-repository-name +get-repository-name: $(LOCAL_ENV_FILE) + @if [[ -n "$(SERVICE)" ]]; then \ + APPS="$(shell make list-apps)"; \ + if [[ " $${APPS[*]} " == *" $(SERVICE) "* ]]; then \ + echo $(shell make config | yq .name)-$(SERVICE); \ + else \ + echo $(SERVICE); \ + fi; \ + fi + .PHONY: image-id image-id: $(LOCAL_ENV_FILE) - @COMPOSE_PROJECT_NAME=$(shell make config | yq .name); \ - COMPOSE_ENV_FILES=$(COMPOSE_ENV_FILES) docker images --format='{{ .ID }} {{ .Repository }}' | awk "/$$COMPOSE_PROJECT_NAME-"'$(SERVICE)$$/{print $$1}' + @REPO_NAME=$(shell make get-repository-name); \ + COMPOSE_ENV_FILES=$(COMPOSE_ENV_FILES) docker images --format='{{ .ID }} {{ .Repository }}' | awk '$$2~/'"$$REPO_NAME"'$$/{print $$1}' .PHONY: clean-image clean-image: From 25f3d2929b2f44c045f6bcf5f84a765a59144b9f Mon Sep 17 00:00:00 2001 From: Geraint Edwards Date: Fri, 22 Mar 2024 11:30:41 +0000 Subject: [PATCH 08/10] add dep on elasticsearch; remove empty 'environment:' - was causing error --- v2/stacks/homepage-publishing/core-ons.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/stacks/homepage-publishing/core-ons.yml b/v2/stacks/homepage-publishing/core-ons.yml index a345595b..7ad49ecd 100644 --- a/v2/stacks/homepage-publishing/core-ons.yml +++ b/v2/stacks/homepage-publishing/core-ons.yml @@ -25,11 +25,12 @@ services: depends_on: dp-api-router: condition: service_healthy + elasticsearch: + condition: service_healthy babbage-web: extends: file: ${PATH_MANIFESTS}/core-ons/babbage.yml service: babbage-web - environment: depends_on: babbage: condition: service_healthy From 8327960a2ad9919c6ab6a330aa381dbe7bf6414e Mon Sep 17 00:00:00 2001 From: Geraint Edwards Date: Fri, 22 Mar 2024 11:52:13 +0000 Subject: [PATCH 09/10] add healthcheck for sitewideelasticsearch --- v2/manifests/deps/elasticsearch.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/v2/manifests/deps/elasticsearch.yml b/v2/manifests/deps/elasticsearch.yml index 33cd1054..1e814482 100644 --- a/v2/manifests/deps/elasticsearch.yml +++ b/v2/manifests/deps/elasticsearch.yml @@ -42,6 +42,11 @@ services: environment: - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - "discovery.type=single-node" + healthcheck: + test: ["CMD", "curl", "-sSf", "http://localhost:9200/_cluster/health?wait_for_status=yellow"] + interval: 60s + timeout: 10s + retries: 10 cmdelasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:6.0.0 From 4afecf74920440a760aaf6feb8a964347b8220c5 Mon Sep 17 00:00:00 2001 From: Geraint Edwards Date: Fri, 22 Mar 2024 12:01:16 +0000 Subject: [PATCH 10/10] add version 'make check' for 'yq' --- v2/stacks/common.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/stacks/common.mk b/v2/stacks/common.mk index 5c53f00c..212581ce 100644 --- a/v2/stacks/common.mk +++ b/v2/stacks/common.mk @@ -150,8 +150,9 @@ check-versions: @source $(SCRIPTS_DIR)/utils.sh; \ is_ver java "$$(java -version 2>&1 | sed -En 's/.* version "(.*)"$$/\1/p')" "1\.8\.*"; \ is_ver maven "$$(mvn --version 2>&1 | sed -En 's/.* Maven ([0-9]+\..*) .*/\1/p')" "3\.*"; \ - is_ver docker "$$(docker --version 2>&1 | sed -En 's/.* version ([^ ]+), .*/\1/p')" "25\.*"; \ + is_ver docker "$$(docker --version 2>&1 | sed -En 's/.* version ([^ ]+), .*/\1/p')" "25\.*"; \ is_ver docker-compose "$$(docker-compose --version 2>&1 | sed -En 's/.* version v?([0-9.]+.*)/\1/p')" "2\.2?\.*"; \ + is_ver yq "$$(yq --version 2>&1 | sed -En 's/.* version v?([0-9.]+.*)/\1/p')" "4\.[4-9][0-9].*"; \ : is_ver nvm "$$(nvm --version 2>&1 )" "0\.[3-9][0-9]\..*"; \ : is_ver npm "$$(npm --version 2>&1 )" "0\.[3-9][0-9]\..*"