From 7bce0d2e75cd3d3db66be905b6234aa6122a5d91 Mon Sep 17 00:00:00 2001
From: Joshua Medley <jmedleyx7@gmail.com>
Date: Fri, 1 Oct 2021 08:57:59 +0100
Subject: [PATCH 1/4] add start-privileged make target, update florence-token
 script and docs

---
 cantabular-import/Makefile                    |  4 ++--
 cantabular-import/README.md                   | 24 +++++++++++++------
 cantabular-import/analysis/README.md          |  3 ++-
 .../analysis/start-analysis/main.go           |  4 ++--
 cantabular-import/get-florence-token.sh       |  3 ---
 cantabular-import/helpers/florence-token      |  2 +-
 .../run-cantabular-without-sudo.sh            |  2 +-
 .../test-docker-compose/README.md             |  5 ++--
 .../test-docker-compose/test-compose.go       |  4 ++--
 9 files changed, 30 insertions(+), 21 deletions(-)
 delete mode 100755 cantabular-import/get-florence-token.sh

diff --git a/cantabular-import/Makefile b/cantabular-import/Makefile
index ba6df20e..17394cd6 100644
--- a/cantabular-import/Makefile
+++ b/cantabular-import/Makefile
@@ -1,12 +1,12 @@
 # start full journey
+.PHONY: start
 start:
-	@echo "starting containers"
 	docker-compose up
 
 # Start full journey with super user privileges
 .PHONY: start-privileged
 start-privileged:
-	sudo docker-compose up
+	sudo -E docker-compose up
 
 # start full journey (containers in background)
 .PHONY: start-detached
diff --git a/cantabular-import/README.md b/cantabular-import/README.md
index 2760b8ea..1b50a1ee 100644
--- a/cantabular-import/README.md
+++ b/cantabular-import/README.md
@@ -34,7 +34,13 @@ as `dp-compose` (this repository):
 `zebedee`
 
 Expects you to have environment variables `zebedee_root` and 
-`SERVICE_AUTH_TOKEN` set in your local environment
+`SERVICE_AUTH_TOKEN` set in your local environment.
+
+To use the `start-import` helpers scripts or analysis tools you will need
+to set an environment variable called `FLORENCE_PASSWORD` to your local
+florence login password for `florence@magicroundabout.gov.uk`. Alternatively
+you can directly edit `helpers/florence-token` to hard code your username
+and password.
 
 You will need to run the `import-recipes` script in `dp-recipe-api` when
 first building the containers before running an import. Alternatively there 
@@ -65,11 +71,17 @@ This only needs to be done once (or until you generate debug assets).
 
 # Bring Up Cantabular Import Services #
 
-`make start`
+The first time you run the import journey you will need to run:
+
+`make start-privileged`
+
+This will initialise the Go cache directories used by the containers to speed
+up start-up time. You may also need to use this make target if you make code
+changes that add more dependencies (go modules).
 
-(note: we use `sudo` to prevent docker having issues accessing the `GOCACHE`
-volume it creates. `sudo` requires the `-E` in order to preserve existing
-environment variables)
+On subsequent runs you can use:
+
+`make start`
 
 # Bring Up Cantabular Import Services Detached (running in background) #
 
@@ -114,8 +126,6 @@ Files:
 
     run-cantabular-without-sudo.sh
 
-    get-florence-token.sh
-
 are used by `cantabular-import/helpers/test-compose/test-compose.go` and are nedded at this level for it to bring up the cantabular containers.
 
 ------------------
diff --git a/cantabular-import/analysis/README.md b/cantabular-import/analysis/README.md
index 67c9ca4d..a6cf53b5 100644
--- a/cantabular-import/analysis/README.md
+++ b/cantabular-import/analysis/README.md
@@ -7,7 +7,8 @@ After running one or more import processes, such as:
 
 you can extract the logs for the containers by doing:
 
-* Edit the `../get-florence-token.sh` script to use your florence username/password
+* Edit the `helpers/florence-token` script to use your florence username/password or set
+your `FLORENCE_PASSWORD` environment variables to your florence password.
 
 run:
 
diff --git a/cantabular-import/analysis/start-analysis/main.go b/cantabular-import/analysis/start-analysis/main.go
index 4f701774..f83c08b9 100644
--- a/cantabular-import/analysis/start-analysis/main.go
+++ b/cantabular-import/analysis/start-analysis/main.go
@@ -117,8 +117,8 @@ func ensureDirectoryExists(dirName string) {
 func getToken() (string, error) {
 	fmt.Printf("Running get-florence-token\n")
 
-	cmd := exec.Command("./get-florence-token.sh") // where to get the command from
-	cmd.Dir = "../.."                              // where to execute the command*/
+	cmd := exec.Command("./florence-token")        // where to get the command from
+	cmd.Dir = "../../helpers"                      // where to execute the command*/
 
 	var out bytes.Buffer
 	cmd.Stdout = &out
diff --git a/cantabular-import/get-florence-token.sh b/cantabular-import/get-florence-token.sh
deleted file mode 100755
index dd35bef8..00000000
--- a/cantabular-import/get-florence-token.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/local/bin/bash
-url=http://localhost:8082/login
-curl -d '{"email":"florence@magicroundabout.ons.gov.uk","password":"<your password here>"}' $url
\ No newline at end of file
diff --git a/cantabular-import/helpers/florence-token b/cantabular-import/helpers/florence-token
index 83783786..5acd734d 100755
--- a/cantabular-import/helpers/florence-token
+++ b/cantabular-import/helpers/florence-token
@@ -1,2 +1,2 @@
 url=http://localhost:8082/login
-curl -d '{"email":"florence@magicroundabout.ons.gov.uk","password":"<your password here>"}' $url
+curl -d "{\"email\":\"florence@magicroundabout.ons.gov.uk\",\"password\":\"$FLORENCE_PASSWORD\"}" $url
diff --git a/cantabular-import/run-cantabular-without-sudo.sh b/cantabular-import/run-cantabular-without-sudo.sh
index de8c9cd9..51934279 100755
--- a/cantabular-import/run-cantabular-without-sudo.sh
+++ b/cantabular-import/run-cantabular-without-sudo.sh
@@ -1,2 +1,2 @@
 #!/usr/local/bin/bash
-docker-compose up
+docker-compose up
\ No newline at end of file
diff --git a/cantabular-import/test-docker-compose/README.md b/cantabular-import/test-docker-compose/README.md
index 3ee640e2..d6ac0371 100644
--- a/cantabular-import/test-docker-compose/README.md
+++ b/cantabular-import/test-docker-compose/README.md
@@ -18,11 +18,12 @@ Usage: (on mac books)
 
     Using Docker 3.3.3 works well (the one before having docker-compose v2.0.0-beta.6 ... which does not work well)
 
-* Edit the `../get-florence-token.sh` script to use your florence username/password
+* Edit the `helpers/florence-token` script to use your florence username/password. Alternatively set the
+`FLORENCE_PASSWORD` environment variable to your florence login's password.
 
 * Adjust the constant `maxRuns` in `test-compose.go` for the number of times you want the process to run. Each loop of the process may take about 3 minutes. You may also need to adjust `maxContainersInJob` to match the number of containers that are run for the cantabular import process (which may change).
 
-* First run `./run-cantabular-without-sudo.sh` in its directory and then when all containers running, run `./start-import.sh` in its directory to test that at least one import process completes OK
+* First run `make start` and then when all containers running, run `./start-import.sh` in its directory to test that at least one import process completes OK
 
 * Then to run multiple tests: `./test-compose.sh`
 
diff --git a/cantabular-import/test-docker-compose/test-compose.go b/cantabular-import/test-docker-compose/test-compose.go
index 1155cb60..1c1c43c1 100644
--- a/cantabular-import/test-docker-compose/test-compose.go
+++ b/cantabular-import/test-docker-compose/test-compose.go
@@ -306,8 +306,8 @@ func prettyPrint(s interface{}) string {
 func getToken() (string, error) {
 	fmt.Printf("Running get-florence-token\n")
 
-	cmd := exec.Command("./get-florence-token.sh") // where to get the command from
-	cmd.Dir = ".."                                 // where to execute the command*/
+	cmd := exec.Command("./florence-token")        // where to get the command from
+	cmd.Dir = "../helpers"                         // where to execute the command*/
 
 	var out bytes.Buffer
 	cmd.Stdout = &out

From 1339a548770f30fb0a4a6aefd6d68dc4a2ce4cc0 Mon Sep 17 00:00:00 2001
From: Joshua Medley <jmedleyx7@gmail.com>
Date: Fri, 8 Oct 2021 12:28:55 +0100
Subject: [PATCH 2/4] Update readme.md for dp-frontend-dataset-controller

---
 cantabular-import/README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cantabular-import/README.md b/cantabular-import/README.md
index 1b50a1ee..3fda663d 100644
--- a/cantabular-import/README.md
+++ b/cantabular-import/README.md
@@ -58,6 +58,9 @@ Assets generated using the  `-debug` flag won't work.
 
 You will also need to run `make generate-prod` in the dp-frontend-dataset-controller to generate the asset files.
 
+For dp-frontend-dataset-controller you will need to generate assets using
+`make generate-prod`
+
 You will also need to make sure you have some
 datasets into your Mongo collections. The easiest way to do this is to use the
 import script in `dp-dataset-api`. Currently it can be found on it's own branch

From b2afee110b4ab76aeeb3bf8671dd4b5f1f4a67ee Mon Sep 17 00:00:00 2001
From: Joshua Medley <jmedleyx7@gmail.com>
Date: Fri, 8 Oct 2021 12:38:54 +0100
Subject: [PATCH 3/4] re-add user id fix

---
 cantabular-import/Makefile                                | 8 ++++----
 cantabular-import/dp-api-router.yml                       | 1 +
 cantabular-import/dp-cantabular-api-ext.yml               | 1 +
 cantabular-import/dp-cantabular-csv-exporter.yml          | 1 +
 cantabular-import/dp-cantabular-server.yml                | 1 +
 cantabular-import/dp-dataset-api.yml                      | 1 +
 cantabular-import/dp-download-service.yml                 | 1 +
 cantabular-import/dp-frontend-dataset-controller.yml      | 1 +
 cantabular-import/dp-frontend-router.yml                  | 1 +
 cantabular-import/dp-import-api.yml                       | 1 +
 cantabular-import/dp-import-cantabular-dataset.yml        | 1 +
 .../dp-import-cantabular-dimension-options.yml            | 1 +
 cantabular-import/dp-publishing-dataset-controller.yml    | 1 +
 cantabular-import/dp-recipe-api.yml                       | 1 +
 cantabular-import/florence.yml                            | 1 +
 cantabular-import/zebedee.yml                             | 1 +
 16 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/cantabular-import/Makefile b/cantabular-import/Makefile
index 17394cd6..2a69eef6 100644
--- a/cantabular-import/Makefile
+++ b/cantabular-import/Makefile
@@ -1,24 +1,24 @@
 # start full journey
 .PHONY: start
 start:
-	docker-compose up
+	UID=$(id -u) GID=$(id -g) docker-compose up
 
 # Start full journey with super user privileges
 .PHONY: start-privileged
 start-privileged:
-	sudo -E docker-compose up
+	sudo docker-compose up
 
 # start full journey (containers in background)
 .PHONY: start-detached
 start-detached:
 	@echo "starting containers in background"
-	docker-compose up -d
+	UID=$(id -u) GID=$(id -g)  docker-compose up -d
 
 # start only backend services
 .PHONY: start-backend
 start-backend:
 	@echo "starting back-end containers"
-	docker-compose --env-file .env.backend up
+	UID=$(id -u) GID=$(id -g)  docker-compose --env-file .env.backend up
 
 .PHONY: stop
 stop:
diff --git a/cantabular-import/dp-api-router.yml b/cantabular-import/dp-api-router.yml
index 60ceb726..c1c452ea 100644
--- a/cantabular-import/dp-api-router.yml
+++ b/cantabular-import/dp-api-router.yml
@@ -18,6 +18,7 @@ services:
         ports:
             - 23200:23200
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             MONGODB_BIND_ADDR:        "mongodb:27017"
             KAFKA_ADDR:               "kafka:9092"
diff --git a/cantabular-import/dp-cantabular-api-ext.yml b/cantabular-import/dp-cantabular-api-ext.yml
index fc668198..72ce6dc8 100644
--- a/cantabular-import/dp-cantabular-api-ext.yml
+++ b/cantabular-import/dp-cantabular-api-ext.yml
@@ -10,5 +10,6 @@ services:
             - ../../dp-cantabular-api-ext/cantabular/data:/app/data
             - ../../dp-cantabular-api-ext/cantabular/scripts:/app/scripts
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             CANTABULAR_API_URL: "http://dp-cantabular-server:8491"
diff --git a/cantabular-import/dp-cantabular-csv-exporter.yml b/cantabular-import/dp-cantabular-csv-exporter.yml
index 3f6ccbd6..42fb9038 100644
--- a/cantabular-import/dp-cantabular-csv-exporter.yml
+++ b/cantabular-import/dp-cantabular-csv-exporter.yml
@@ -18,6 +18,7 @@ services:
         ports:
             - 26300:26300
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             BIND_ADDR:              ":26300"
             KAFKA_ADDR:             "kafka:9092"
diff --git a/cantabular-import/dp-cantabular-server.yml b/cantabular-import/dp-cantabular-server.yml
index c4ad03fd..a3c264c9 100644
--- a/cantabular-import/dp-cantabular-server.yml
+++ b/cantabular-import/dp-cantabular-server.yml
@@ -7,6 +7,7 @@ services:
         ports:
             - 8491:8491
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         volumes:
             - ../../dp-cantabular-server/cantabular/data:/app/data
             - ../../dp-cantabular-server/cantabular/scripts:/app/scripts
diff --git a/cantabular-import/dp-dataset-api.yml b/cantabular-import/dp-dataset-api.yml
index a08d0181..a96980cf 100644
--- a/cantabular-import/dp-dataset-api.yml
+++ b/cantabular-import/dp-dataset-api.yml
@@ -18,6 +18,7 @@ services:
         ports:
             - 22000:22000
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             MONGODB_BIND_ADDR:             "mongodb:27017"
             KAFKA_ADDR:                    "kafka:9092"
diff --git a/cantabular-import/dp-download-service.yml b/cantabular-import/dp-download-service.yml
index 73255dd1..542054d1 100644
--- a/cantabular-import/dp-download-service.yml
+++ b/cantabular-import/dp-download-service.yml
@@ -19,6 +19,7 @@ services:
         ports:
             - 23600:23600
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             BIND_ADDR:              ":23600"
             KAFKA_ADDR:             "kafka:9092"
diff --git a/cantabular-import/dp-frontend-dataset-controller.yml b/cantabular-import/dp-frontend-dataset-controller.yml
index 982a367c..ad235d09 100644
--- a/cantabular-import/dp-frontend-dataset-controller.yml
+++ b/cantabular-import/dp-frontend-dataset-controller.yml
@@ -15,6 +15,7 @@ services:
         ports:
             - 20200
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             BIND_ADDR:                ":20200"
             DOWNLOAD_SERVICE_URL:     "http://dp-download-service:23600"
diff --git a/cantabular-import/dp-frontend-router.yml b/cantabular-import/dp-frontend-router.yml
index 89c489c4..f2d8e278 100644
--- a/cantabular-import/dp-frontend-router.yml
+++ b/cantabular-import/dp-frontend-router.yml
@@ -15,6 +15,7 @@ services:
         ports:
             - 20000:20000
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             BABBAGE_URL:            "http://babbage:8080"
             DATASET_CONTROLLER_URL: "http://dp-frontend-dataset-controller:20200"
diff --git a/cantabular-import/dp-import-api.yml b/cantabular-import/dp-import-api.yml
index 2d1f19a4..42630d08 100644
--- a/cantabular-import/dp-import-api.yml
+++ b/cantabular-import/dp-import-api.yml
@@ -18,6 +18,7 @@ services:
         ports:
             - 21800:21800
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             MONGODB_IMPORTS_ADDR:     "mongodb:27017"
             KAFKA_ADDR:               "kafka:9092"
diff --git a/cantabular-import/dp-import-cantabular-dataset.yml b/cantabular-import/dp-import-cantabular-dataset.yml
index fd8fa599..4266cfdd 100644
--- a/cantabular-import/dp-import-cantabular-dataset.yml
+++ b/cantabular-import/dp-import-cantabular-dataset.yml
@@ -17,6 +17,7 @@ services:
         ports:
             - 26100:26100
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             BIND_ADDR:          ":26100"
             KAFKA_ADDR:         "kafka:9092"
diff --git a/cantabular-import/dp-import-cantabular-dimension-options.yml b/cantabular-import/dp-import-cantabular-dimension-options.yml
index d3c5caa4..02083ccc 100644
--- a/cantabular-import/dp-import-cantabular-dimension-options.yml
+++ b/cantabular-import/dp-import-cantabular-dimension-options.yml
@@ -17,6 +17,7 @@ services:
         ports:
             - 26200:26200
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             BIND_ADDR:          ":26200"
             KAFKA_ADDR:         "kafka:9092"
diff --git a/cantabular-import/dp-publishing-dataset-controller.yml b/cantabular-import/dp-publishing-dataset-controller.yml
index c65f17cb..a17060e6 100644
--- a/cantabular-import/dp-publishing-dataset-controller.yml
+++ b/cantabular-import/dp-publishing-dataset-controller.yml
@@ -15,6 +15,7 @@ services:
         ports:
             - 24000
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             BIND_ADDR:          ":24000"
             API_ROUTER_URL:     "http://dp-api-router:23200/v1"
diff --git a/cantabular-import/dp-recipe-api.yml b/cantabular-import/dp-recipe-api.yml
index 74692215..3a5ac1c1 100644
--- a/cantabular-import/dp-recipe-api.yml
+++ b/cantabular-import/dp-recipe-api.yml
@@ -18,6 +18,7 @@ services:
         ports:
             - 22300:22300
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             MONGODB_BIND_ADDR:        "mongodb:27017"
             KAFKA_ADDR:               "kafka:9092"
diff --git a/cantabular-import/florence.yml b/cantabular-import/florence.yml
index 19277f4e..3c06228c 100644
--- a/cantabular-import/florence.yml
+++ b/cantabular-import/florence.yml
@@ -19,6 +19,7 @@ services:
             - dp-frontend-router
             - dp-frontend-dataset-controller
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             ENCRYPTION_DISABLED:    "true"
             ENABLE_DATASET_IMPORT:  "true"
diff --git a/cantabular-import/zebedee.yml b/cantabular-import/zebedee.yml
index 6d5512e0..8225a41c 100644
--- a/cantabular-import/zebedee.yml
+++ b/cantabular-import/zebedee.yml
@@ -13,6 +13,7 @@ services:
         volumes:
             - $zebedee_root:/zebedee_root
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             DATASET_API_URL:            "http://dp-dataset-api:22000"
             DATASET_API_AUTH_TOKEN:     "FD0108EA-825D-411C-9B1D-41EF7727F465"

From d1c6a36e1b4c3f86d991994802f12101d9db2e56 Mon Sep 17 00:00:00 2001
From: Joshua Medley <jmedleyx7@gmail.com>
Date: Fri, 8 Oct 2021 16:28:10 +0100
Subject: [PATCH 4/4] add the-train to compose journey

---
 cantabular-import/.env                           |  2 --
 cantabular-import/Makefile                       | 13 ++++++++++---
 cantabular-import/README.md                      |  2 +-
 cantabular-import/run-cantabular-without-sudo.sh |  2 +-
 cantabular-import/zebedee.yml                    |  1 -
 5 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/cantabular-import/.env b/cantabular-import/.env
index b0a0f1b0..9fd0e4f6 100644
--- a/cantabular-import/.env
+++ b/cantabular-import/.env
@@ -1,3 +1 @@
 COMPOSE_FILE=deps.yml:dp-import-api.yml:dp-import-cantabular-dataset.yml:dp-import-cantabular-dimension-options.yml:dp-cantabular-server.yml:dp-cantabular-api-ext.yml:dp-dataset-api.yml:dp-cantabular-csv-exporter.yml:dp-recipe-api.yml:zebedee.yml:dp-download-service.yml:dp-frontend-dataset-controller.yml:dp-api-router.yml:dp-frontend-router.yml:dp-publishing-dataset-controller.yml:florence.yml:the-train.yml:babbage.yml
-COMPOSE_PATH_SEPARATOR=:
-COMPOSE_PROJECT_NAME=cantabular-import-journey
diff --git a/cantabular-import/Makefile b/cantabular-import/Makefile
index 2a69eef6..65e3682e 100644
--- a/cantabular-import/Makefile
+++ b/cantabular-import/Makefile
@@ -1,7 +1,14 @@
+UID := $(shell id -u)
+GID := $(shell id -g)
+
+export UID
+export GID
+
 # start full journey
 .PHONY: start
 start:
-	UID=$(id -u) GID=$(id -g) docker-compose up
+	@echo "starting containers"
+	docker-compose up
 
 # Start full journey with super user privileges
 .PHONY: start-privileged
@@ -12,13 +19,13 @@ start-privileged:
 .PHONY: start-detached
 start-detached:
 	@echo "starting containers in background"
-	UID=$(id -u) GID=$(id -g)  docker-compose up -d
+	docker-compose up -d
 
 # start only backend services
 .PHONY: start-backend
 start-backend:
 	@echo "starting back-end containers"
-	UID=$(id -u) GID=$(id -g)  docker-compose --env-file .env.backend up
+	docker-compose --env-file .env.backend up
 
 .PHONY: stop
 stop:
diff --git a/cantabular-import/README.md b/cantabular-import/README.md
index 3fda663d..7222c7b3 100644
--- a/cantabular-import/README.md
+++ b/cantabular-import/README.md
@@ -38,7 +38,7 @@ Expects you to have environment variables `zebedee_root` and
 
 To use the `start-import` helpers scripts or analysis tools you will need
 to set an environment variable called `FLORENCE_PASSWORD` to your local
-florence login password for `florence@magicroundabout.gov.uk`. Alternatively
+florence login password for `florence@magicroundabout.ons.gov.uk`. Alternatively
 you can directly edit `helpers/florence-token` to hard code your username
 and password.
 
diff --git a/cantabular-import/run-cantabular-without-sudo.sh b/cantabular-import/run-cantabular-without-sudo.sh
index 51934279..de8c9cd9 100755
--- a/cantabular-import/run-cantabular-without-sudo.sh
+++ b/cantabular-import/run-cantabular-without-sudo.sh
@@ -1,2 +1,2 @@
 #!/usr/local/bin/bash
-docker-compose up
\ No newline at end of file
+docker-compose up
diff --git a/cantabular-import/zebedee.yml b/cantabular-import/zebedee.yml
index 8225a41c..5badc6a2 100644
--- a/cantabular-import/zebedee.yml
+++ b/cantabular-import/zebedee.yml
@@ -36,4 +36,3 @@ services:
             ENABLE_CENTRALISED_KEYRING: "false"
             KEYRING_SECRET_KEY:         "38c03PzhNuSrYV8J0537XQ=="
             KEYRING_INIT_VECTOR:        "RkL9MmjfRcPB86alO82gHQ=="
-