Add user/group id config to compose to remove need for sudo

cantabular-import/.env

@@ -1,3 +1 @@
 COMPOSE_FILE=deps.yml:dp-import-api.yml:dp-import-cantabular-dataset.yml:dp-import-cantabular-dimension-options.yml:dp-cantabular-server.yml:dp-cantabular-api-ext.yml:dp-dataset-api.yml:dp-cantabular-csv-exporter.yml:dp-recipe-api.yml:zebedee.yml:dp-download-service.yml:dp-frontend-dataset-controller.yml:dp-api-router.yml:dp-frontend-router.yml:dp-publishing-dataset-controller.yml:florence.yml:the-train.yml:babbage.yml
-COMPOSE_PATH_SEPARATOR=:
-COMPOSE_PROJECT_NAME=cantabular-import-journey

cantabular-import/Makefile

@@ -1,4 +1,11 @@
+UID := $(shell id -u)
+GID := $(shell id -g)
+
+export UID
+export GID
+
 # start full journey
+.PHONY: start
 start:
 	@echo "starting containers"
 	docker-compose up

cantabular-import/README.md

@@ -34,7 +34,13 @@ as `dp-compose` (this repository):
 `zebedee`
 
 Expects you to have environment variables `zebedee_root` and 
-`SERVICE_AUTH_TOKEN` set in your local environment
+`SERVICE_AUTH_TOKEN` set in your local environment.
+
+To use the `start-import` helpers scripts or analysis tools you will need
+to set an environment variable called `FLORENCE_PASSWORD` to your local
+florence login password for `florence@magicroundabout.ons.gov.uk`. Alternatively
+you can directly edit `helpers/florence-token` to hard code your username
+and password.
 
 You will need to run the `import-recipes` script in `dp-recipe-api` when
 first building the containers before running an import. Alternatively there 
@@ -52,6 +58,9 @@ Assets generated using the  `-debug` flag won't work.
 
 You will also need to run `make generate-prod` in the dp-frontend-dataset-controller to generate the asset files.
 
+For dp-frontend-dataset-controller you will need to generate assets using
+`make generate-prod`
+
 You will also need to make sure you have some
 datasets into your Mongo collections. The easiest way to do this is to use the
 import script in `dp-dataset-api`. Currently it can be found on it's own branch
@@ -65,11 +74,17 @@ This only needs to be done once (or until you generate debug assets).
 
 # Bring Up Cantabular Import Services #
 
-`make start`
+The first time you run the import journey you will need to run:
+
+`make start-privileged`
 
-(note: we use `sudo` to prevent docker having issues accessing the `GOCACHE`
-volume it creates. `sudo` requires the `-E` in order to preserve existing
-environment variables)
+This will initialise the Go cache directories used by the containers to speed
+up start-up time. You may also need to use this make target if you make code
+changes that add more dependencies (go modules).
+
+On subsequent runs you can use:
+
+`make start`
 
 # Bring Up Cantabular Import Services Detached (running in background) #
 
@@ -114,8 +129,6 @@ Files:
 
     run-cantabular-without-sudo.sh
 
-    get-florence-token.sh
-
 are used by `cantabular-import/helpers/test-compose/test-compose.go` and are nedded at this level for it to bring up the cantabular containers.
 
 ------------------

cantabular-import/analysis/README.md

@@ -7,7 +7,8 @@ After running one or more import processes, such as:
 
 you can extract the logs for the containers by doing:
 
-* Edit the `../get-florence-token.sh` script to use your florence username/password
+* Edit the `helpers/florence-token` script to use your florence username/password or set
+your `FLORENCE_PASSWORD` environment variables to your florence password.
 
 run:
 

cantabular-import/analysis/start-analysis/main.go

@@ -117,8 +117,8 @@ func ensureDirectoryExists(dirName string) {
 func getToken() (string, error) {
 	fmt.Printf("Running get-florence-token\n")
 
-	cmd := exec.Command("./get-florence-token.sh") // where to get the command from
-	cmd.Dir = "../.."                              // where to execute the command*/
+	cmd := exec.Command("./florence-token")        // where to get the command from
+	cmd.Dir = "../../helpers"                      // where to execute the command*/
 
 	var out bytes.Buffer
 	cmd.Stdout = &out

cantabular-import/dp-api-router.yml

@@ -18,6 +18,7 @@ services:
         ports:
             - 23200:23200
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             MONGODB_BIND_ADDR:        "mongodb:27017"
             KAFKA_ADDR:               "kafka:9092"

cantabular-import/dp-cantabular-api-ext.yml

@@ -10,5 +10,6 @@ services:
             - ../../dp-cantabular-api-ext/cantabular/data:/app/data
             - ../../dp-cantabular-api-ext/cantabular/scripts:/app/scripts
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             CANTABULAR_API_URL: "http://dp-cantabular-server:8491"

cantabular-import/dp-cantabular-csv-exporter.yml

@@ -18,6 +18,7 @@ services:
         ports:
             - 26300:26300
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             BIND_ADDR:              ":26300"
             KAFKA_ADDR:             "kafka:9092"

cantabular-import/dp-cantabular-server.yml

@@ -7,6 +7,7 @@ services:
         ports:
             - 8491:8491
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         volumes:
             - ../../dp-cantabular-server/cantabular/data:/app/data
             - ../../dp-cantabular-server/cantabular/scripts:/app/scripts

cantabular-import/dp-dataset-api.yml

@@ -18,6 +18,7 @@ services:
         ports:
             - 22000:22000
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             MONGODB_BIND_ADDR:             "mongodb:27017"
             KAFKA_ADDR:                    "kafka:9092"

cantabular-import/dp-download-service.yml

@@ -19,6 +19,7 @@ services:
         ports:
             - 23600:23600
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             BIND_ADDR:              ":23600"
             KAFKA_ADDR:             "kafka:9092"

cantabular-import/dp-frontend-dataset-controller.yml

@@ -15,6 +15,7 @@ services:
         ports:
             - 20200
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             BIND_ADDR:                ":20200"
             DOWNLOAD_SERVICE_URL:     "http://dp-download-service:23600"

cantabular-import/dp-frontend-router.yml

@@ -15,6 +15,7 @@ services:
         ports:
             - 20000:20000
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             BABBAGE_URL:            "http://babbage:8080"
             DATASET_CONTROLLER_URL: "http://dp-frontend-dataset-controller:20200"

cantabular-import/dp-import-api.yml

@@ -18,6 +18,7 @@ services:
         ports:
             - 21800:21800
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             MONGODB_IMPORTS_ADDR:     "mongodb:27017"
             KAFKA_ADDR:               "kafka:9092"

cantabular-import/dp-import-cantabular-dataset.yml

@@ -17,6 +17,7 @@ services:
         ports:
             - 26100:26100
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             BIND_ADDR:          ":26100"
             KAFKA_ADDR:         "kafka:9092"

cantabular-import/dp-import-cantabular-dimension-options.yml

@@ -17,6 +17,7 @@ services:
         ports:
             - 26200:26200
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             BIND_ADDR:          ":26200"
             KAFKA_ADDR:         "kafka:9092"

cantabular-import/dp-publishing-dataset-controller.yml

@@ -15,6 +15,7 @@ services:
         ports:
             - 24000
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             BIND_ADDR:          ":24000"
             API_ROUTER_URL:     "http://dp-api-router:23200/v1"

cantabular-import/dp-recipe-api.yml

@@ -18,6 +18,7 @@ services:
         ports:
             - 22300:22300
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             MONGODB_BIND_ADDR:        "mongodb:27017"
             KAFKA_ADDR:               "kafka:9092"

cantabular-import/florence.yml

@@ -19,6 +19,7 @@ services:
             - dp-frontend-router
             - dp-frontend-dataset-controller
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             ENCRYPTION_DISABLED:    "true"
             ENABLE_DATASET_IMPORT:  "true"

cantabular-import/helpers/florence-token

@@ -1,2 +1,2 @@
 url=http://localhost:8082/login
-curl -d '{"email":"florence@magicroundabout.ons.gov.uk","password":"<your password here>"}' $url
+curl -d "{\"email\":\"florence@magicroundabout.ons.gov.uk\",\"password\":\"$FLORENCE_PASSWORD\"}" $url

cantabular-import/test-docker-compose/README.md

@@ -18,11 +18,12 @@ Usage: (on mac books)
 
     Using Docker 3.3.3 works well (the one before having docker-compose v2.0.0-beta.6 ... which does not work well)
 
-* Edit the `../get-florence-token.sh` script to use your florence username/password
+* Edit the `helpers/florence-token` script to use your florence username/password. Alternatively set the
+`FLORENCE_PASSWORD` environment variable to your florence login's password.
 
 * Adjust the constant `maxRuns` in `test-compose.go` for the number of times you want the process to run. Each loop of the process may take about 3 minutes. You may also need to adjust `maxContainersInJob` to match the number of containers that are run for the cantabular import process (which may change).
 
-* First run `./run-cantabular-without-sudo.sh` in its directory and then when all containers running, run `./start-import.sh` in its directory to test that at least one import process completes OK
+* First run `make start` and then when all containers running, run `./start-import.sh` in its directory to test that at least one import process completes OK
 
 * Then to run multiple tests: `./test-compose.sh`
 

cantabular-import/test-docker-compose/test-compose.go

@@ -306,8 +306,8 @@ func prettyPrint(s interface{}) string {
 func getToken() (string, error) {
 	fmt.Printf("Running get-florence-token\n")
 
-	cmd := exec.Command("./get-florence-token.sh") // where to get the command from
-	cmd.Dir = ".."                                 // where to execute the command*/
+	cmd := exec.Command("./florence-token")        // where to get the command from
+	cmd.Dir = "../helpers"                         // where to execute the command*/
 
 	var out bytes.Buffer
 	cmd.Stdout = &out

cantabular-import/zebedee.yml

@@ -13,6 +13,7 @@ services:
         volumes:
             - $zebedee_root:/zebedee_root
         restart: unless-stopped
+        user: ${UID-1000}:${GID-1000}
         environment:
             DATASET_API_URL:            "http://dp-dataset-api:22000"
             DATASET_API_AUTH_TOKEN:     "FD0108EA-825D-411C-9B1D-41EF7727F465"
@@ -35,4 +36,3 @@ services:
             ENABLE_CENTRALISED_KEYRING: "false"
             KEYRING_SECRET_KEY:         "38c03PzhNuSrYV8J0537XQ=="
             KEYRING_INIT_VECTOR:        "RkL9MmjfRcPB86alO82gHQ=="
-