From 7bced41d7b7cfc095990773b9ed0e17a54d3fb5c Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Fri, 15 May 2026 22:45:33 +0100 Subject: [PATCH 1/8] Split website organism configs --- .gitignore | 4 ++ deploy.py | 37 +++++++++++--- .../loculus/templates/_config-processor.tpl | 20 ++++++++ .../templates/loculus-website-config.yaml | 15 +++++- .../loculus/templates/loculus-website.yaml | 11 +++- website/.gitignore | 1 + website/README.md | 3 +- website/src/config.spec.ts | 50 ++++++++++++++++++- website/src/config.ts | 45 +++++++++++++++-- 9 files changed, 168 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index e07c44ccfe..5de018be2b 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,10 @@ __pycache__/ .astro/ +/website/tests/config/*.json +/website/tests/config/*.yaml +/website/tests/config/organisms/ + .venv* diff --git a/deploy.py b/deploy.py index 332c2efce0..ab4377ea0e 100755 --- a/deploy.py +++ b/deploy.py @@ -486,19 +486,40 @@ def generate_config( return parsed_yaml = list(yaml.full_load_all(helm_output)) - if len(parsed_yaml) == 1: - config_data = parsed_yaml[0]["data"][configmap_path.name] - - with open(output_path, "w") as f: - f.write(config_data) - - print(f"Wrote config to {output_path}") - elif any(substring in template for substring in ["ingest", "preprocessing"]): + if any(substring in template for substring in ["ingest", "preprocessing"]): for doc in parsed_yaml: config_data = yaml.safe_load(doc["data"][configmap_path.name]) with open(output_path.with_suffix(f".{config_data['organism']}.yaml"), "w") as f: yaml.dump(config_data, f) print(f"Wrote config to {f.name}") + return + + wrote_config = False + for doc in parsed_yaml: + config_data = doc.get("data", {}) + if configmap_path.name not in config_data: + continue + + with open(output_path, "w") as f: + f.write(config_data[configmap_path.name]) + + print(f"Wrote config to {output_path}") + wrote_config = True + + if configmap_path.name == "website_config.json": + organisms_dir = output_path.parent / "organisms" + for doc in parsed_yaml: + for filename, config_data in doc.get("data", {}).items(): + if filename in ["website_config.json", "runtime_config.json"] or not filename.endswith(".json"): + continue + organisms_dir.mkdir(exist_ok=True) + organism_config_path = organisms_dir / filename + with open(organism_config_path, "w") as f: + f.write(config_data) + print(f"Wrote config to {organism_config_path}") + + if not wrote_config: + raise RuntimeError(f"Could not find {configmap_path.name} in rendered template {template}") def get_codespace_params(codespace_name): diff --git a/kubernetes/loculus/templates/_config-processor.tpl b/kubernetes/loculus/templates/_config-processor.tpl index cfea6861d9..dcaf110f20 100644 --- a/kubernetes/loculus/templates/_config-processor.tpl +++ b/kubernetes/loculus/templates/_config-processor.tpl @@ -57,8 +57,28 @@ {{- define "loculus.configVolume" -}} - name: {{ .name }} + {{- if .configmaps }} + projected: + sources: + {{- range .configmaps }} + - configMap: + name: {{ .name }} + {{- if .items }} + items: + {{- range .items }} + - key: {{ .key }} + path: {{ .path | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- else }} configMap: name: {{ if .configmap }}{{ .configmap }}{{ else }}{{ .name }}{{ end }} + {{- end }} - name: {{ .name }}-processed emptyDir: {} {{- end }} + +{{- define "loculus.websiteOrganismConfigMapName" -}} +{{- printf "loculus-web-org-config-%s" . | trunc 63 | trimSuffix "-" -}} +{{- end }} diff --git a/kubernetes/loculus/templates/loculus-website-config.yaml b/kubernetes/loculus/templates/loculus-website-config.yaml index 3c5df57465..2143f244a5 100644 --- a/kubernetes/loculus/templates/loculus-website-config.yaml +++ b/kubernetes/loculus/templates/loculus-website-config.yaml @@ -1,3 +1,6 @@ +{{- $websiteConfig := include "loculus.generateWebsiteConfig" . | fromYaml }} +{{- $organisms := $websiteConfig.organisms }} +{{- $_ := set $websiteConfig "organisms" (dict) }} --- apiVersion: v1 kind: ConfigMap @@ -5,7 +8,7 @@ metadata: name: loculus-website-config data: website_config.json: | - {{ include "loculus.generateWebsiteConfig" . | fromYaml | toJson }} + {{ $websiteConfig | toJson }} runtime_config.json: | { "name" : "{{ $.Values.name }}", @@ -28,3 +31,13 @@ data: }, "backendKeycloakClientSecret" : "[[backendKeycloakClientSecret]]" } +{{- range $key, $organismConfig := $organisms }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "loculus.websiteOrganismConfigMapName" $key }} +data: + {{ $key }}.json: | + {{ $organismConfig | toJson }} +{{- end }} diff --git a/kubernetes/loculus/templates/loculus-website.yaml b/kubernetes/loculus/templates/loculus-website.yaml index a7e692aabb..9644b1581d 100644 --- a/kubernetes/loculus/templates/loculus-website.yaml +++ b/kubernetes/loculus/templates/loculus-website.yaml @@ -1,5 +1,12 @@ {{- $dockerTag := include "loculus.dockerTag" .Values }} {{- if not .Values.disableWebsite }} +{{- $websiteConfigMaps := list (dict "name" "loculus-website-config") }} +{{- range $_, $item := (include "loculus.enabledOrganisms" . | fromJson).organisms }} +{{- $websiteConfigMaps = append $websiteConfigMaps (dict + "name" (include "loculus.websiteOrganismConfigMapName" $item.key) + "items" (list (dict "key" (printf "%s.json" $item.key) "path" (printf "organisms/%s.json" $item.key))) +) }} +{{- end }} --- apiVersion: apps/v1 kind: Deployment @@ -49,5 +56,5 @@ spec: imagePullSecrets: - name: custom-website-sealed-secret volumes: -{{ include "loculus.configVolume" (dict "name" "loculus-website-config") | nindent 8 }} -{{- end }} \ No newline at end of file +{{ include "loculus.configVolume" (dict "name" "loculus-website-config" "configmaps" $websiteConfigMaps) | nindent 8 }} +{{- end }} diff --git a/website/.gitignore b/website/.gitignore index 0fa653ef3f..24c18b66b7 100644 --- a/website/.gitignore +++ b/website/.gitignore @@ -22,6 +22,7 @@ justfile /tests/config/backend_config.json /tests/config/website_config.json /tests/config/runtime_config.json +/tests/config/organisms/ /tests/config/*.yaml diff --git a/website/README.md b/website/README.md index 4da8d28dc4..d06ddf1b7f 100644 --- a/website/README.md +++ b/website/README.md @@ -32,7 +32,8 @@ See `.env.docker` for the required variables. Furthermore, the website requires config files that need to be present at runtime in the directory specified in the `CONFIG_DIR` environment variable: -- `website_config.json`: Contains configuration on the underlying organism. It's similar to the database config file that LAPIS uses. +- `website_config.json`: Contains global website configuration. It can also contain organism configuration for local or legacy setups. +- `organisms/*.json`: Optional per-organism configuration files. When present, these are merged into the `organisms` field from `website_config.json`. - `reference_genomes.json`: Defines names for segments of the genome and amino acids. It's equal to the file that LAPIS uses. - `runtime_config.json`: Contains configuration that specific for a deployed instance of the website. diff --git a/website/src/config.spec.ts b/website/src/config.spec.ts index 38e9b54425..da32c26de4 100644 --- a/website/src/config.spec.ts +++ b/website/src/config.spec.ts @@ -1,7 +1,11 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + import { describe, expect, it } from 'vitest'; -import { configuredOrganismsFromConfig, validateWebsiteConfig } from './config.ts'; -import type { WebsiteConfig } from './types/config.ts'; +import { configuredOrganismsFromConfig, readWebsiteConfigFromDir, validateWebsiteConfig } from './config.ts'; +import type { InstanceConfig, WebsiteConfig } from './types/config.ts'; import { SINGLE_SEG_MULTI_REF_REFERENCEGENOMES_SCHEMA } from './types/referenceGenomes.spec.ts'; const defaultConfig: WebsiteConfig = { @@ -18,6 +22,19 @@ const defaultConfig: WebsiteConfig = { readOnlyMode: false, }; +const defaultOrganismConfig = (organismName: string): InstanceConfig => ({ + schema: { + organismName, + inputFields: [], + tableColumns: [], + primaryKey: '', + metadata: [], + defaultOrderBy: '', + defaultOrder: 'ascending', + submissionDataTypes: { consensusSequences: false }, + }, +}); + describe('validateWebsiteConfig', () => { it('should fail when "referenceIdentifierField" is not defined for an organism with multiple references', () => { const errors = validateWebsiteConfig({ @@ -46,6 +63,35 @@ describe('validateWebsiteConfig', () => { }); }); +describe('readWebsiteConfigFromDir', () => { + it('merges organism configs from the organisms directory', () => { + const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'loculus-website-config-')); + try { + fs.writeFileSync( + path.join(configDir, 'website_config.json'), + JSON.stringify({ + ...defaultConfig, + organisms: { + zika: defaultOrganismConfig('Zika Virus'), + }, + }), + ); + fs.mkdirSync(path.join(configDir, 'organisms')); + fs.writeFileSync( + path.join(configDir, 'organisms', 'andes.json'), + JSON.stringify(defaultOrganismConfig('Andes Virus [Hantavirus]')), + ); + + const config = readWebsiteConfigFromDir(configDir); + + expect(Object.keys(config.organisms).sort()).toEqual(['andes', 'zika']); + expect(config.organisms.andes.schema.organismName).toBe('Andes Virus [Hantavirus]'); + } finally { + fs.rmSync(configDir, { recursive: true, force: true }); + } + }); +}); + describe('configuredOrganismsFromConfig', () => { it('sorts organisms by display name instead of key', () => { const organisms = configuredOrganismsFromConfig({ diff --git a/website/src/config.ts b/website/src/config.ts index 69a635febe..48073a898a 100644 --- a/website/src/config.ts +++ b/website/src/config.ts @@ -10,6 +10,7 @@ import { type Schema, type SequenceFlaggingConfig, type WebsiteConfig, + instanceConfig, websiteConfig, } from './types/config.ts'; import { type ReferenceGenomesInfo } from './types/referencesGenomes.ts'; @@ -60,7 +61,7 @@ export function validateWebsiteConfig(config: WebsiteConfig): Error[] { export function getWebsiteConfig(): WebsiteConfig { if (_config === null) { - const config = readTypedConfigFile('website_config.json', websiteConfig); + const config = readWebsiteConfigFromDir(getConfigDir()); const validationErrors = validateWebsiteConfig(config); if (validationErrors.length > 0) { throw new AggregateError(validationErrors, 'There were validation errors in the website_config.json'); @@ -70,6 +71,38 @@ export function getWebsiteConfig(): WebsiteConfig { return _config; } +export function readWebsiteConfigFromDir(configDir: string): WebsiteConfig { + const config = readTypedConfigFile(configDir, 'website_config.json', websiteConfig); + const organismConfigs = readSplitOrganismConfigs(configDir); + if (Object.keys(organismConfigs).length === 0) { + return config; + } + return { + ...config, + organisms: { + ...config.organisms, + ...organismConfigs, + }, + }; +} + +function readSplitOrganismConfigs(configDir: string): Record { + const organismsDir = path.join(configDir, 'organisms'); + if (!fs.existsSync(organismsDir)) { + return {}; + } + return Object.fromEntries( + fs + .readdirSync(organismsDir) + .filter((fileName) => fileName.endsWith('.json')) + .sort() + .map((fileName) => { + const organism = path.basename(fileName, '.json'); + return [organism, readTypedConfigFile(configDir, path.join('organisms', fileName), instanceConfig)]; + }), + ); +} + export function getContactConfig(websiteConfig: WebsiteConfig) { return { gitHubIssuesUrl: websiteConfig.gitHubIssuesUrl, @@ -234,7 +267,7 @@ export function getGroupedInputFields( } export function getRuntimeConfig(): RuntimeConfig { - _runtimeConfig ??= readTypedConfigFile('runtime_config.json', runtimeConfig); + _runtimeConfig ??= readTypedConfigFile(getConfigDir(), 'runtime_config.json', runtimeConfig); return _runtimeConfig; } @@ -261,8 +294,12 @@ export function getDataUseTermsAgreementHTML() { return getWebsiteConfig().dataUseTermsAgreementHTML; } -function readTypedConfigFile(fileName: string, schema: Schema): z.infer { - const configFilePath = path.join(getConfigDir(), fileName); +function readTypedConfigFile( + configDir: string, + fileName: string, + schema: Schema, +): z.infer { + const configFilePath = path.join(configDir, fileName); const json = JSON.parse(fs.readFileSync(configFilePath, 'utf8')); try { return schema.parse(json) as z.infer; From 9a47e8c820cf9508f3d6ea634fb0322a3a31c624 Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Thu, 21 May 2026 12:42:44 +0100 Subject: [PATCH 2/8] Fix config processor projected volume copying --- .../workflows/config-preprocessor-image.yml | 7 +++ .github/workflows/integration-tests.yml | 34 +++++++++++++++ .../config-processor/config-processor.py | 21 ++++++++- .../config-processor/test_config_processor.py | 43 +++++++++++++++++++ 4 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 kubernetes/config-processor/test_config_processor.py diff --git a/.github/workflows/config-preprocessor-image.yml b/.github/workflows/config-preprocessor-image.yml index 7795bab4f2..b04d1bfa4e 100644 --- a/.github/workflows/config-preprocessor-image.yml +++ b/.github/workflows/config-preprocessor-image.yml @@ -42,6 +42,13 @@ jobs: - uses: actions/checkout@v6 - name: Shorten sha run: echo "sha=${sha::7}" >> $GITHUB_ENV + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + - name: Run unit tests + run: | + python3 -m pip install -r requirements.txt + python3 -m unittest - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index e640029ef0..bb0521cee5 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -61,6 +61,40 @@ jobs: run: echo "sha=${sha::7}" >> $GITHUB_ENV - name: Get runner IP run: curl -4 ifconfig.me + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Wait for Docker images + run: | + images=( + backend + config-processor + ena-submission + ena-submission-flyway + ingest + keycloakify + loculus-silo + preprocessing-dummy + preprocessing-nextclade + taxonomy-service + website + ) + deadline=$((SECONDS + 900)) + + for image in "${images[@]}"; do + full_image="ghcr.io/loculus-project/${image}:commit-${sha}" + until docker manifest inspect "$full_image" > /dev/null 2>&1; do + if [ "$SECONDS" -ge "$deadline" ]; then + echo "Timed out waiting for $full_image" + exit 1 + fi + echo "Waiting for $full_image" + sleep 10 + done + done - name: Collect Workflow Telemetry uses: catchpoint/workflow-telemetry-action@v2 - name: Checkout repository diff --git a/kubernetes/config-processor/config-processor.py b/kubernetes/config-processor/config-processor.py index 1760a5123a..73152ebb21 100644 --- a/kubernetes/config-processor/config-processor.py +++ b/kubernetes/config-processor/config-processor.py @@ -6,7 +6,20 @@ def copy_structure(input_dir, output_dir): - for root, dirs, files in os.walk(input_dir): + seen_dirs = set() + for root, dirs, files in os.walk(input_dir, followlinks=True): + # Kubernetes ConfigMap/projected volumes expose visible files as symlinks + # into hidden `..data` directories. Copy the visible tree only, while + # still following visible symlinked directories such as `organisms/`. + dirs[:] = [dir for dir in dirs if not dir.startswith("..")] + files = [file for file in files if not file.startswith("..")] + + real_root = os.path.realpath(root) + if real_root in seen_dirs: + dirs[:] = [] + continue + seen_dirs.add(real_root) + for dir in dirs: dir_path = os.path.join(output_dir, os.path.relpath(os.path.join(root, dir), input_dir)) os.makedirs(dir_path, exist_ok=True) @@ -16,6 +29,7 @@ def copy_structure(input_dir, output_dir): os.makedirs(os.path.dirname(file_path), exist_ok=True) shutil.copy(os.path.join(root, file), file_path) + def replace_url_with_content(file_content): urls = re.findall(r'\[\[URL:([^\]]*)\]\]', file_content) for url in set(urls): @@ -27,11 +41,13 @@ def replace_url_with_content(file_content): raise ValueError(f"Problem downloading {error_details}") return file_content + def make_substitutions(file_content, substitutions): for key, value in substitutions.items(): file_content = file_content.replace(f"[[{key}]]", value) return file_content + def process_files(output_dir, substitutions): for root, dirs, files in os.walk(output_dir): for file in files: @@ -46,17 +62,18 @@ def process_files(output_dir, substitutions): f.write(new_content) f.truncate() + def main(input_dir, output_dir, substitutions): print(f"Processing {input_dir} to {output_dir}") copy_structure(input_dir, output_dir) print(f"Copied directory structure from {input_dir} to {output_dir}") process_files(output_dir, substitutions) + if __name__ == "__main__": import sys input_dir = sys.argv[1] output_dir = sys.argv[2] - substitutions = {} for var in os.environ: diff --git a/kubernetes/config-processor/test_config_processor.py b/kubernetes/config-processor/test_config_processor.py new file mode 100644 index 0000000000..406548ba8f --- /dev/null +++ b/kubernetes/config-processor/test_config_processor.py @@ -0,0 +1,43 @@ +import importlib.util +import os +import tempfile +import unittest +from pathlib import Path + + +spec = importlib.util.spec_from_file_location( + "config_processor", + Path(__file__).with_name("config-processor.py"), +) +config_processor = importlib.util.module_from_spec(spec) +spec.loader.exec_module(config_processor) + + +class CopyStructureTest(unittest.TestCase): + def test_copies_visible_tree_from_kubernetes_projected_volume(self): + with tempfile.TemporaryDirectory() as temp_dir: + input_dir = Path(temp_dir) / "input" + output_dir = Path(temp_dir) / "output" + data_dir = input_dir / "..2026_05_21_11_23_27.000000000" + organisms_dir = data_dir / "organisms" + + organisms_dir.mkdir(parents=True) + (data_dir / "website_config.json").write_text('{"organisms": {}}') + (organisms_dir / "cchf.json").write_text('{"schema": {"organismName": "CCHF"}}') + + os.symlink(data_dir.name, input_dir / "..data") + os.symlink("..data/website_config.json", input_dir / "website_config.json") + os.symlink("..data/organisms", input_dir / "organisms") + + config_processor.copy_structure(input_dir, output_dir) + + self.assertEqual((output_dir / "website_config.json").read_text(), '{"organisms": {}}') + self.assertEqual( + (output_dir / "organisms" / "cchf.json").read_text(), + '{"schema": {"organismName": "CCHF"}}', + ) + self.assertFalse((output_dir / data_dir.name).exists()) + + +if __name__ == "__main__": + unittest.main() From c16f706fbaf6668ea8e2772caa81f6dd287f7e8c Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Thu, 21 May 2026 13:06:09 +0100 Subject: [PATCH 3/8] Remove integration image wait --- .github/workflows/integration-tests.yml | 34 ------------------------- 1 file changed, 34 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index bb0521cee5..e640029ef0 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -61,40 +61,6 @@ jobs: run: echo "sha=${sha::7}" >> $GITHUB_ENV - name: Get runner IP run: curl -4 ifconfig.me - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Wait for Docker images - run: | - images=( - backend - config-processor - ena-submission - ena-submission-flyway - ingest - keycloakify - loculus-silo - preprocessing-dummy - preprocessing-nextclade - taxonomy-service - website - ) - deadline=$((SECONDS + 900)) - - for image in "${images[@]}"; do - full_image="ghcr.io/loculus-project/${image}:commit-${sha}" - until docker manifest inspect "$full_image" > /dev/null 2>&1; do - if [ "$SECONDS" -ge "$deadline" ]; then - echo "Timed out waiting for $full_image" - exit 1 - fi - echo "Waiting for $full_image" - sleep 10 - done - done - name: Collect Workflow Telemetry uses: catchpoint/workflow-telemetry-action@v2 - name: Checkout repository From 4019536b71aa1733ca50d6143291ca8b0b014228 Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 26 May 2026 16:43:12 +0100 Subject: [PATCH 4/8] Simplify split website config handling --- deploy.py | 23 +++++------- .../config-processor/config-processor.py | 24 +----------- .../loculus/templates/_config-processor.tpl | 16 -------- .../loculus/templates/loculus-website.yaml | 22 +++++++---- website/src/config.spec.ts | 37 ++----------------- website/src/config.ts | 33 +++++------------ 6 files changed, 38 insertions(+), 117 deletions(-) diff --git a/deploy.py b/deploy.py index ab4377ea0e..41a7dda665 100755 --- a/deploy.py +++ b/deploy.py @@ -486,12 +486,17 @@ def generate_config( return parsed_yaml = list(yaml.full_load_all(helm_output)) + + def write_config(path, contents): + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + f.write(contents) + print(f"Wrote config to {path}") + if any(substring in template for substring in ["ingest", "preprocessing"]): for doc in parsed_yaml: config_data = yaml.safe_load(doc["data"][configmap_path.name]) - with open(output_path.with_suffix(f".{config_data['organism']}.yaml"), "w") as f: - yaml.dump(config_data, f) - print(f"Wrote config to {f.name}") + write_config(output_path.with_suffix(f".{config_data['organism']}.yaml"), yaml.dump(config_data)) return wrote_config = False @@ -500,23 +505,15 @@ def generate_config( if configmap_path.name not in config_data: continue - with open(output_path, "w") as f: - f.write(config_data[configmap_path.name]) - - print(f"Wrote config to {output_path}") + write_config(output_path, config_data[configmap_path.name]) wrote_config = True if configmap_path.name == "website_config.json": - organisms_dir = output_path.parent / "organisms" for doc in parsed_yaml: for filename, config_data in doc.get("data", {}).items(): if filename in ["website_config.json", "runtime_config.json"] or not filename.endswith(".json"): continue - organisms_dir.mkdir(exist_ok=True) - organism_config_path = organisms_dir / filename - with open(organism_config_path, "w") as f: - f.write(config_data) - print(f"Wrote config to {organism_config_path}") + write_config(output_path.parent / "organisms" / filename, config_data) if not wrote_config: raise RuntimeError(f"Could not find {configmap_path.name} in rendered template {template}") diff --git a/kubernetes/config-processor/config-processor.py b/kubernetes/config-processor/config-processor.py index 73152ebb21..86f865d912 100644 --- a/kubernetes/config-processor/config-processor.py +++ b/kubernetes/config-processor/config-processor.py @@ -6,28 +6,8 @@ def copy_structure(input_dir, output_dir): - seen_dirs = set() - for root, dirs, files in os.walk(input_dir, followlinks=True): - # Kubernetes ConfigMap/projected volumes expose visible files as symlinks - # into hidden `..data` directories. Copy the visible tree only, while - # still following visible symlinked directories such as `organisms/`. - dirs[:] = [dir for dir in dirs if not dir.startswith("..")] - files = [file for file in files if not file.startswith("..")] - - real_root = os.path.realpath(root) - if real_root in seen_dirs: - dirs[:] = [] - continue - seen_dirs.add(real_root) - - for dir in dirs: - dir_path = os.path.join(output_dir, os.path.relpath(os.path.join(root, dir), input_dir)) - os.makedirs(dir_path, exist_ok=True) - for file in files: - file_path = os.path.join(output_dir, os.path.relpath(os.path.join(root, file), input_dir)) - # Make sure the directory exists - os.makedirs(os.path.dirname(file_path), exist_ok=True) - shutil.copy(os.path.join(root, file), file_path) + ignore_hidden_kubernetes_dirs = shutil.ignore_patterns("..*") + shutil.copytree(input_dir, output_dir, dirs_exist_ok=True, ignore=ignore_hidden_kubernetes_dirs) def replace_url_with_content(file_content): diff --git a/kubernetes/loculus/templates/_config-processor.tpl b/kubernetes/loculus/templates/_config-processor.tpl index dcaf110f20..a4d9483ef9 100644 --- a/kubernetes/loculus/templates/_config-processor.tpl +++ b/kubernetes/loculus/templates/_config-processor.tpl @@ -57,24 +57,8 @@ {{- define "loculus.configVolume" -}} - name: {{ .name }} - {{- if .configmaps }} - projected: - sources: - {{- range .configmaps }} - - configMap: - name: {{ .name }} - {{- if .items }} - items: - {{- range .items }} - - key: {{ .key }} - path: {{ .path | quote }} - {{- end }} - {{- end }} - {{- end }} - {{- else }} configMap: name: {{ if .configmap }}{{ .configmap }}{{ else }}{{ .name }}{{ end }} - {{- end }} - name: {{ .name }}-processed emptyDir: {} {{- end }} diff --git a/kubernetes/loculus/templates/loculus-website.yaml b/kubernetes/loculus/templates/loculus-website.yaml index 9644b1581d..3e3c862983 100644 --- a/kubernetes/loculus/templates/loculus-website.yaml +++ b/kubernetes/loculus/templates/loculus-website.yaml @@ -1,12 +1,5 @@ {{- $dockerTag := include "loculus.dockerTag" .Values }} {{- if not .Values.disableWebsite }} -{{- $websiteConfigMaps := list (dict "name" "loculus-website-config") }} -{{- range $_, $item := (include "loculus.enabledOrganisms" . | fromJson).organisms }} -{{- $websiteConfigMaps = append $websiteConfigMaps (dict - "name" (include "loculus.websiteOrganismConfigMapName" $item.key) - "items" (list (dict "key" (printf "%s.json" $item.key) "path" (printf "organisms/%s.json" $item.key))) -) }} -{{- end }} --- apiVersion: apps/v1 kind: Deployment @@ -56,5 +49,18 @@ spec: imagePullSecrets: - name: custom-website-sealed-secret volumes: -{{ include "loculus.configVolume" (dict "name" "loculus-website-config" "configmaps" $websiteConfigMaps) | nindent 8 }} + - name: loculus-website-config + projected: + sources: + - configMap: + name: loculus-website-config + {{ range $_, $item := (include "loculus.enabledOrganisms" . | fromJson).organisms }} + - configMap: + name: {{ include "loculus.websiteOrganismConfigMapName" $item.key }} + items: + - key: {{ printf "%s.json" $item.key | quote }} + path: {{ printf "organisms/%s.json" $item.key | quote }} + {{ end }} + - name: loculus-website-config-processed + emptyDir: {} {{- end }} diff --git a/website/src/config.spec.ts b/website/src/config.spec.ts index da32c26de4..b5258aae03 100644 --- a/website/src/config.spec.ts +++ b/website/src/config.spec.ts @@ -41,16 +41,7 @@ describe('validateWebsiteConfig', () => { ...defaultConfig, organisms: { dummyOrganism: { - schema: { - organismName: 'dummy', - inputFields: [], - tableColumns: [], - primaryKey: '', - metadata: [], - defaultOrderBy: '', - defaultOrder: 'ascending', - submissionDataTypes: { consensusSequences: false }, - }, + ...defaultOrganismConfig('dummy'), referenceGenomes: SINGLE_SEG_MULTI_REF_REFERENCEGENOMES_SCHEMA, }, }, @@ -97,30 +88,8 @@ describe('configuredOrganismsFromConfig', () => { const organisms = configuredOrganismsFromConfig({ ...defaultConfig, organisms: { - zika: { - schema: { - organismName: 'Zika Virus', - inputFields: [], - tableColumns: [], - primaryKey: '', - metadata: [], - defaultOrderBy: '', - defaultOrder: 'ascending', - submissionDataTypes: { consensusSequences: false }, - }, - }, - andes: { - schema: { - organismName: 'Andes Virus [Hantavirus]', - inputFields: [], - tableColumns: [], - primaryKey: '', - metadata: [], - defaultOrderBy: '', - defaultOrder: 'ascending', - submissionDataTypes: { consensusSequences: false }, - }, - }, + zika: defaultOrganismConfig('Zika Virus'), + andes: defaultOrganismConfig('Andes Virus [Hantavirus]'), }, }); diff --git a/website/src/config.ts b/website/src/config.ts index 48073a898a..14256975b1 100644 --- a/website/src/config.ts +++ b/website/src/config.ts @@ -73,34 +73,19 @@ export function getWebsiteConfig(): WebsiteConfig { export function readWebsiteConfigFromDir(configDir: string): WebsiteConfig { const config = readTypedConfigFile(configDir, 'website_config.json', websiteConfig); - const organismConfigs = readSplitOrganismConfigs(configDir); - if (Object.keys(organismConfigs).length === 0) { + const organismsDir = path.join(configDir, 'organisms'); + if (!fs.existsSync(organismsDir)) { return config; } - return { - ...config, - organisms: { - ...config.organisms, - ...organismConfigs, - }, - }; -} -function readSplitOrganismConfigs(configDir: string): Record { - const organismsDir = path.join(configDir, 'organisms'); - if (!fs.existsSync(organismsDir)) { - return {}; + for (const fileName of fs.readdirSync(organismsDir).filter((fileName) => fileName.endsWith('.json')).sort()) { + config.organisms[path.basename(fileName, '.json')] = readTypedConfigFile( + configDir, + path.join('organisms', fileName), + instanceConfig, + ); } - return Object.fromEntries( - fs - .readdirSync(organismsDir) - .filter((fileName) => fileName.endsWith('.json')) - .sort() - .map((fileName) => { - const organism = path.basename(fileName, '.json'); - return [organism, readTypedConfigFile(configDir, path.join('organisms', fileName), instanceConfig)]; - }), - ); + return config; } export function getContactConfig(websiteConfig: WebsiteConfig) { From 53781255fc5f6aec8f7639c7d18f649507f18a28 Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 26 May 2026 16:58:22 +0100 Subject: [PATCH 5/8] Simplify local website config generation --- .gitignore | 1 - deploy.py | 27 +++++++++++++++------------ website/.gitignore | 1 - 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 5de018be2b..ec795ca814 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,6 @@ __pycache__/ /website/tests/config/*.json /website/tests/config/*.yaml -/website/tests/config/organisms/ .venv* diff --git a/deploy.py b/deploy.py index 41a7dda665..d3be0021c2 100755 --- a/deploy.py +++ b/deploy.py @@ -499,24 +499,27 @@ def write_config(path, contents): write_config(output_path.with_suffix(f".{config_data['organism']}.yaml"), yaml.dump(config_data)) return - wrote_config = False for doc in parsed_yaml: config_data = doc.get("data", {}) if configmap_path.name not in config_data: continue - write_config(output_path, config_data[configmap_path.name]) - wrote_config = True - - if configmap_path.name == "website_config.json": - for doc in parsed_yaml: - for filename, config_data in doc.get("data", {}).items(): - if filename in ["website_config.json", "runtime_config.json"] or not filename.endswith(".json"): - continue - write_config(output_path.parent / "organisms" / filename, config_data) + output_config = config_data[configmap_path.name] + if configmap_path.name == "website_config.json": + output_config = json.dumps({ + **json.loads(output_config), + "organisms": { + Path(filename).stem: json.loads(data) + for doc in parsed_yaml + for filename, data in doc.get("data", {}).items() + if filename not in ["website_config.json", "runtime_config.json"] and filename.endswith(".json") + }, + }) + + write_config(output_path, output_config) + return - if not wrote_config: - raise RuntimeError(f"Could not find {configmap_path.name} in rendered template {template}") + raise RuntimeError(f"Could not find {configmap_path.name} in rendered template {template}") def get_codespace_params(codespace_name): diff --git a/website/.gitignore b/website/.gitignore index 24c18b66b7..0fa653ef3f 100644 --- a/website/.gitignore +++ b/website/.gitignore @@ -22,7 +22,6 @@ justfile /tests/config/backend_config.json /tests/config/website_config.json /tests/config/runtime_config.json -/tests/config/organisms/ /tests/config/*.yaml From bff4fda6109b0d5682968dadecf6f3f6c0c15890 Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 26 May 2026 17:00:20 +0100 Subject: [PATCH 6/8] Remove config processor unit test CI --- .../workflows/config-preprocessor-image.yml | 7 --- .../config-processor/test_config_processor.py | 43 ------------------- 2 files changed, 50 deletions(-) delete mode 100644 kubernetes/config-processor/test_config_processor.py diff --git a/.github/workflows/config-preprocessor-image.yml b/.github/workflows/config-preprocessor-image.yml index b04d1bfa4e..7795bab4f2 100644 --- a/.github/workflows/config-preprocessor-image.yml +++ b/.github/workflows/config-preprocessor-image.yml @@ -42,13 +42,6 @@ jobs: - uses: actions/checkout@v6 - name: Shorten sha run: echo "sha=${sha::7}" >> $GITHUB_ENV - - uses: actions/setup-python@v6 - with: - python-version: '3.13' - - name: Run unit tests - run: | - python3 -m pip install -r requirements.txt - python3 -m unittest - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry diff --git a/kubernetes/config-processor/test_config_processor.py b/kubernetes/config-processor/test_config_processor.py deleted file mode 100644 index 406548ba8f..0000000000 --- a/kubernetes/config-processor/test_config_processor.py +++ /dev/null @@ -1,43 +0,0 @@ -import importlib.util -import os -import tempfile -import unittest -from pathlib import Path - - -spec = importlib.util.spec_from_file_location( - "config_processor", - Path(__file__).with_name("config-processor.py"), -) -config_processor = importlib.util.module_from_spec(spec) -spec.loader.exec_module(config_processor) - - -class CopyStructureTest(unittest.TestCase): - def test_copies_visible_tree_from_kubernetes_projected_volume(self): - with tempfile.TemporaryDirectory() as temp_dir: - input_dir = Path(temp_dir) / "input" - output_dir = Path(temp_dir) / "output" - data_dir = input_dir / "..2026_05_21_11_23_27.000000000" - organisms_dir = data_dir / "organisms" - - organisms_dir.mkdir(parents=True) - (data_dir / "website_config.json").write_text('{"organisms": {}}') - (organisms_dir / "cchf.json").write_text('{"schema": {"organismName": "CCHF"}}') - - os.symlink(data_dir.name, input_dir / "..data") - os.symlink("..data/website_config.json", input_dir / "website_config.json") - os.symlink("..data/organisms", input_dir / "organisms") - - config_processor.copy_structure(input_dir, output_dir) - - self.assertEqual((output_dir / "website_config.json").read_text(), '{"organisms": {}}') - self.assertEqual( - (output_dir / "organisms" / "cchf.json").read_text(), - '{"schema": {"organismName": "CCHF"}}', - ) - self.assertFalse((output_dir / data_dir.name).exists()) - - -if __name__ == "__main__": - unittest.main() From 46cd080ad0be258174fe7410719873aa3154665f Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 26 May 2026 17:02:43 +0100 Subject: [PATCH 7/8] Clarify split website config merge --- deploy.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/deploy.py b/deploy.py index d3be0021c2..777fe11fcf 100755 --- a/deploy.py +++ b/deploy.py @@ -32,6 +32,7 @@ CLUSTER_NAME = "testCluster" HELM_RELEASE_NAME = "preview" HELM_CHART_DIR = ROOT_DIR / "kubernetes" / "loculus" +WEBSITE_ORGANISM_CONFIGMAP_PREFIX = "loculus-web-org-config-" # By default, uses K3s v1.31: https://hub.docker.com/r/rancher/k3s/tags?name=v1.31 # K3s v1.31 is the latest version which installs Traefik v2 (Traefik v3 is not yet supported by Loculus) @@ -506,15 +507,7 @@ def write_config(path, contents): output_config = config_data[configmap_path.name] if configmap_path.name == "website_config.json": - output_config = json.dumps({ - **json.loads(output_config), - "organisms": { - Path(filename).stem: json.loads(data) - for doc in parsed_yaml - for filename, data in doc.get("data", {}).items() - if filename not in ["website_config.json", "runtime_config.json"] and filename.endswith(".json") - }, - }) + output_config = merge_split_website_config(output_config, parsed_yaml) write_config(output_path, output_config) return @@ -522,6 +515,21 @@ def write_config(path, contents): raise RuntimeError(f"Could not find {configmap_path.name} in rendered template {template}") +def merge_split_website_config(website_config, rendered_docs): + config = json.loads(website_config) + config["organisms"] = {} + + for doc in rendered_docs: + configmap_name = doc.get("metadata", {}).get("name", "") + if not configmap_name.startswith(WEBSITE_ORGANISM_CONFIGMAP_PREFIX): + continue + + for filename, data in doc.get("data", {}).items(): + config["organisms"][Path(filename).stem] = json.loads(data) + + return json.dumps(config) + + def get_codespace_params(codespace_name): public_runtime_config = { "websiteUrl": f"https://{codespace_name}-3000.app.github.dev", From 962bffe5214274e687128ebcbdd6b1e0eb4d3bb9 Mon Sep 17 00:00:00 2001 From: theosanderson-agent Date: Tue, 26 May 2026 17:18:10 +0100 Subject: [PATCH 8/8] Format website config reader --- website/src/config.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/website/src/config.ts b/website/src/config.ts index 14256975b1..15e75a47ec 100644 --- a/website/src/config.ts +++ b/website/src/config.ts @@ -78,7 +78,10 @@ export function readWebsiteConfigFromDir(configDir: string): WebsiteConfig { return config; } - for (const fileName of fs.readdirSync(organismsDir).filter((fileName) => fileName.endsWith('.json')).sort()) { + for (const fileName of fs + .readdirSync(organismsDir) + .filter((fileName) => fileName.endsWith('.json')) + .sort()) { config.organisms[path.basename(fileName, '.json')] = readTypedConfigFile( configDir, path.join('organisms', fileName),