Split website organism configs

.github/workflows/config-preprocessor-image.yml

@@ -42,6 +42,13 @@ jobs:
       - uses: actions/checkout@v6
       - name: Shorten sha
         run: echo "sha=${sha::7}" >> $GITHUB_ENV
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.13'
+      - name: Run unit tests
+        run: |
+          python3 -m pip install -r requirements.txt
+          python3 -m unittest
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
       - name: Login to GitHub Container Registry

.gitignore

@@ -28,6 +28,10 @@ __pycache__/
 
 .astro/
 
+/website/tests/config/*.json
+/website/tests/config/*.yaml
+/website/tests/config/organisms/
+
 
 .venv*
 

deploy.py

@@ -486,19 +486,40 @@ def generate_config(
         return
 
     parsed_yaml = list(yaml.full_load_all(helm_output))
-    if len(parsed_yaml) == 1:
-        config_data = parsed_yaml[0]["data"][configmap_path.name]
-
-        with open(output_path, "w") as f:
-            f.write(config_data)
-
-        print(f"Wrote config to {output_path}")
-    elif any(substring in template for substring in ["ingest", "preprocessing"]):
+    if any(substring in template for substring in ["ingest", "preprocessing"]):
         for doc in parsed_yaml:
             config_data = yaml.safe_load(doc["data"][configmap_path.name])
             with open(output_path.with_suffix(f".{config_data['organism']}.yaml"), "w") as f:
                 yaml.dump(config_data, f)
                 print(f"Wrote config to {f.name}")
+        return
+
+    wrote_config = False
+    for doc in parsed_yaml:
+        config_data = doc.get("data", {})
+        if configmap_path.name not in config_data:
+            continue
+
+        with open(output_path, "w") as f:
+            f.write(config_data[configmap_path.name])
+
+        print(f"Wrote config to {output_path}")
+        wrote_config = True
+
+    if configmap_path.name == "website_config.json":
+        organisms_dir = output_path.parent / "organisms"
+        for doc in parsed_yaml:
+            for filename, config_data in doc.get("data", {}).items():
+                if filename in ["website_config.json", "runtime_config.json"] or not filename.endswith(".json"):
+                    continue
+                organisms_dir.mkdir(exist_ok=True)
+                organism_config_path = organisms_dir / filename
+                with open(organism_config_path, "w") as f:
+                    f.write(config_data)
+                print(f"Wrote config to {organism_config_path}")
+
+    if not wrote_config:
+        raise RuntimeError(f"Could not find {configmap_path.name} in rendered template {template}")
 
 
 def get_codespace_params(codespace_name):

kubernetes/config-processor/config-processor.py

@@ -6,7 +6,20 @@
 
 
 def copy_structure(input_dir, output_dir):
-    for root, dirs, files in os.walk(input_dir):
+    seen_dirs = set()
+    for root, dirs, files in os.walk(input_dir, followlinks=True):
+        # Kubernetes ConfigMap/projected volumes expose visible files as symlinks
+        # into hidden `..data` directories. Copy the visible tree only, while
+        # still following visible symlinked directories such as `organisms/`.
+        dirs[:] = [dir for dir in dirs if not dir.startswith("..")]
+        files = [file for file in files if not file.startswith("..")]
+
+        real_root = os.path.realpath(root)
+        if real_root in seen_dirs:
+            dirs[:] = []
+            continue
+        seen_dirs.add(real_root)
+
         for dir in dirs:
             dir_path = os.path.join(output_dir, os.path.relpath(os.path.join(root, dir), input_dir))
             os.makedirs(dir_path, exist_ok=True)
@@ -16,6 +29,7 @@ def copy_structure(input_dir, output_dir):
             os.makedirs(os.path.dirname(file_path), exist_ok=True)
             shutil.copy(os.path.join(root, file), file_path)
 
+
 def replace_url_with_content(file_content):
     urls = re.findall(r'\[\[URL:([^\]]*)\]\]', file_content)
     for url in set(urls):
@@ -27,11 +41,13 @@ def replace_url_with_content(file_content):
             raise ValueError(f"Problem downloading {error_details}")
     return file_content
 
+
 def make_substitutions(file_content, substitutions):
     for key, value in substitutions.items():
         file_content = file_content.replace(f"[[{key}]]", value)
     return file_content
 
+
 def process_files(output_dir, substitutions):
     for root, dirs, files in os.walk(output_dir):
         for file in files:
@@ -46,17 +62,18 @@ def process_files(output_dir, substitutions):
                     f.write(new_content)
                     f.truncate()
 
+
 def main(input_dir, output_dir, substitutions):
     print(f"Processing {input_dir} to {output_dir}")
     copy_structure(input_dir, output_dir)
     print(f"Copied directory structure from {input_dir} to {output_dir}")
     process_files(output_dir, substitutions)
 
+
 if __name__ == "__main__":
     import sys
     input_dir = sys.argv[1]
     output_dir = sys.argv[2]
-
 
     substitutions = {}
     for var in os.environ:

kubernetes/config-processor/test_config_processor.py

@@ -0,0 +1,43 @@
+import importlib.util
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+
+spec = importlib.util.spec_from_file_location(
+    "config_processor",
+    Path(__file__).with_name("config-processor.py"),
+)
+config_processor = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(config_processor)
+
+
+class CopyStructureTest(unittest.TestCase):
+    def test_copies_visible_tree_from_kubernetes_projected_volume(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            input_dir = Path(temp_dir) / "input"
+            output_dir = Path(temp_dir) / "output"
+            data_dir = input_dir / "..2026_05_21_11_23_27.000000000"
+            organisms_dir = data_dir / "organisms"
+
+            organisms_dir.mkdir(parents=True)
+            (data_dir / "website_config.json").write_text('{"organisms": {}}')
+            (organisms_dir / "cchf.json").write_text('{"schema": {"organismName": "CCHF"}}')
+
+            os.symlink(data_dir.name, input_dir / "..data")
+            os.symlink("..data/website_config.json", input_dir / "website_config.json")
+            os.symlink("..data/organisms", input_dir / "organisms")
+
+            config_processor.copy_structure(input_dir, output_dir)
+
+            self.assertEqual((output_dir / "website_config.json").read_text(), '{"organisms": {}}')
+            self.assertEqual(
+                (output_dir / "organisms" / "cchf.json").read_text(),
+                '{"schema": {"organismName": "CCHF"}}',
+            )
+            self.assertFalse((output_dir / data_dir.name).exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

kubernetes/loculus/templates/_config-processor.tpl

@@ -57,8 +57,28 @@
 
 {{- define "loculus.configVolume" -}}
 - name: {{ .name }}
+  {{- if .configmaps }}
+  projected:
+    sources:
+      {{- range .configmaps }}
+      - configMap:
+          name: {{ .name }}
+          {{- if .items }}
+          items:
+            {{- range .items }}
+            - key: {{ .key }}
+              path: {{ .path | quote }}
+            {{- end }}
+          {{- end }}
+      {{- end }}
+  {{- else }}
   configMap:
     name: {{ if .configmap }}{{ .configmap }}{{ else }}{{ .name }}{{ end }}
+  {{- end }}
 - name: {{ .name }}-processed
   emptyDir: {}
 {{- end }}
+
+{{- define "loculus.websiteOrganismConfigMapName" -}}
+{{- printf "loculus-web-org-config-%s" . | trunc 63 | trimSuffix "-" -}}
+{{- end }}

kubernetes/loculus/templates/loculus-website-config.yaml

@@ -1,11 +1,14 @@
+{{- $websiteConfig := include "loculus.generateWebsiteConfig" . | fromYaml }}
+{{- $organisms := $websiteConfig.organisms }}
+{{- $_ := set $websiteConfig "organisms" (dict) }}
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: loculus-website-config
 data:
   website_config.json: |
-    {{ include "loculus.generateWebsiteConfig" . | fromYaml | toJson }}
+    {{ $websiteConfig | toJson }}
   runtime_config.json: |
     {
         "name" : "{{ $.Values.name }}",
@@ -28,3 +31,13 @@ data:
         },
         "backendKeycloakClientSecret" : "[[backendKeycloakClientSecret]]"
     }
+{{- range $key, $organismConfig := $organisms }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "loculus.websiteOrganismConfigMapName" $key }}
+data:
+  {{ $key }}.json: |
+    {{ $organismConfig | toJson }}
+{{- end }}

kubernetes/loculus/templates/loculus-website.yaml

@@ -1,5 +1,12 @@
 {{- $dockerTag := include "loculus.dockerTag" .Values }}
 {{- if not .Values.disableWebsite }}
+{{- $websiteConfigMaps := list (dict "name" "loculus-website-config") }}
+{{- range $_, $item := (include "loculus.enabledOrganisms" . | fromJson).organisms }}
+{{- $websiteConfigMaps = append $websiteConfigMaps (dict
+  "name" (include "loculus.websiteOrganismConfigMapName" $item.key)
+  "items" (list (dict "key" (printf "%s.json" $item.key) "path" (printf "organisms/%s.json" $item.key)))
+) }}
+{{- end }}
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -49,5 +56,5 @@ spec:
       imagePullSecrets:
         - name: custom-website-sealed-secret
       volumes:
-{{ include "loculus.configVolume" (dict "name" "loculus-website-config") | nindent 8 }}
-{{- end }}
+{{ include "loculus.configVolume" (dict "name" "loculus-website-config" "configmaps" $websiteConfigMaps) | nindent 8 }}
+{{- end }}

website/.gitignore

@@ -22,6 +22,7 @@ justfile
 /tests/config/backend_config.json
 /tests/config/website_config.json
 /tests/config/runtime_config.json
+/tests/config/organisms/
 
 /tests/config/*.yaml
 

website/README.md

@@ -32,7 +32,8 @@ See `.env.docker` for the required variables.
 Furthermore, the website requires config files that need to be present at runtime in the directory
 specified in the `CONFIG_DIR` environment variable:
 
--   `website_config.json`: Contains configuration on the underlying organism. It's similar to the database config file that LAPIS uses.
+-   `website_config.json`: Contains global website configuration. It can also contain organism configuration for local or legacy setups.
+-   `organisms/*.json`: Optional per-organism configuration files. When present, these are merged into the `organisms` field from `website_config.json`.
 -   `reference_genomes.json`: Defines names for segments of the genome and amino acids. It's equal to the file that LAPIS uses.
 -   `runtime_config.json`: Contains configuration that specific for a deployed instance of the website.
 

website/src/config.spec.ts

@@ -1,7 +1,11 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
 import { describe, expect, it } from 'vitest';
 
-import { configuredOrganismsFromConfig, validateWebsiteConfig } from './config.ts';
-import type { WebsiteConfig } from './types/config.ts';
+import { configuredOrganismsFromConfig, readWebsiteConfigFromDir, validateWebsiteConfig } from './config.ts';
+import type { InstanceConfig, WebsiteConfig } from './types/config.ts';
 import { SINGLE_SEG_MULTI_REF_REFERENCEGENOMES_SCHEMA } from './types/referenceGenomes.spec.ts';
 
 const defaultConfig: WebsiteConfig = {
@@ -18,6 +22,19 @@ const defaultConfig: WebsiteConfig = {
     readOnlyMode: false,
 };
 
+const defaultOrganismConfig = (organismName: string): InstanceConfig => ({
+    schema: {
+        organismName,
+        inputFields: [],
+        tableColumns: [],
+        primaryKey: '',
+        metadata: [],
+        defaultOrderBy: '',
+        defaultOrder: 'ascending',
+        submissionDataTypes: { consensusSequences: false },
+    },
+});
+
 describe('validateWebsiteConfig', () => {
     it('should fail when "referenceIdentifierField" is not defined for an organism with multiple references', () => {
         const errors = validateWebsiteConfig({
@@ -46,6 +63,35 @@ describe('validateWebsiteConfig', () => {
     });
 });
 
+describe('readWebsiteConfigFromDir', () => {
+    it('merges organism configs from the organisms directory', () => {
+        const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'loculus-website-config-'));
+        try {
+            fs.writeFileSync(
+                path.join(configDir, 'website_config.json'),
+                JSON.stringify({
+                    ...defaultConfig,
+                    organisms: {
+                        zika: defaultOrganismConfig('Zika Virus'),
+                    },
+                }),
+            );
+            fs.mkdirSync(path.join(configDir, 'organisms'));
+            fs.writeFileSync(
+                path.join(configDir, 'organisms', 'andes.json'),
+                JSON.stringify(defaultOrganismConfig('Andes Virus [Hantavirus]')),
+            );
+
+            const config = readWebsiteConfigFromDir(configDir);
+
+            expect(Object.keys(config.organisms).sort()).toEqual(['andes', 'zika']);
+            expect(config.organisms.andes.schema.organismName).toBe('Andes Virus [Hantavirus]');
+        } finally {
+            fs.rmSync(configDir, { recursive: true, force: true });
+        }
+    });
+});
+
 describe('configuredOrganismsFromConfig', () => {
     it('sorts organisms by display name instead of key', () => {
         const organisms = configuredOrganismsFromConfig({