diff --git a/.gitignore b/.gitignore index e07c44ccfe..ec795ca814 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,9 @@ __pycache__/ .astro/ +/website/tests/config/*.json +/website/tests/config/*.yaml + .venv* diff --git a/deploy.py b/deploy.py index 332c2efce0..777fe11fcf 100755 --- a/deploy.py +++ b/deploy.py @@ -32,6 +32,7 @@ CLUSTER_NAME = "testCluster" HELM_RELEASE_NAME = "preview" HELM_CHART_DIR = ROOT_DIR / "kubernetes" / "loculus" +WEBSITE_ORGANISM_CONFIGMAP_PREFIX = "loculus-web-org-config-" # By default, uses K3s v1.31: https://hub.docker.com/r/rancher/k3s/tags?name=v1.31 # K3s v1.31 is the latest version which installs Traefik v2 (Traefik v3 is not yet supported by Loculus) @@ -486,19 +487,47 @@ def generate_config( return parsed_yaml = list(yaml.full_load_all(helm_output)) - if len(parsed_yaml) == 1: - config_data = parsed_yaml[0]["data"][configmap_path.name] - with open(output_path, "w") as f: - f.write(config_data) + def write_config(path, contents): + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + f.write(contents) + print(f"Wrote config to {path}") - print(f"Wrote config to {output_path}") - elif any(substring in template for substring in ["ingest", "preprocessing"]): + if any(substring in template for substring in ["ingest", "preprocessing"]): for doc in parsed_yaml: config_data = yaml.safe_load(doc["data"][configmap_path.name]) - with open(output_path.with_suffix(f".{config_data['organism']}.yaml"), "w") as f: - yaml.dump(config_data, f) - print(f"Wrote config to {f.name}") + write_config(output_path.with_suffix(f".{config_data['organism']}.yaml"), yaml.dump(config_data)) + return + + for doc in parsed_yaml: + config_data = doc.get("data", {}) + if configmap_path.name not in config_data: + continue + + output_config = config_data[configmap_path.name] + if configmap_path.name == "website_config.json": + output_config = merge_split_website_config(output_config, parsed_yaml) + + write_config(output_path, output_config) + return + + raise RuntimeError(f"Could not find {configmap_path.name} in rendered template {template}") + + +def merge_split_website_config(website_config, rendered_docs): + config = json.loads(website_config) + config["organisms"] = {} + + for doc in rendered_docs: + configmap_name = doc.get("metadata", {}).get("name", "") + if not configmap_name.startswith(WEBSITE_ORGANISM_CONFIGMAP_PREFIX): + continue + + for filename, data in doc.get("data", {}).items(): + config["organisms"][Path(filename).stem] = json.loads(data) + + return json.dumps(config) def get_codespace_params(codespace_name): diff --git a/kubernetes/config-processor/config-processor.py b/kubernetes/config-processor/config-processor.py index 8661766699..7994774503 100644 --- a/kubernetes/config-processor/config-processor.py +++ b/kubernetes/config-processor/config-processor.py @@ -11,15 +11,8 @@ def copy_structure(input_dir, output_dir): - for root, dirs, files in os.walk(input_dir): - for dir in dirs: - dir_path = os.path.join(output_dir, os.path.relpath(os.path.join(root, dir), input_dir)) - os.makedirs(dir_path, exist_ok=True) - for file in files: - file_path = os.path.join(output_dir, os.path.relpath(os.path.join(root, file), input_dir)) - # Make sure the directory exists - os.makedirs(os.path.dirname(file_path), exist_ok=True) - shutil.copy(os.path.join(root, file), file_path) + ignore_hidden_kubernetes_dirs = shutil.ignore_patterns("..*") + shutil.copytree(input_dir, output_dir, dirs_exist_ok=True, ignore=ignore_hidden_kubernetes_dirs) def download_urls(urls): @@ -66,11 +59,13 @@ def replace_url_with_content(file_content, downloaded_content): file_content = file_content.replace(f"[[URL:{url}]]", downloaded_content[url]) return file_content + def make_substitutions(file_content, substitutions): for key, value in substitutions.items(): file_content = file_content.replace(f"[[{key}]]", value) return file_content + def collect_urls(output_dir): urls = set() for root, dirs, files in os.walk(output_dir): @@ -80,6 +75,7 @@ def collect_urls(output_dir): urls.update(re.findall(r'\[\[URL:([^\]]*)\]\]', f.read())) return urls + def process_files(output_dir, substitutions): downloaded_content = download_urls(collect_urls(output_dir)) for root, dirs, files in os.walk(output_dir): @@ -95,17 +91,18 @@ def process_files(output_dir, substitutions): f.write(new_content) f.truncate() + def main(input_dir, output_dir, substitutions): print(f"Processing {input_dir} to {output_dir}") copy_structure(input_dir, output_dir) print(f"Copied directory structure from {input_dir} to {output_dir}") process_files(output_dir, substitutions) + if __name__ == "__main__": import sys input_dir = sys.argv[1] output_dir = sys.argv[2] - substitutions = {} for var in os.environ: diff --git a/kubernetes/loculus/templates/_config-processor.tpl b/kubernetes/loculus/templates/_config-processor.tpl index cfea6861d9..a4d9483ef9 100644 --- a/kubernetes/loculus/templates/_config-processor.tpl +++ b/kubernetes/loculus/templates/_config-processor.tpl @@ -62,3 +62,7 @@ - name: {{ .name }}-processed emptyDir: {} {{- end }} + +{{- define "loculus.websiteOrganismConfigMapName" -}} +{{- printf "loculus-web-org-config-%s" . | trunc 63 | trimSuffix "-" -}} +{{- end }} diff --git a/kubernetes/loculus/templates/loculus-website-config.yaml b/kubernetes/loculus/templates/loculus-website-config.yaml index 3c5df57465..2143f244a5 100644 --- a/kubernetes/loculus/templates/loculus-website-config.yaml +++ b/kubernetes/loculus/templates/loculus-website-config.yaml @@ -1,3 +1,6 @@ +{{- $websiteConfig := include "loculus.generateWebsiteConfig" . | fromYaml }} +{{- $organisms := $websiteConfig.organisms }} +{{- $_ := set $websiteConfig "organisms" (dict) }} --- apiVersion: v1 kind: ConfigMap @@ -5,7 +8,7 @@ metadata: name: loculus-website-config data: website_config.json: | - {{ include "loculus.generateWebsiteConfig" . | fromYaml | toJson }} + {{ $websiteConfig | toJson }} runtime_config.json: | { "name" : "{{ $.Values.name }}", @@ -28,3 +31,13 @@ data: }, "backendKeycloakClientSecret" : "[[backendKeycloakClientSecret]]" } +{{- range $key, $organismConfig := $organisms }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "loculus.websiteOrganismConfigMapName" $key }} +data: + {{ $key }}.json: | + {{ $organismConfig | toJson }} +{{- end }} diff --git a/kubernetes/loculus/templates/loculus-website.yaml b/kubernetes/loculus/templates/loculus-website.yaml index a7e692aabb..3e3c862983 100644 --- a/kubernetes/loculus/templates/loculus-website.yaml +++ b/kubernetes/loculus/templates/loculus-website.yaml @@ -49,5 +49,18 @@ spec: imagePullSecrets: - name: custom-website-sealed-secret volumes: -{{ include "loculus.configVolume" (dict "name" "loculus-website-config") | nindent 8 }} -{{- end }} \ No newline at end of file + - name: loculus-website-config + projected: + sources: + - configMap: + name: loculus-website-config + {{ range $_, $item := (include "loculus.enabledOrganisms" . | fromJson).organisms }} + - configMap: + name: {{ include "loculus.websiteOrganismConfigMapName" $item.key }} + items: + - key: {{ printf "%s.json" $item.key | quote }} + path: {{ printf "organisms/%s.json" $item.key | quote }} + {{ end }} + - name: loculus-website-config-processed + emptyDir: {} +{{- end }} diff --git a/website/README.md b/website/README.md index 4da8d28dc4..d06ddf1b7f 100644 --- a/website/README.md +++ b/website/README.md @@ -32,7 +32,8 @@ See `.env.docker` for the required variables. Furthermore, the website requires config files that need to be present at runtime in the directory specified in the `CONFIG_DIR` environment variable: -- `website_config.json`: Contains configuration on the underlying organism. It's similar to the database config file that LAPIS uses. +- `website_config.json`: Contains global website configuration. It can also contain organism configuration for local or legacy setups. +- `organisms/*.json`: Optional per-organism configuration files. When present, these are merged into the `organisms` field from `website_config.json`. - `reference_genomes.json`: Defines names for segments of the genome and amino acids. It's equal to the file that LAPIS uses. - `runtime_config.json`: Contains configuration that specific for a deployed instance of the website. diff --git a/website/src/config.spec.ts b/website/src/config.spec.ts index 38e9b54425..b5258aae03 100644 --- a/website/src/config.spec.ts +++ b/website/src/config.spec.ts @@ -1,7 +1,11 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + import { describe, expect, it } from 'vitest'; -import { configuredOrganismsFromConfig, validateWebsiteConfig } from './config.ts'; -import type { WebsiteConfig } from './types/config.ts'; +import { configuredOrganismsFromConfig, readWebsiteConfigFromDir, validateWebsiteConfig } from './config.ts'; +import type { InstanceConfig, WebsiteConfig } from './types/config.ts'; import { SINGLE_SEG_MULTI_REF_REFERENCEGENOMES_SCHEMA } from './types/referenceGenomes.spec.ts'; const defaultConfig: WebsiteConfig = { @@ -18,22 +22,26 @@ const defaultConfig: WebsiteConfig = { readOnlyMode: false, }; +const defaultOrganismConfig = (organismName: string): InstanceConfig => ({ + schema: { + organismName, + inputFields: [], + tableColumns: [], + primaryKey: '', + metadata: [], + defaultOrderBy: '', + defaultOrder: 'ascending', + submissionDataTypes: { consensusSequences: false }, + }, +}); + describe('validateWebsiteConfig', () => { it('should fail when "referenceIdentifierField" is not defined for an organism with multiple references', () => { const errors = validateWebsiteConfig({ ...defaultConfig, organisms: { dummyOrganism: { - schema: { - organismName: 'dummy', - inputFields: [], - tableColumns: [], - primaryKey: '', - metadata: [], - defaultOrderBy: '', - defaultOrder: 'ascending', - submissionDataTypes: { consensusSequences: false }, - }, + ...defaultOrganismConfig('dummy'), referenceGenomes: SINGLE_SEG_MULTI_REF_REFERENCEGENOMES_SCHEMA, }, }, @@ -46,35 +54,42 @@ describe('validateWebsiteConfig', () => { }); }); +describe('readWebsiteConfigFromDir', () => { + it('merges organism configs from the organisms directory', () => { + const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'loculus-website-config-')); + try { + fs.writeFileSync( + path.join(configDir, 'website_config.json'), + JSON.stringify({ + ...defaultConfig, + organisms: { + zika: defaultOrganismConfig('Zika Virus'), + }, + }), + ); + fs.mkdirSync(path.join(configDir, 'organisms')); + fs.writeFileSync( + path.join(configDir, 'organisms', 'andes.json'), + JSON.stringify(defaultOrganismConfig('Andes Virus [Hantavirus]')), + ); + + const config = readWebsiteConfigFromDir(configDir); + + expect(Object.keys(config.organisms).sort()).toEqual(['andes', 'zika']); + expect(config.organisms.andes.schema.organismName).toBe('Andes Virus [Hantavirus]'); + } finally { + fs.rmSync(configDir, { recursive: true, force: true }); + } + }); +}); + describe('configuredOrganismsFromConfig', () => { it('sorts organisms by display name instead of key', () => { const organisms = configuredOrganismsFromConfig({ ...defaultConfig, organisms: { - zika: { - schema: { - organismName: 'Zika Virus', - inputFields: [], - tableColumns: [], - primaryKey: '', - metadata: [], - defaultOrderBy: '', - defaultOrder: 'ascending', - submissionDataTypes: { consensusSequences: false }, - }, - }, - andes: { - schema: { - organismName: 'Andes Virus [Hantavirus]', - inputFields: [], - tableColumns: [], - primaryKey: '', - metadata: [], - defaultOrderBy: '', - defaultOrder: 'ascending', - submissionDataTypes: { consensusSequences: false }, - }, - }, + zika: defaultOrganismConfig('Zika Virus'), + andes: defaultOrganismConfig('Andes Virus [Hantavirus]'), }, }); diff --git a/website/src/config.ts b/website/src/config.ts index 69a635febe..15e75a47ec 100644 --- a/website/src/config.ts +++ b/website/src/config.ts @@ -10,6 +10,7 @@ import { type Schema, type SequenceFlaggingConfig, type WebsiteConfig, + instanceConfig, websiteConfig, } from './types/config.ts'; import { type ReferenceGenomesInfo } from './types/referencesGenomes.ts'; @@ -60,7 +61,7 @@ export function validateWebsiteConfig(config: WebsiteConfig): Error[] { export function getWebsiteConfig(): WebsiteConfig { if (_config === null) { - const config = readTypedConfigFile('website_config.json', websiteConfig); + const config = readWebsiteConfigFromDir(getConfigDir()); const validationErrors = validateWebsiteConfig(config); if (validationErrors.length > 0) { throw new AggregateError(validationErrors, 'There were validation errors in the website_config.json'); @@ -70,6 +71,26 @@ export function getWebsiteConfig(): WebsiteConfig { return _config; } +export function readWebsiteConfigFromDir(configDir: string): WebsiteConfig { + const config = readTypedConfigFile(configDir, 'website_config.json', websiteConfig); + const organismsDir = path.join(configDir, 'organisms'); + if (!fs.existsSync(organismsDir)) { + return config; + } + + for (const fileName of fs + .readdirSync(organismsDir) + .filter((fileName) => fileName.endsWith('.json')) + .sort()) { + config.organisms[path.basename(fileName, '.json')] = readTypedConfigFile( + configDir, + path.join('organisms', fileName), + instanceConfig, + ); + } + return config; +} + export function getContactConfig(websiteConfig: WebsiteConfig) { return { gitHubIssuesUrl: websiteConfig.gitHubIssuesUrl, @@ -234,7 +255,7 @@ export function getGroupedInputFields( } export function getRuntimeConfig(): RuntimeConfig { - _runtimeConfig ??= readTypedConfigFile('runtime_config.json', runtimeConfig); + _runtimeConfig ??= readTypedConfigFile(getConfigDir(), 'runtime_config.json', runtimeConfig); return _runtimeConfig; } @@ -261,8 +282,12 @@ export function getDataUseTermsAgreementHTML() { return getWebsiteConfig().dataUseTermsAgreementHTML; } -function readTypedConfigFile(fileName: string, schema: Schema): z.infer { - const configFilePath = path.join(getConfigDir(), fileName); +function readTypedConfigFile( + configDir: string, + fileName: string, + schema: Schema, +): z.infer { + const configFilePath = path.join(configDir, fileName); const json = JSON.parse(fs.readFileSync(configFilePath, 'utf8')); try { return schema.parse(json) as z.infer;