feat(website, backend): SeqSet citation tracking

backend/docs/db/schema.sql

@@ -4,8 +4,8 @@
 
 \restrict dummy
 
--- Dumped from database version 15.17 (Debian 15.17-1.pgdg13+1)
--- Dumped by pg_dump version 16.13 (Debian 16.13-1.pgdg13+1)
+-- Dumped from database version 15.18 (Debian 15.18-1.pgdg13+1)
+-- Dumped by pg_dump version 16.14 (Debian 16.14-1.pgdg13+1)
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -311,6 +311,44 @@ CREATE TABLE public.metadata_upload_aux_table (
 
 ALTER TABLE public.metadata_upload_aux_table OWNER TO postgres;
 
+--
+-- Name: seqset_citation_source; Type: TABLE; Schema: public; Owner: postgres
+--
+
+CREATE TABLE public.seqset_citation_source (
+    citation_source_id bigint NOT NULL,
+    source_doi text NOT NULL,
+    origin text NOT NULL,
+    title text NOT NULL,
+    year integer NOT NULL,
+    contributors jsonb NOT NULL,
+    CONSTRAINT seqset_citation_source_origin_check CHECK ((origin = ANY (ARRAY['CROSSREF'::text, 'CURATED'::text])))
+);
+
+
+ALTER TABLE public.seqset_citation_source OWNER TO postgres;
+
+--
+-- Name: seqset_citation_source_citation_source_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres
+--
+
+CREATE SEQUENCE public.seqset_citation_source_citation_source_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+ALTER SEQUENCE public.seqset_citation_source_citation_source_id_seq OWNER TO postgres;
+
+--
+-- Name: seqset_citation_source_citation_source_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres
+--
+
+ALTER SEQUENCE public.seqset_citation_source_citation_source_id_seq OWNED BY public.seqset_citation_source.citation_source_id;
+
+
 --
 -- Name: seqset_id_sequence; Type: SEQUENCE; Schema: public; Owner: postgres
 --
@@ -360,6 +398,19 @@ ALTER SEQUENCE public.seqset_records_seqset_record_id_seq OWNER TO postgres;
 ALTER SEQUENCE public.seqset_records_seqset_record_id_seq OWNED BY public.seqset_records.seqset_record_id;
 
 
+--
+-- Name: seqset_to_citation_source; Type: TABLE; Schema: public; Owner: postgres
+--
+
+CREATE TABLE public.seqset_to_citation_source (
+    citation_source_id bigint NOT NULL,
+    seqset_id text NOT NULL,
+    seqset_version bigint NOT NULL
+);
+
+
+ALTER TABLE public.seqset_to_citation_source OWNER TO postgres;
+
 --
 -- Name: seqset_to_records; Type: TABLE; Schema: public; Owner: postgres
 --
@@ -581,6 +632,13 @@ ALTER TABLE ONLY public.audit_log ALTER COLUMN id SET DEFAULT nextval('public.au
 ALTER TABLE ONLY public.groups_table ALTER COLUMN group_id SET DEFAULT nextval('public.groups_table_group_id_seq'::regclass);
 
 
+--
+-- Name: seqset_citation_source citation_source_id; Type: DEFAULT; Schema: public; Owner: postgres
+--
+
+ALTER TABLE ONLY public.seqset_citation_source ALTER COLUMN citation_source_id SET DEFAULT nextval('public.seqset_citation_source_citation_source_id_seq'::regclass);
+
+
 --
 -- Name: seqset_records seqset_record_id; Type: DEFAULT; Schema: public; Owner: postgres
 --
@@ -682,6 +740,22 @@ ALTER TABLE ONLY public.metadata_upload_aux_table
     ADD CONSTRAINT metadata_upload_aux_table_upload_id_accession_key UNIQUE (upload_id, accession);
 
 
+--
+-- Name: seqset_citation_source seqset_citation_source_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres
+--
+
+ALTER TABLE ONLY public.seqset_citation_source
+    ADD CONSTRAINT seqset_citation_source_pkey PRIMARY KEY (citation_source_id);
+
+
+--
+-- Name: seqset_citation_source seqset_citation_source_source_doi_key; Type: CONSTRAINT; Schema: public; Owner: postgres
+--
+
+ALTER TABLE ONLY public.seqset_citation_source
+    ADD CONSTRAINT seqset_citation_source_source_doi_key UNIQUE (source_doi);
+
+
 --
 -- Name: seqset_records seqset_records_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres
 --
@@ -690,6 +764,14 @@ ALTER TABLE ONLY public.seqset_records
     ADD CONSTRAINT seqset_records_pkey PRIMARY KEY (seqset_record_id);
 
 
+--
+-- Name: seqset_to_citation_source seqset_to_citation_source_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres
+--
+
+ALTER TABLE ONLY public.seqset_to_citation_source
+    ADD CONSTRAINT seqset_to_citation_source_pkey PRIMARY KEY (citation_source_id, seqset_id, seqset_version);
+
+
 --
 -- Name: seqset_to_records seqset_to_records_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres
 --
@@ -881,6 +963,22 @@ ALTER TABLE ONLY public.files
     ADD CONSTRAINT files_group_id_fkey FOREIGN KEY (group_id) REFERENCES public.groups_table(group_id);
 
 
+--
+-- Name: seqset_to_citation_source foreign_key_citation_source; Type: FK CONSTRAINT; Schema: public; Owner: postgres
+--
+
+ALTER TABLE ONLY public.seqset_to_citation_source
+    ADD CONSTRAINT foreign_key_citation_source FOREIGN KEY (citation_source_id) REFERENCES public.seqset_citation_source(citation_source_id) ON DELETE CASCADE;
+
+
+--
+-- Name: seqset_to_citation_source foreign_key_seqset; Type: FK CONSTRAINT; Schema: public; Owner: postgres
+--
+
+ALTER TABLE ONLY public.seqset_to_citation_source
+    ADD CONSTRAINT foreign_key_seqset FOREIGN KEY (seqset_id, seqset_version) REFERENCES public.seqsets(seqset_id, seqset_version) ON DELETE CASCADE;
+
+
 --
 -- Name: seqset_to_records foreign_key_seqset_id; Type: FK CONSTRAINT; Schema: public; Owner: postgres
 --

backend/src/main/kotlin/org/loculus/backend/api/SeqSetCitations.kt

@@ -3,7 +3,6 @@ package org.loculus.backend.api
 import io.swagger.v3.oas.annotations.media.Schema
 import org.loculus.backend.utils.Accession
 import java.sql.Timestamp
-import java.util.*
 
 data class SubmittedSeqSetRecord(
     @Schema(
@@ -57,6 +56,24 @@ data class SeqSet(
     val seqSetDOI: String?,
 )
 
+data class CitationContributor(val givenName: String, val surname: String)
+
+enum class CitationOrigin {
+    CROSSREF,
+    CURATED,
+}
+
+data class CitationSource(
+    val sourceDOI: String,
+    val title: String,
+    val year: Int,
+    val contributors: List<CitationContributor>,
+)
+
+data class SeqSetCitationSource(val source: CitationSource, val seqSetDOIs: Set<String> = emptySet())
+
+data class SeqSetCitation(val source: CitationSource)
+
 data class ResponseSeqSet(val seqSetId: String, val seqSetVersion: Long)
 
 data class CitedBy(

backend/src/main/kotlin/org/loculus/backend/controller/SeqSetCitationsController.kt

@@ -6,6 +6,7 @@ import org.loculus.backend.api.AuthorProfile
 import org.loculus.backend.api.CitedBy
 import org.loculus.backend.api.ResponseSeqSet
 import org.loculus.backend.api.SeqSet
+import org.loculus.backend.api.SeqSetCitation
 import org.loculus.backend.api.SeqSetRecord
 import org.loculus.backend.api.SubmittedSeqSet
 import org.loculus.backend.api.SubmittedSeqSetRecord
@@ -103,9 +104,9 @@ class SeqSetCitationsController(
             submissionDatabaseService.getApprovedUserAccessionVersions(authenticatedUser),
         )
 
-    @Operation(description = "Get count of SeqSet cited by publications")
+    @Operation(description = "Get SeqSet citations from publications")
     @GetMapping("/get-seqset-cited-by-publication")
-    fun getSeqSetCitedByPublication(@RequestParam seqSetId: String, @RequestParam version: Long): CitedBy =
+    fun getSeqSetCitedByPublication(@RequestParam seqSetId: String, @RequestParam version: Long): List<SeqSetCitation> =
         seqSetCitationsService.getSeqSetCitedByPublication(seqSetId, version)
 
     @Operation(description = "Get an author")

backend/src/main/kotlin/org/loculus/backend/service/crossref/CrossRefService.kt

@@ -2,11 +2,17 @@ package org.loculus.backend.service.crossref
 
 import mu.KotlinLogging
 import org.jsoup.Jsoup
+import org.jsoup.parser.Parser
+import org.loculus.backend.api.CitationContributor
+import org.loculus.backend.api.CitationSource
+import org.loculus.backend.api.SeqSetCitationSource
+import org.loculus.backend.utils.DateProvider
 import org.redundent.kotlin.xml.PrintOptions
 import org.redundent.kotlin.xml.xml
 import org.springframework.boot.context.properties.ConfigurationProperties
 import org.springframework.stereotype.Service
 import java.io.DataOutputStream
+import java.io.IOException
 import java.io.OutputStreamWriter
 import java.io.PrintWriter
 import java.net.HttpURLConnection
@@ -40,7 +46,7 @@ data class DoiEntry(
 )
 
 @Service
-class CrossRefService(final val properties: CrossRefServiceProperties) {
+class CrossRefService(private val properties: CrossRefServiceProperties, private val dateProvider: DateProvider) {
     val isActive = properties.endpoint != null &&
         properties.username != null &&
         properties.password != null &&
@@ -49,16 +55,106 @@ class CrossRefService(final val properties: CrossRefServiceProperties) {
         properties.email != null &&
         properties.organization != null &&
         properties.hostUrl != null
-    val dateTimeFormatterMM = DateTimeFormatter.ofPattern("MM")
-    val dateTimeFormatterdd = DateTimeFormatter.ofPattern("dd")
-    val dateTimeFormatteryyyy = DateTimeFormatter.ofPattern("yyyy")
+    val doiPrefix: String? = properties.doiPrefix
+    val dateTimeFormatterMM: DateTimeFormatter = DateTimeFormatter.ofPattern("MM")
+    val dateTimeFormatterdd: DateTimeFormatter = DateTimeFormatter.ofPattern("dd")
+    val dateTimeFormatteryyyy: DateTimeFormatter = DateTimeFormatter.ofPattern("yyyy")
 
     private fun checkIsActive() {
         if (!isActive) {
             throw RuntimeException("The CrossRefService is not active as it has not been configured.")
         }
     }
 
+    fun parseCrossRefCitedByXML(citedByXML: String): List<SeqSetCitationSource> {
+        val parser = Parser.xmlParser().setTrackErrors(1)
+        val doc = Jsoup.parse(citedByXML, "", parser)
+
+        if (parser.errors.isNotEmpty()) {
+            throw IllegalStateException("Invalid XML: ${parser.errors}")
+        }
+
+        val crossRefResult = doc.children().firstOrNull()
+        if (crossRefResult?.tagName() != "crossref_result") {
+            throw IllegalStateException("Invalid CrossRef root element: ${crossRefResult?.tagName()}")
+        }
+
+        return crossRefResult.select("forward_link").map { forwardLink ->
+            val seqSetDOI = forwardLink.attr("doi").takeIf { it.isNotBlank() }
+                ?: throw IllegalStateException("CrossRef forward_link missing SeqSet DOI: $forwardLink")
+
+            val citationElement =
+                forwardLink.children().firstOrNull()
+                    ?: throw IllegalStateException(
+                        "CrossRef forward_link has no citation element under SeqSet $seqSetDOI: $forwardLink",
+                    )
+
+            val sourceDOI = citationElement.selectFirst("doi")?.text()?.takeIf { it.isNotBlank() }
+                ?: throw IllegalStateException(
+                    "CrossRef citation source missing DOI for SeqSet $seqSetDOI: $citationElement",
+                )
+            val title = citationElement.selectFirst("title")?.text()?.takeIf { it.isNotBlank() }
+                ?: throw IllegalStateException(
+                    "CrossRef citation source missing title for SeqSet $seqSetDOI: $citationElement",
+                )
+            val year = citationElement.selectFirst("year")?.text()?.toIntOrNull()
+                ?: throw IllegalStateException(
+                    "CrossRef citation source missing or non-numeric year for SeqSet $seqSetDOI: $citationElement",
+                )
+            val contributors = citationElement.select("contributor").mapNotNull { c ->
+                val givenName = c.selectFirst("given_name")?.text().orEmpty()
+                val surname = c.selectFirst("surname")?.text().orEmpty()
+                if (givenName.isEmpty() && surname.isEmpty()) {
+                    null
+                } else {
+                    CitationContributor(givenName, surname)
+                }
+            }
+
+            SeqSetCitationSource(
+                source = CitationSource(
+                    sourceDOI = sourceDOI,
+                    title = title,
+                    year = year,
+                    contributors = contributors,
+                ),
+                seqSetDOIs = setOf(seqSetDOI),
+            )
+        }
+    }
+
+    fun getCrossRefCitedBy(doiPrefix: String): List<SeqSetCitationSource> {
+        checkIsActive()
+
+        // End date is the current date at time of request
+        val endDate = dateProvider.getCurrentDate()
+        val connection = URI(
+            properties.endpoint +
+                "/servlet/getForwardLinks?usr=${properties.username}&pwd=${properties.password}&doi=$doiPrefix&endDate=$endDate&include_postedcontent=true",
+        ).toURL().openConnection() as HttpURLConnection
+        connection.connectTimeout = 10_000
+        connection.readTimeout = 30_000
+        connection.requestMethod = "GET"
+
+        val response = try {
+            val responseCode = connection.responseCode
+            if (responseCode != HttpURLConnection.HTTP_OK) {
+                throw RuntimeException("CrossRef citedBy request returned $responseCode")
+            }
+            connection.inputStream.use { String(it.readAllBytes()) }
+        } catch (e: IOException) {
+            throw RuntimeException("CrossRef citedBy request failed for DOI $doiPrefix", e)
+        } finally {
+            connection.disconnect()
+        }
+
+        return try {
+            parseCrossRefCitedByXML(response)
+        } catch (e: Exception) {
+            throw RuntimeException("Failed to parse CrossRef citedBy response for DOI $doiPrefix", e)
+        }
+    }
+
     fun generateCrossRefXML(entry: DoiEntry): String {
         checkIsActive()