Major refactor of underlying SBT SQL queries

SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_by_lineage.query.xml

@@ -9,6 +9,14 @@
                         <isKeyField>true</isKeyField>
                         <isHidden>true</isHidden>
                     </column>
+                    <column columnName="analysis_id">
+                        <columnTitle>Analysis Id</columnTitle>
+                        <fk>
+                            <fkDbSchema>sequenceanalysis</fkDbSchema>
+                            <fkTable>sequence_analyses</fkTable>
+                            <fkColumnName>rowid</fkColumnName>
+                        </fk>
+                    </column>
                     <column columnName="lineages">
                         <displayWidth>150</displayWidth>
                     </column>

SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_by_lineage.sql

@@ -1,51 +1,45 @@
-PARAMETERS(AnalysisId INTEGER)
-
-select
-  (CAST(AnalysisId as varchar) || '<>' || a.lineages) as key,
-  a.analysis_id,
-  a.lineages,
-  max(a.totalLineages) as totalLineages,
-  a.loci,
-
-  sum(a.total) as total,
-  max(a.total_reads) as total_reads,
-  round(100 * (cast(sum(a.total) as float) / cast(max(a.total_reads) as float)), 2) as percent,
-  group_concat(distinct a.haplotypesWithAllele) as haplotypesWithAllele,
-
-  CAST((select sum(s.total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = AnalysisId AND s.rowid IN (
-      SELECT distinct asj.alignment_id from sequenceanalysis.alignment_summary_junction asj WHERE asj.analysis_id = AnalysisId AND asj.ref_nt_id.locus = a.loci and asj.status = true
-    )
-  ) as integer) as total_reads_from_locus,
-
-  round(100 * (cast(sum(a.total) as float) / cast((select sum(s.total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = AnalysisId AND s.rowid IN (
-      SELECT distinct asj.alignment_id from sequenceanalysis.alignment_summary_junction asj WHERE asj.analysis_id = AnalysisId AND asj.ref_nt_id.locus = a.loci and asj.status = true
-    )
-  ) as float)), 2) as percent_from_locus,
-  group_concat(distinct a.rowid, ',') as rowids
+SELECT
+    t.*,
+    CASE WHEN t.total_reads_from_locus = 0 THEN 0 ELSE round(100 * (cast(t.total_reads as float) / cast(t.total_reads_from_locus as float)), 2) END as percent_from_locus,
 
 FROM (
-
-  select
-    a.analysis_id,
-    a.rowid,
-
-    group_concat(distinct coalesce(j.ref_nt_id.lineage, j.ref_nt_id.name), chr(10)) as lineages,
-    count(distinct j.ref_nt_id.lineage) as totalLineages,
-    group_concat(distinct coalesce(j.ref_nt_id.locus, j.ref_nt_id.name), chr(10)) as loci,
-
-    total,
-    cast((select sum(total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = AnalysisId) as integer) as total_reads,
-    group_concat(distinct hs.haplotype, chr(10)) as haplotypesWithAllele
-
-  from sequenceanalysis.alignment_summary a
-  join sequenceanalysis.alignment_summary_junction j ON (j.analysis_id = AnalysisId AND j.alignment_id = a.rowid and j.status = true)
-  left join sequenceanalysis.haplotype_sequences hs ON ((
-    (hs.name = j.ref_nt_id.lineage AND hs.type = 'Lineage') OR
-    (hs.name = j.ref_nt_id.name AND hs.type = 'Allele')
-  ) AND hs.haplotype.datedisabled IS NULL)
-  WHERE a.analysis_id = AnalysisId
-  group by a.analysis_id, a.rowid, a.total
-
-) a
-
-GROUP BY a.analysis_id, a.lineages, a.loci
+    SELECT
+      (CAST(a.analysis_id as varchar) || '<>' || a.lineages) as key,
+      a.analysis_id,
+      max(a.lineages) as lineages,
+      coalesce(max(a.totalLineages), 0) as totalLineages,
+      a.loci,
+
+      sum(a.total) as total_reads,
+      max(a.total_reads_in_analysis) as total_reads_in_analysis,
+      CASE WHEN max(a.total_reads_in_analysis) = 0 THEN 0 ELSE round(100 * (cast(sum(a.total) as float) / cast(max(a.total_reads_in_analysis) as float)), 2) END as percent,
+
+      group_concat(a.rowid, ',') as rowids,
+      group_concat(distinct a.haplotypesWithAllele) as haplotypesWithAllele,
+
+      max(a.total_reads_from_locus) as total_reads_from_locus,
+      max(a.lastModified) as lastModified,
+      count(distinct a.rowid) as nAlignments
+
+    FROM (
+
+      select
+        ac.analysis_id,
+        ac.rowid,
+
+        group_concat(distinct coalesce(ac.lineage, ac.ntName), chr(10)) as lineages,
+        count(distinct ac.lineage) as totalLineages,
+        group_concat(distinct coalesce(ac.locus, ac.ntName), chr(10)) as loci,
+
+        group_concat(distinct haplotypesWithAllele, chr(10)) as haplotypesWithAllele,
+
+        max(ac.total) as total,
+        max(ac.total_reads_in_analysis) as total_reads_in_analysis,
+        max(ac.total_reads_from_locus) as total_reads_from_locus,
+        max(ac.modified) as lastModified
+      from sequenceanalysis.alignment_summary_combined ac
+      group by ac.analysis_id, ac.rowid, ac.total
+    ) a
+
+    GROUP BY a.analysis_id, a.lineages, a.loci
+) t

SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_by_lineage/.qview.xml

@@ -19,8 +19,8 @@
            </properties>
        </column>
        <column name="totalLineages" />
-       <column name="total" />
        <column name="total_reads" />
+       <column name="total_reads_in_analysis" />
        <column name="percent"/>
        <column name="total_reads_from_locus" />
        <column name="percent_from_locus"/>

SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_by_lineage/With Haplotype Matches.qview.xml

@@ -19,8 +19,8 @@
            </properties>
        </column>
        <column name="totalLineages" />
-       <column name="total" />
        <column name="total_reads" />
+       <column name="total_reads_in_analysis" />
        <column name="percent"/>
        <column name="total_reads_from_locus" />
        <column name="percent_from_locus"/>

SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_by_lineage_orig.sql

@@ -0,0 +1,51 @@
+PARAMETERS(AnalysisId INTEGER)
+
+select
+  (CAST(AnalysisId as varchar) || '<>' || a.lineages) as key,
+  a.analysis_id,
+  a.lineages,
+  max(a.totalLineages) as totalLineages,
+  a.loci,
+
+  sum(a.total) as total,
+  max(a.total_reads) as total_reads,
+  round(100 * (cast(sum(a.total) as float) / cast(max(a.total_reads) as float)), 2) as percent,
+  group_concat(distinct a.haplotypesWithAllele) as haplotypesWithAllele,
+
+  CAST((select sum(s.total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = AnalysisId AND s.rowid IN (
+      SELECT distinct asj.alignment_id from sequenceanalysis.alignment_summary_junction asj WHERE asj.analysis_id = AnalysisId AND asj.ref_nt_id.locus = a.loci and asj.status = true
+    )
+  ) as integer) as total_reads_from_locus,
+
+  round(100 * (cast(sum(a.total) as float) / cast((select sum(s.total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = AnalysisId AND s.rowid IN (
+      SELECT distinct asj.alignment_id from sequenceanalysis.alignment_summary_junction asj WHERE asj.analysis_id = AnalysisId AND asj.ref_nt_id.locus = a.loci and asj.status = true
+    )
+  ) as float)), 2) as percent_from_locus,
+  group_concat(distinct a.rowid, ',') as rowids
+
+FROM (
+
+  select
+    a.analysis_id,
+    a.rowid,
+
+    group_concat(distinct coalesce(j.ref_nt_id.lineage, j.ref_nt_id.name), chr(10)) as lineages,
+    count(distinct j.ref_nt_id.lineage) as totalLineages,
+    group_concat(distinct coalesce(j.ref_nt_id.locus, j.ref_nt_id.name), chr(10)) as loci,
+
+    total,
+    cast((select sum(total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = AnalysisId) as integer) as total_reads,
+    group_concat(distinct hs.haplotype, chr(10)) as haplotypesWithAllele
+
+  from sequenceanalysis.alignment_summary a
+  join sequenceanalysis.alignment_summary_junction j ON (j.analysis_id = AnalysisId AND j.alignment_id = a.rowid and j.status = true)
+  left join sequenceanalysis.haplotype_sequences hs ON ((
+    (hs.name = j.ref_nt_id.lineage AND hs.type = 'Lineage') OR
+    (hs.name = j.ref_nt_id.name AND hs.type = 'Allele')
+  ) AND hs.haplotype.datedisabled IS NULL)
+  WHERE a.analysis_id = AnalysisId
+  group by a.analysis_id, a.rowid, a.total
+
+) a
+
+GROUP BY a.analysis_id, a.lineages, a.loci

SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_combined.query.xml

@@ -0,0 +1,23 @@
+<query xmlns="http://labkey.org/data/xml/query">
+    <metadata>
+        <tables xmlns="http://labkey.org/data/xml">
+            <table tableName="alignment_summary_combined" tableDbType="TABLE">
+                <tableTitle>Alignment Summary, Combined</tableTitle>
+                <columns>
+                    <column columnName="total_forward">
+                        <columnTitle>Total First-Mate Reads</columnTitle>
+                        <description>The column shows the number of alignments that contain a first-mate or singleton read</description>
+                    </column>
+                    <column columnName="total_reverse">
+                        <columnTitle>Total Second-Mate Reads</columnTitle>
+                        <description>The column shows the number of alignments that contain a second-mate read</description>
+                    </column>
+                    <column columnName="valid_pairs">
+                        <columnTitle>Total Valid Pairs</columnTitle>
+                        <description>The column shows the number of alignments that contain valid paired reads (both forward and reverse)</description>
+                    </column>
+                </columns>
+            </table>
+        </tables>
+    </metadata>
+</query>

SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_grouped.query.xml

@@ -6,6 +6,14 @@
                 <tableTitle>Alignment Summary</tableTitle>
                 <pkColumnName>rowids</pkColumnName>
                 <columns>
+                    <column columnName="analysis_id">
+                        <columnTitle>Analysis Id</columnTitle>
+                        <fk>
+                            <fkDbSchema>sequenceanalysis</fkDbSchema>
+                            <fkTable>sequence_analyses</fkTable>
+                            <fkColumnName>rowid</fkColumnName>
+                        </fk>
+                    </column>
                     <column columnName="rowids">
                         <columnTitle>RowIds</columnTitle>
                         <isHidden>true</isHidden>

SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_grouped.sql

@@ -1,70 +1,55 @@
-PARAMETERS(AnalysisId INTEGER)
-
-select
-  a.analysis_id,
-  a.alleles,
-  a.alleleIds,
-  max(a.lineages) as lineages,
-  coalesce(max(a.totalLineages), 0) as totalLineages,
-  a.loci,
-
-  sum(a.total) as total_reads,
-  sum(a.total_forward) as total_forward,
-  sum(a.total_reverse) as total_reverse,
-  sum(a.valid_pairs) as valid_pairs,
-  max(cast(a.total_reads as integer)) as total_reads_in_analysis,
-  --max(a.loci_total_reads) as total_reads_in_analysis_from_locus,
-  CASE WHEN max(a.total_reads) = 0 THEN 0 ELSE round(100 * (cast(sum(a.total) as float) / cast(max(a.total_reads) as float)), 2) END as percent,
---   case
---     when (cast(sum(a.total) as float) / cast(max(a.total_reads) as float)) >= .04 THEN 'Major'
---     else 'Minor'
---   end as category,
-
-  group_concat(a.rowid, ',') as rowids,
-  group_concat(distinct a.haplotypesWithAllele) as haplotypesWithAllele,
-
-  CAST((select sum(s.total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = AnalysisId AND s.rowid IN (
-      SELECT distinct asj.alignment_id from sequenceanalysis.alignment_summary_junction asj WHERE asj.analysis_id = AnalysisId AND asj.ref_nt_id.locus = a.loci and asj.status = true
-    )
-  ) as INTEGER) as total_reads_from_locus,
-
-  round(100 * (cast(sum(a.total) as float) / CASE WHEN count(a.lineages) = 0 THEN max(a.total_reads) ELSE cast((select sum(s.total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = AnalysisId AND s.rowid IN (
-      SELECT distinct asj.alignment_id from sequenceanalysis.alignment_summary_junction asj WHERE asj.analysis_id = AnalysisId AND asj.ref_nt_id.locus = a.loci and asj.status = true
-    )
-  ) as float) END), 2) as percent_from_locus,
-  max(lastModified) as lastModified,
-  count(distinct a.rowid) as nAlignments,
-  max(a.nloci) as nLoci
+SELECT
+    t.*,
+    CASE WHEN t.total_reads_from_locus = 0 THEN 0 ELSE round(100 * (cast(t.total_reads as float) / cast(t.total_reads_from_locus as float)), 2) END as percent_from_locus,
 
 FROM (
-
-  select
-    a.analysis_id,
-    a.rowid,
-
-    group_concat(distinct j.ref_nt_id) as alleleIds,
-    group_concat(distinct j.ref_nt_id.name, chr(10)) as alleles,
-    group_concat(distinct j.ref_nt_id.lineage, chr(10)) as lineages,
-    count(distinct j.ref_nt_id.lineage) as totalLineages,
-    group_concat(distinct j.ref_nt_id.locus, chr(10)) as loci,
-    count(distinct j.ref_nt_id.locus) as nloci,
-    group_concat(distinct hs.haplotype, chr(10)) as haplotypesWithAllele,
-
-    total,
-    total_forward,
-    total_reverse,
-    valid_pairs,
-    (select sum(total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = AnalysisId) as total_reads,
-    max(j.modified) as lastModified
-  from sequenceanalysis.alignment_summary a
-  left join sequenceanalysis.alignment_summary_junction j ON (j.analysis_id = AnalysisId AND j.alignment_id = a.rowid and j.status = true)
-  left join sequenceanalysis.haplotype_sequences hs ON ((
-    (hs.name = j.ref_nt_id.lineage AND hs.type = 'Lineage') OR
-    (hs.name = j.ref_nt_id.name AND hs.type = 'Allele')
-  ) AND hs.haplotype.datedisabled IS NULL)
-  WHERE a.analysis_id = AnalysisId
-  group by a.analysis_id, a.rowid, a.total, total_forward, total_reverse, valid_pairs
-
-) a
-
-GROUP BY a.analysis_id, a.alleles, a.alleleIds, a.loci
+    SELECT
+      a.analysis_id,
+      a.alleles,
+      a.alleleIds,
+      max(a.lineages) as lineages,
+      coalesce(max(a.totalLineages), 0) as totalLineages,
+      a.loci,
+
+      sum(a.total) as total_reads,
+      sum(a.total_forward) as total_forward,
+      sum(a.total_reverse) as total_reverse,
+      sum(a.valid_pairs) as valid_pairs,
+      max(a.total_reads_in_analysis) as total_reads_in_analysis,
+      CASE WHEN max(a.total_reads_in_analysis) = 0 THEN 0 ELSE round(100 * (cast(sum(a.total) as float) / cast(max(a.total_reads_in_analysis) as float)), 2) END as percent,
+
+      group_concat(a.rowid, ',') as rowids,
+      group_concat(distinct a.haplotypesWithAllele) as haplotypesWithAllele,
+
+      max(a.total_reads_from_locus) as total_reads_from_locus,
+      max(a.lastModified) as lastModified,
+      count(distinct a.rowid) as nAlignments,
+      max(a.nloci) as nLoci
+
+    FROM (
+
+      select
+        ac.analysis_id,
+        ac.rowid,
+
+        group_concat(distinct ac.ref_nt_id) as alleleIds,
+        group_concat(distinct ac.ntName, chr(10)) as alleles,
+        group_concat(distinct ac.lineage, chr(10)) as lineages,
+        count(distinct ac.lineage) as totalLineages,
+        group_concat(distinct ac.locus, chr(10)) as loci,
+        count(distinct ac.locus) as nloci,
+        group_concat(distinct haplotypesWithAllele, chr(10)) as haplotypesWithAllele,
+
+        max(ac.total) as total,
+        max(ac.total_forward) as total_forward,
+        max(ac.total_reverse) as total_reverse,
+        max(ac.valid_pairs) as valid_pairs,
+        max(ac.total_reads_in_analysis) as total_reads_in_analysis,
+        max(ac.total_reads_from_locus) as total_reads_from_locus,
+        max(ac.modified) as lastModified
+      from sequenceanalysis.alignment_summary_combined ac
+      group by ac.analysis_id, ac.rowid
+    ) a
+
+    GROUP BY a.analysis_id, a.alleles, a.alleleIds, a.loci
+) t