Skip to content

Simplify query for runs to export for samples and data during folder export #7448

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
XingY merged 3 commits into from
Feb 27, 2026
Merged

Simplify query for runs to export for samples and data during folder export #7448

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c162613
Simplify query for runs to export for samples and data
XingY Feb 23, 2026
a857606
Merge branch 'develop' into fb_folderExportPerf
XingY Feb 25, 2026
0a463be
Merge remote-tracking branch 'origin/develop' into fb_folderExportPerf
labkey-adam Feb 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions api/src/org/labkey/api/exp/api/ExperimentService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -804,6 +804,12 @@ static void validateParentAlias(Map<String, String> aliasMap, Set<String> reserv

List<? extends ExpRun> getRunsUsingDataClasses(Collection<ExpDataClass> dataClasses);

/** Get derivation run IDs for a data class — runs with SAMPLE_DERIVATION_PROTOCOL that have no material inputs/outputs */
List<Long> getDerivationRunIdsForDataClassExport(long dataClassRowId);

/** Get derivation/aliquot run IDs for sample types — filtered by protocol and optionally excluding runs with data inputs/outputs */
List<Long> getDerivationRunIdsForSampleTypesExport(Collection<String> sampleTypeLsids, Container c, boolean includeRunsWithDataIO);

/**
* @return the subset of these runs which are supposed to be deleted when one of their inputs is deleted.
*/
Expand Down
102 changes: 102 additions & 0 deletions experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5799,6 +5799,108 @@ public List<Pair<String, String>> getRunsAndRolesUsingMaterial(ExpMaterial mater
return getRunsAndRolesUsingInput(material, getTinfoMaterialInput(), "MaterialId", () -> getRunsUsingMaterials(material.getRowId()));
}

@Override
public List<Long> getDerivationRunIdsForDataClassExport(long dataClassRowId)
{
SQLFragment sql = new SQLFragment("""
SELECT DISTINCT er.RowId
FROM exp.ExperimentRun er
WHERE er.ProtocolLSID = ?
AND er.RowId IN (
SELECT pa.RunId FROM exp.ProtocolApplication pa
INNER JOIN exp.DataInput di ON di.TargetApplicationId = pa.RowId
INNER JOIN exp.Data d ON di.DataId = d.RowId
WHERE d.classId = ?
UNION
SELECT pa.RunId FROM exp.ProtocolApplication pa
INNER JOIN exp.Data d ON d.SourceApplicationId = pa.RowId
WHERE d.classId = ?
)
AND NOT EXISTS (
SELECT 1 FROM exp.ProtocolApplication pa2
INNER JOIN exp.MaterialInput mi ON mi.TargetApplicationId = pa2.RowId
WHERE pa2.RunId = er.RowId
)
AND NOT EXISTS (
SELECT 1 FROM exp.Material m
INNER JOIN exp.ProtocolApplication pa3 ON m.SourceApplicationId = pa3.RowId
WHERE pa3.RunId = er.RowId
)
""",
SAMPLE_DERIVATION_PROTOCOL_LSID,
dataClassRowId,
dataClassRowId
);

return new SqlSelector(getExpSchema(), sql).getArrayList(Long.class);
}

@Override
public List<Long> getDerivationRunIdsForSampleTypesExport(Collection<String> sampleTypeLsids, Container c, boolean includeRunsWithDataIO)
{
if (sampleTypeLsids.isEmpty())
return Collections.emptyList();

SQLFragment inClause = getExpSchema().getSqlDialect().appendInClauseSql(new SQLFragment(), sampleTypeLsids);

// Subquery to find runs that use materials from the given sample types and container
SQLFragment materialsSubquery = new SQLFragment("""
SELECT pa.RunId FROM exp.ProtocolApplication pa
INNER JOIN exp.MaterialInput mi ON mi.TargetApplicationId = pa.RowId
INNER JOIN exp.Material m ON mi.MaterialId = m.RowId
WHERE m.cpasType """);
materialsSubquery.append(inClause);
materialsSubquery.append(" AND m.Container = ?\n");
materialsSubquery.add(c);
materialsSubquery.append("""
UNION
SELECT pa.RunId FROM exp.ProtocolApplication pa
INNER JOIN exp.Material m ON m.SourceApplicationId = pa.RowId
WHERE m.cpasType """);
materialsSubquery.append(inClause);
materialsSubquery.append(" AND m.Container = ?");
materialsSubquery.add(c);

SQLFragment sql = new SQLFragment();
sql.append("SELECT DISTINCT er.RowId\nFROM exp.ExperimentRun er\nWHERE er.RowId IN (\n");
sql.append(materialsSubquery);
sql.append(")\n");

if (includeRunsWithDataIO)
{
// Include all derivation and aliquot runs
sql.append("AND er.ProtocolLSID IN (?, ?)\n");
sql.add(SAMPLE_DERIVATION_PROTOCOL_LSID);
sql.add(SAMPLE_ALIQUOT_PROTOCOL_LSID);
}
else
{
// Aliquot runs are always included; derivation runs only if they have no data inputs/outputs
sql.append("AND (\n");
sql.append(" er.ProtocolLSID = ?\n");
sql.add(SAMPLE_ALIQUOT_PROTOCOL_LSID);
sql.append(" OR (\n");
sql.append(" er.ProtocolLSID = ?\n");
sql.add(SAMPLE_DERIVATION_PROTOCOL_LSID);
sql.append("""
AND NOT EXISTS (
SELECT 1 FROM exp.ProtocolApplication pa2
INNER JOIN exp.DataInput di ON di.TargetApplicationId = pa2.RowId
WHERE pa2.RunId = er.RowId
)
AND NOT EXISTS (
SELECT 1 FROM exp.Data d
INNER JOIN exp.ProtocolApplication pa3 ON d.SourceApplicationId = pa3.RowId
WHERE pa3.RunId = er.RowId
)
)
)
""");
}

return new SqlSelector(getExpSchema(), sql).getArrayList(Long.class);
}

@Override
public List<ExpRunImpl> runsDeletedWithInput(List<? extends ExpRun> runs)
{
Expand Down
33 changes: 6 additions & 27 deletions experiment/src/org/labkey/experiment/samples/DataClassFolderWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,7 @@
import org.labkey.api.data.SimpleFilter;
import org.labkey.api.data.Sort;
import org.labkey.api.data.TableInfo;
import org.labkey.api.exp.api.ExpData;
import org.labkey.api.exp.api.ExpDataClass;
import org.labkey.api.exp.api.ExpObject;
import org.labkey.api.exp.api.ExpRun;
import org.labkey.api.exp.api.ExperimentService;
import org.labkey.api.exp.query.ExpSchema;
import org.labkey.api.query.FieldKey;
Expand All @@ -29,9 +26,6 @@
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

import static java.util.stream.Collectors.toList;

public abstract class DataClassFolderWriter extends AbstractExpFolderWriter
{
Expand Down Expand Up @@ -78,19 +72,14 @@ public void write(Container c, FolderExportContext ctx, VirtualFile vf) throws E
typesSelection.addDataClass(dataClass);
exportTypes = true;

// get the list of runs with the data we expect to export, these will be the sample derivation
// protocol runs to track the lineage
// get the list of derivation runs for this data class — only sample derivation protocol runs
// with no material inputs/outputs (those are handled by the sample type writer)
// Sample derivation protocols involving data classes can be either to/from another data
// class or also to/from a sample type. If it's the latter, we will let the sample writer handle it
// since on import, data classes run before sample types.
if (exportDataClassData)
{
List<Long> dataIdsToExport = dataClass.getDatas().stream().map(ExpData::getRowId).collect(toList());

// only want the sample derivation runs; other runs will get included in the experiment xar.
List<Long> exportedRunIds = ExperimentService.get().getRunsUsingDataIds(dataIdsToExport).stream().filter(run -> {
String lsid = run.getProtocol().getLSID();
return lsid.equals(ExperimentService.SAMPLE_DERIVATION_PROTOCOL_LSID) && isValidRunType(ctx, run);
})
.collect(Collectors.toSet())
.stream().map(ExpObject::getRowId).toList();
List<Long> exportedRunIds = ExperimentService.get().getDerivationRunIdsForDataClassExport(dataClass.getRowId());
Copy link
Contributor

@labkey-nicka labkey-nicka Feb 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: I realize this is a one-for-one replacement of the current logic but we could consider doing a coalesced query here that takes a collection of data class rowIds and produces the result with a single query. Much like the getDerivationRunIdsForSampleTypesExport implementation. Something like this:

List<Long> dataClassRowIdsForExport = new ArrayList<>();
for (ExpDataClass dataClass : ExperimentService.get().getDataClasses(c, false))
{
    // ignore data classes that are filtered out
    if (EXCLUDED_TYPES.contains(dataClass.getName()))
        continue;

    dataClasses.add(dataClass);
    typesSelection.addDataClass(dataClass);
    exportTypes = true;

    if (exportDataClassData)
        dataClassRowIdsForExport.add(dataClass.getRowId());
}

if (!dataClassRowIdsForExport.isEmpty())
{
    // get the list of derivation runs for these data classes — only sample derivation protocol runs
    // with no material inputs/outputs (those are handled by the sample type writer)
    // Sample derivation protocols involving data classes can be either to/from another data
    //   class or also to/from a sample type. If it's the latter, we will let the sample writer handle it
    //   since on import, data classes run before sample types.
    List<Long> exportedRunIds = ExperimentService.get().getDerivationRunIdsForDataClassExport(dataClassRowIdsForExport);

    if (!exportedRunIds.isEmpty())
    {
        runsSelection.addRunIds(exportedRunIds);
        exportRuns = true;
    }
}

The query could then be:

// Note: I did not test this
public List<Long> getDerivationRunIdsForDataClassExport(@NotNull Collection<Long> dataClassRowIds)
{
    if (dataClassRowIds.isEmpty())
        return List.of();

    SQLFragment inClause = new SQLFragment().appendInClause(dataClassRowIds, getExpSchema().getSqlDialect());

    SQLFragment sql = new SQLFragment("SELECT DISTINCT er.RowId FROM ")
            .append(getTinfoExperimentRun(), "er")
            .append(" WHERE er.ProtocolLSID = ? ").add(SAMPLE_DERIVATION_PROTOCOL_LSID);
    sql.append("""
        AND EXISTS (
            SELECT 1 FROM exp.ProtocolApplication pa
            LEFT JOIN exp.DataInput di ON di.TargetApplicationId = pa.RowId
            LEFT JOIN exp.Data d1 ON di.DataId = d1.RowId AND d1.classId\s
    """);
    sql.append(inClause);
    sql.append(" LEFT JOIN exp.Data d2 ON d2.SourceApplicationId = pa.RowId AND d2.classId ");
    sql.append(inClause);
    sql.append("""
            WHERE pa.RunId = er.RowId
            AND (d1.RowId IS NOT NULL OR d2.RowId IS NOT NULL)
        )
        AND NOT EXISTS (
            SELECT 1 FROM exp.ProtocolApplication pa_m
            LEFT JOIN exp.MaterialInput mi ON mi.TargetApplicationId = pa_m.RowId
            LEFT JOIN exp.Material m ON m.SourceApplicationId = pa_m.RowId
            WHERE pa_m.RunId = er.RowId
            AND (mi.RowId IS NOT NULL OR m.RowId IS NOT NULL)
        )
        ORDER BY er.RowId
    """);

    return new SqlSelector(getExpSchema(), sql).getArrayList(Long.class);
}

Copy link
Contributor Author

@XingY XingY Feb 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point. But given we didn't adopt code review then manual testing workflow on this PR, I'll leave the code as currently is...


if (!exportedRunIds.isEmpty())
{
Expand Down Expand Up @@ -129,16 +118,6 @@ public void write(Container c, FolderExportContext ctx, VirtualFile vf) throws E
exportContext.setDataClassXarCreated(true);
}

/**
* Sample derivation protocols involving data classes can be either to/from another data
* class or also to/from a sample type. If it's the latter, we will let the sample writer handle it
* since on import, data classes run before sample types.
*/
private boolean isValidRunType(FolderExportContext ctx, ExpRun run)
{
return run.getMaterialOutputs().isEmpty() && run.getMaterialInputs().isEmpty();
}

private void writeDataClassDataFiles(Set<ExpDataClass> dataClasses, FolderExportContext ctx, VirtualFile dir) throws Exception
{
// write out the DataClass rows
Expand Down
47 changes: 14 additions & 33 deletions experiment/src/org/labkey/experiment/samples/SampleTypeFolderWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
import org.labkey.api.data.WrappedColumnInfo;
import org.labkey.api.exp.Lsid;
import org.labkey.api.exp.api.ExpMaterial;
import org.labkey.api.exp.api.ExpRun;
import org.labkey.api.exp.api.ExpSampleType;
import org.labkey.api.exp.api.ExperimentService;
import org.labkey.api.exp.api.SampleTypeService;
Expand All @@ -40,7 +39,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

public abstract class SampleTypeFolderWriter extends AbstractExpFolderWriter
{
Expand All @@ -62,7 +60,6 @@ public void write(Container c, FolderExportContext ctx, VirtualFile vf) throws E
XarExportSelection typesSelection = new XarExportSelection();
XarExportSelection runsSelection = new XarExportSelection();
Set<ExpSampleType> sampleTypes = new HashSet<>();
List<ExpMaterial> materialsToExport = new ArrayList<>();
_exportPhiLevel = ctx.getPhiLevel();
boolean exportTypes = false;
boolean exportRuns = false;
Expand All @@ -80,6 +77,7 @@ public void write(Container c, FolderExportContext ctx, VirtualFile vf) throws E
return;

Lsid sampleTypeLsid = new Lsid(ExperimentService.get().generateLSID(c, ExpSampleType.class, "export"));
Set<String> sampleTypeLsids = new HashSet<>();
for (ExpSampleType sampleType : SampleTypeService.get().getSampleTypes(c, true))
{
// ignore the magic sample type that is used for the specimen repository, it is managed by the specimen importer
Expand All @@ -98,30 +96,27 @@ public void write(Container c, FolderExportContext ctx, VirtualFile vf) throws E
{
sampleTypes.add(sampleType);
typesSelection.addSampleType(sampleType);
materialsToExport.addAll(sampleType.getSamples(c));
sampleTypeLsids.add(sampleType.getLSID());
exportTypes = true;
}
}

// get the list of runs with the materials or data we expect to export, these will be the sample derivation
// protocol runs to track the lineage
Set<ExpRun> exportedRuns = new HashSet<>();
if (!materialsToExport.isEmpty() && exportSampleTypeData)
exportedRuns.addAll(ExperimentService.get().getRunsUsingMaterials(materialsToExport));

// only want the sample derivation runs; other runs will get included in the experiment xar.
exportedRuns = exportedRuns.stream().filter(run -> {
String lsid = run.getProtocol().getLSID();
if (lsid.equals(ExperimentService.SAMPLE_DERIVATION_PROTOCOL_LSID))
return isValidRunType(ctx, run);
else
return lsid.equals(ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID);
}).collect(Collectors.toSet());

if (!exportedRuns.isEmpty())
// Sample derivation protocols involving samples can be either to/from another sample
// or to/from a data class. If it's the latter, don't include the run if data class data
// is not included in the archive.
if (exportSampleTypeData && !sampleTypes.isEmpty())
{
runsSelection.addRuns(exportedRuns);
exportRuns = true;
boolean includeRunsWithDataIO = ctx.getDataTypes().contains(FolderArchiveDataTypes.DATA_CLASS_DATA);
List<Long> exportedRunIds = ExperimentService.get().getDerivationRunIdsForSampleTypesExport(sampleTypeLsids, c, includeRunsWithDataIO);

if (!exportedRunIds.isEmpty())
{
runsSelection.addRunIds(exportedRunIds);
exportRuns = true;
}
}

// UNDONE: The other exporters use FOLDER_RELATIVE, but it wants to use ${AutoFileLSID} replacements for DataClass LSIDs when exporting the TSV data.. see comment in ExportLsidDataColumn
Expand Down Expand Up @@ -153,20 +148,6 @@ public void write(Container c, FolderExportContext ctx, VirtualFile vf) throws E
exportContext.setSampleXarCreated(true);
}

/**
* Sample derivation protocols involving samples can be either to/from another sample
* or to/from a data class. If it's the latter, don't include the run if data class data
* is not included in the archive.
*/
private boolean isValidRunType(FolderExportContext ctx, ExpRun run)
{
if (!run.getDataInputs().isEmpty() || !run.getDataOutputs().isEmpty())
{
return ctx.getDataTypes().contains(FolderArchiveDataTypes.DATA_CLASS_DATA);
}
return true;
}

private void writeSampleTypeDataFiles(Set<ExpSampleType> sampleTypes, FolderExportContext ctx, VirtualFile dir) throws Exception
{
// write out the sample rows
Expand Down