Make sure study ids exist before using them in filter SQL (#11272)

* make sure study ids exist before using them in filter SQL

* add involved cancer studies into StudyViewFilterHelper

src/main/java/org/cbioportal/model/StudyViewFilterContext.java

@@ -5,7 +5,10 @@
 
 import java.util.List;
 
-public record StudyViewFilterContext( StudyViewFilter studyViewFilter,
-                                     List<CustomSampleIdentifier> customDataFilterSamples) {
+public record StudyViewFilterContext(
+    StudyViewFilter studyViewFilter,
+    List<CustomSampleIdentifier> customDataFilterSamples,
+    List<String> involvedCancerStudies
+) {
 
 }

src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java

@@ -23,7 +23,8 @@
 public final class StudyViewFilterHelper {
     public static StudyViewFilterHelper build(@Nullable StudyViewFilter studyViewFilter,
                                               @Nullable Map<DataSource, List<MolecularProfile>> genericAssayProfilesMap,
-                                              @Nullable List<CustomSampleIdentifier> customDataSamples) {
+                                              @Nullable List<CustomSampleIdentifier> customDataSamples,
+                                              @Nullable List<String> involvedCancerStudies) {
         if (Objects.isNull(studyViewFilter)) {
             studyViewFilter = new StudyViewFilter();
         }
@@ -33,6 +34,9 @@ public static StudyViewFilterHelper build(@Nullable StudyViewFilter studyViewFil
         if (Objects.isNull(customDataSamples)) {
             customDataSamples = new ArrayList<>();
         }
+        if (Objects.isNull(involvedCancerStudies)) {
+            involvedCancerStudies = new ArrayList<>();
+        }
         if (studyViewFilter.getGenomicDataFilters() != null && !studyViewFilter.getGenomicDataFilters().isEmpty()) {
             List<GenomicDataFilter> mergedGenomicDataFilters = mergeDataFilters(studyViewFilter.getGenomicDataFilters());
             studyViewFilter.setGenomicDataFilters(mergedGenomicDataFilters);
@@ -45,19 +49,22 @@ public static StudyViewFilterHelper build(@Nullable StudyViewFilter studyViewFil
             List<GenericAssayDataFilter> mergedGenericAssayDataFilters = mergeDataFilters(studyViewFilter.getGenericAssayDataFilters());
             studyViewFilter.setGenericAssayDataFilters(mergedGenericAssayDataFilters);
         }
-        return new StudyViewFilterHelper(studyViewFilter, genericAssayProfilesMap, customDataSamples);
+        return new StudyViewFilterHelper(studyViewFilter, genericAssayProfilesMap, customDataSamples, involvedCancerStudies);
     }
 
     private final StudyViewFilter studyViewFilter;
     private final CategorizedGenericAssayDataCountFilter categorizedGenericAssayDataCountFilter;
     private final List<CustomSampleIdentifier> customDataSamples;
+    private final List<String> involvedCancerStudies;
 
     private StudyViewFilterHelper(@NonNull StudyViewFilter studyViewFilter, 
                                   @NonNull Map<DataSource, List<MolecularProfile>> genericAssayProfilesMap,
-                                  @NonNull List<CustomSampleIdentifier> customDataSamples) {
+                                  @NonNull List<CustomSampleIdentifier> customDataSamples,
+                                  @NonNull List<String> involvedCancerStudies) {
         this.studyViewFilter = studyViewFilter;
         this.categorizedGenericAssayDataCountFilter = extractGenericAssayDataCountFilters(studyViewFilter, genericAssayProfilesMap);
         this.customDataSamples = customDataSamples;
+        this.involvedCancerStudies = involvedCancerStudies;
     }
 
     public StudyViewFilter studyViewFilter() {
@@ -71,6 +78,10 @@ public CategorizedGenericAssayDataCountFilter categorizedGenericAssayDataCountFi
     public List<CustomSampleIdentifier> customDataSamples() {
         return this.customDataSamples;
     }
+
+    public List<String> involvedCancerStudies() {
+        return involvedCancerStudies;
+    }
 
     private CategorizedGenericAssayDataCountFilter extractGenericAssayDataCountFilters(final StudyViewFilter studyViewFilter, Map<DataSource, List<MolecularProfile>> genericAssayProfilesMap) {
         if ((studyViewFilter.getGenericAssayDataFilters() == null || genericAssayProfilesMap.isEmpty()))

src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java

@@ -97,7 +97,12 @@ public List<GenomicDataCount> getMolecularProfileSampleCounts(StudyViewFilterCon
     }
 
     public StudyViewFilterHelper createStudyViewFilterHelper(StudyViewFilterContext studyViewFilterContext) {
-        return StudyViewFilterHelper.build(studyViewFilterContext.studyViewFilter() , getGenericAssayProfilesMap(), studyViewFilterContext.customDataFilterSamples());    
+        return StudyViewFilterHelper.build(
+            studyViewFilterContext.studyViewFilter(),
+            getGenericAssayProfilesMap(),
+            studyViewFilterContext.customDataFilterSamples(),
+            studyViewFilterContext.involvedCancerStudies()
+        );    
     }
 
     @Override

src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java

@@ -161,16 +161,7 @@ public List<ClinicalDataCountItem> getClinicalDataCounts(StudyViewFilter studyVi
 
         var context = createContext(studyViewFilter);
 
-        List<String> involvedCancerStudies = new ArrayList<>();
-        if (studyViewFilter.getStudyIds() != null && !studyViewFilter.getStudyIds().isEmpty()) {
-            involvedCancerStudies = studyViewFilter.getStudyIds();
-        } else if (studyViewFilter.getSampleIdentifiers() != null && !studyViewFilter.getSampleIdentifiers().isEmpty()) {
-            Set<String> studyIdSet = new HashSet<>();
-            for (SampleIdentifier sampleIdentifier : studyViewFilter.getSampleIdentifiers()) {
-                studyIdSet.add(sampleIdentifier.getStudyId());
-            }
-            involvedCancerStudies = studyIdSet.stream().toList();
-        }
+        List<String> involvedCancerStudies = context.involvedCancerStudies();
 
         var result = studyViewRepository.getClinicalDataCounts(context, filteredAttributes);
 
@@ -288,7 +279,8 @@ public List<GenomicDataCountItem> getMutationTypeCountsByGeneSpecific(StudyViewF
 
     private StudyViewFilterContext createContext(StudyViewFilter studyViewFilter) {
         List<CustomSampleIdentifier> customSampleIdentifiers = customDataFilterUtil.extractCustomDataSamples(studyViewFilter);
-        return new StudyViewFilterContext(studyViewFilter, customSampleIdentifiers);
+        List<String> involvedCancerStudies = customDataFilterUtil.extractInvolvedCancerStudies(studyViewFilter);
+        return new StudyViewFilterContext(studyViewFilter, customSampleIdentifiers, involvedCancerStudies);
     }
 
     private List<ClinicalDataCountItem> generateDataCountItemsFromDataCounts(List<ClinicalDataCount> dataCounts) {

src/main/java/org/cbioportal/web/columnar/util/CustomDataFilterUtil.java

@@ -16,10 +16,7 @@
 import org.springframework.stereotype.Component;
 
 import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
+import java.util.*;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -37,6 +34,24 @@ public CustomDataFilterUtil(StudyViewFilterUtil studyViewFilterUtil, CustomDataS
         this.patientService = patientService;
     }
 
+    public List<String> extractInvolvedCancerStudies(final StudyViewFilter studyViewFilter) {
+        List<String> involvedCancerStudies;
+
+        if (studyViewFilter.getStudyIds() != null && !studyViewFilter.getStudyIds().isEmpty()) {
+            involvedCancerStudies = studyViewFilter.getStudyIds();
+        } else if (studyViewFilter.getSampleIdentifiers() != null && !studyViewFilter.getSampleIdentifiers().isEmpty()) {
+            Set<String> studyIdSet = new HashSet<>();
+            for (SampleIdentifier sampleIdentifier : studyViewFilter.getSampleIdentifiers()) {
+                studyIdSet.add(sampleIdentifier.getStudyId());
+            }
+            involvedCancerStudies = studyIdSet.stream().toList();
+        }
+        else {
+            involvedCancerStudies = Collections.emptyList();
+        }
+
+        return involvedCancerStudies;
+    }
     public List<CustomSampleIdentifier> extractCustomDataSamples(final StudyViewFilter studyViewFilter) {
         if (studyViewFilter == null) {
             return null;

src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml

@@ -4,12 +4,12 @@
 <mapper  namespace="org.cbioportal.persistence.mybatisclickhouse.StudyViewMapper">
     <sql id="sampleUniqueIdsFromStudyViewFilter">
         <trim prefixOverrides="INTERSECT">
-            <if test="studyViewFilterHelper.studyViewFilter.studyIds != null and !studyViewFilterHelper.studyViewFilter.studyIds.isEmpty()">
+            <if test="studyViewFilterHelper.involvedCancerStudies != null and !studyViewFilterHelper.involvedCancerStudies.isEmpty()">
                 INTERSECT
                 SELECT sample_unique_id
                 FROM sample_derived
                 WHERE cancer_study_identifier IN
-                <foreach item="studyId" collection="studyViewFilterHelper.studyViewFilter.studyIds" open="(" separator="," close=")">
+                <foreach item="studyId" collection="studyViewFilterHelper.involvedCancerStudies" open="(" separator="," close=")">
                     #{studyId}
                 </foreach>
             </if>
@@ -42,18 +42,20 @@
                     JOIN genetic_profile gp ON sample_profile.genetic_profile_id = gp.genetic_profile_id
                     JOIN cancer_study cs ON gp.cancer_study_id = cs.cancer_study_id
                     JOIN sample_derived on sample_profile.sample_id = sample_derived.internal_id
-                    <where>
-                        sample_derived.cancer_study_identifier IN
-                        <foreach item="studyId" collection="studyViewFilterHelper.studyViewFilter.studyIds" open="(" separator="," close=")">
-                            #{studyId}
-                        </foreach>
-                        AND
-                        <foreach item="studyId" collection="studyViewFilterHelper.studyViewFilter.studyIds" open="(" close=")" separator="OR">
-                            <foreach item="genomicProfileId" collection="ANDGroup" open="(" separator=" OR " close=")">
-                                gp.stable_id=concat(#{studyId}, '_', #{genomicProfileId})
+                    <if test="studyViewFilterHelper.involvedCancerStudies != null and !studyViewFilterHelper.involvedCancerStudies.isEmpty()">
+                        <where>
+                            sample_derived.cancer_study_identifier IN
+                            <foreach item="studyId" collection="studyViewFilterHelper.involvedCancerStudies" open="(" separator="," close=")">
+                                #{studyId}
                             </foreach>
-                        </foreach>
-                    </where>
+                            AND
+                            <foreach item="studyId" collection="studyViewFilterHelper.involvedCancerStudies" open="(" close=")" separator="OR">
+                                <foreach item="genomicProfileId" collection="ANDGroup" open="(" separator=" OR " close=")">
+                                    gp.stable_id=concat(#{studyId}, '_', #{genomicProfileId})
+                                </foreach>
+                            </foreach>
+                        </where>
+                    </if>
                 </foreach>
                 )
 
@@ -430,10 +432,12 @@
         SELECT sample_unique_id, patient_unique_id, attribute_value
         FROM clinical_data_derived
         WHERE attribute_name = #{clinicalDataFilter.attributeId} AND type='${type}'
-        AND cancer_study_identifier IN
-        <foreach item="studyId" collection="studyViewFilterHelper.studyViewFilter.studyIds" open="(" separator="," close=")">
-            #{studyId}
-        </foreach>
+        <if test="studyViewFilterHelper.involvedCancerStudies != null and !studyViewFilterHelper.involvedCancerStudies.isEmpty()">
+            AND cancer_study_identifier IN
+            <foreach item="studyId" collection="studyViewFilterHelper.involvedCancerStudies" open="(" separator="," close=")">
+                #{studyId}
+            </foreach>
+        </if>
     </sql>
 
     <sql id="categoricalClinicalDataCountFilter">
@@ -511,10 +515,12 @@
         FROM genetic_alteration_derived
         WHERE profile_type = #{genomicDataFilter.profileType}
             AND hugo_gene_symbol = #{genomicDataFilter.hugoGeneSymbol}
+        <if test="studyViewFilterHelper.involvedCancerStudies != null and !studyViewFilterHelper.involvedCancerStudies.isEmpty()">
             AND cancer_study_identifier IN
-            <foreach item="studyId" collection="studyViewFilterHelper.studyViewFilter.studyIds" open="(" separator="," close=")">
+            <foreach item="studyId" collection="studyViewFilterHelper.involvedCancerStudies" open="(" separator="," close=")">
                 #{studyId}
             </foreach>
+        </if>
     </sql>
 
     <sql id="applyGenericAssayDataFilter">
@@ -641,30 +647,40 @@
                 SELECT sample_unique_id
                 FROM sample_derived
                 WHERE cancer_study_identifier IN
-                    <foreach item="studyId" collection="studyViewFilterHelper.studyViewFilter.studyIds" open="(" separator="," close=")">
+                <if test="studyViewFilterHelper.involvedCancerStudies != null and !studyViewFilterHelper.involvedCancerStudies.isEmpty()">
+                    <foreach item="studyId" collection="studyViewFilterHelper.involvedCancerStudies" open="(" separator="," close=")">
                         #{studyId}
                     </foreach>
+                </if>
             ),
             profiled_samples AS (
                 SELECT DISTINCT sgp.sample_unique_id
                 FROM sample_to_gene_panel_derived sgp
                     JOIN gene_panel_to_gene_derived gpg ON sgp.gene_panel_id = gpg.gene_panel_id
-                WHERE cancer_study_identifier IN
-                    <foreach item="studyId" collection="studyViewFilterHelper.studyViewFilter.studyIds" open="(" separator="," close=")">
+                WHERE
+                <if test="studyViewFilterHelper.involvedCancerStudies != null and !studyViewFilterHelper.involvedCancerStudies.isEmpty()">
+                    cancer_study_identifier IN
+                    <foreach item="studyId" collection="studyViewFilterHelper.involvedCancerStudies" open="(" separator="," close=")">
                         #{studyId}
                     </foreach>
-                    AND gpg.gene = #{mutationDataFilter.hugoGeneSymbol}
-                    AND sgp.alteration_type = 'MUTATION_EXTENDED'
+                  AND
+                </if>
+                    gpg.gene = #{mutationDataFilter.hugoGeneSymbol}
+                  AND sgp.alteration_type = 'MUTATION_EXTENDED'
             ),
             mutated_samples AS (
                 SELECT DISTINCT sample_unique_id
                 FROM genomic_event_derived
-                WHERE cancer_study_identifier IN
-                    <foreach item="studyId" collection="studyViewFilterHelper.studyViewFilter.studyIds" open="(" separator="," close=")">
+                WHERE
+                <if test="studyViewFilterHelper.involvedCancerStudies != null and !studyViewFilterHelper.involvedCancerStudies.isEmpty()">
+                    cancer_study_identifier IN
+                    <foreach item="studyId" collection="studyViewFilterHelper.involvedCancerStudies" open="(" separator="," close=")">
                         #{studyId}
                     </foreach>
-                    AND hugo_gene_symbol = #{mutationDataFilter.hugoGeneSymbol}
-                    AND variant_type = 'mutation'
+                  AND
+                </if>
+                    hugo_gene_symbol = #{mutationDataFilter.hugoGeneSymbol}
+                  AND variant_type = 'mutation'
             )
             SELECT DISTINCT sample_unique_id
             FROM
@@ -706,20 +722,26 @@
             FROM genetic_alteration_derived
             WHERE profile_type = #{genomicDataFilter.profileType}
                 AND hugo_gene_symbol = #{genomicDataFilter.hugoGeneSymbol}
+            <if test="studyViewFilterHelper.involvedCancerStudies != null and !studyViewFilterHelper.involvedCancerStudies.isEmpty()">
                 AND cancer_study_identifier IN
-                <foreach item="studyId" collection="studyViewFilterHelper.studyViewFilter.studyIds" open="(" separator="," close=")">
+                <foreach item="studyId" collection="studyViewFilterHelper.involvedCancerStudies" open="(" separator="," close=")">
                     #{studyId}
                 </foreach>
+            </if>
         )
         SELECT DISTINCT sd.sample_unique_id
         <!-- join with sample table to get all 'NA' samples -->
         FROM sample_derived sd
             LEFT JOIN cna_query ON sd.sample_unique_id = cna_query.sampleUniqueId
-        WHERE cancer_study_identifier IN
-            <foreach item="studyId" collection="studyViewFilterHelper.studyViewFilter.studyIds" open="(" separator="," close=")">
+        WHERE
+        <if test="studyViewFilterHelper.involvedCancerStudies != null and !studyViewFilterHelper.involvedCancerStudies.isEmpty()">
+            cancer_study_identifier IN
+            <foreach item="studyId" collection="studyViewFilterHelper.involvedCancerStudies" open="(" separator="," close=")">
                 #{studyId}
             </foreach>
-            <foreach item="dataFilterValue" collection="genomicDataFilter.values" open="AND (" separator=" OR " close=")">
+            AND
+        </if>
+            <foreach item="dataFilterValue" collection="genomicDataFilter.values" open="(" separator=" OR " close=")">
                 <choose>
                     <!-- NA value samples -->
                     <when test="dataFilterValue.value == 'NA'">alteration_value IS null</when>

src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml

@@ -346,11 +346,14 @@
                     (
                         SELECT count() > 0
                         FROM genetic_profile
-                        WHERE patient_level = 1
-                        AND stable_id IN
-                        <foreach item="studyId" collection="studyViewFilterHelper.studyViewFilter.studyIds" open="(" separator="," close=")">
-                            concat(#{studyId}, '_', #{profileType})
-                        </foreach>
+                        WHERE
+                            patient_level = 1
+                        <if test="studyViewFilterHelper.involvedCancerStudies != null and !studyViewFilterHelper.involvedCancerStudies.isEmpty()">
+                            AND stable_id IN
+                            <foreach item="studyId" collection="studyViewFilterHelper.involvedCancerStudies" open="(" separator="," close=")">
+                                concat(#{studyId}, '_', #{profileType})
+                            </foreach>
+                        </if>
                     ),
                     (SELECT * FROM (<include refid="getTotalPatientCount"/>)),
                     (SELECT * FROM (<include refid="getTotalSampleCount"/>))

src/test/java/org/cbioportal/persistence/mybatisclickhouse/CNAGenesTest.java

@@ -40,7 +40,7 @@ public class CNAGenesTest extends AbstractTestcontainers {
     public void getCnaGenes() {
         StudyViewFilter studyViewFilter = new StudyViewFilter();
         studyViewFilter.setStudyIds(List.of(STUDY_TCGA_PUB));
-        var alterationCountByGenes = studyViewMapper.getCnaGenes(StudyViewFilterHelper.build(studyViewFilter, null, null),
+        var alterationCountByGenes = studyViewMapper.getCnaGenes(StudyViewFilterHelper.build(studyViewFilter, null, null, studyViewFilter.getStudyIds()),
             AlterationFilterHelper.build(studyViewFilter.getAlterationFilter()));
         assertEquals(3, alterationCountByGenes.size());
 
@@ -63,7 +63,7 @@ public void getCnaGenesWithAlterationFilter() {
         cnaEventTypeFilterMap.put(CNA.AMP, true);
         alterationFilter.setCopyNumberAlterationEventTypes(cnaEventTypeFilterMap);
 
-        var alterationCountByGenes = studyViewMapper.getCnaGenes(StudyViewFilterHelper.build(studyViewFilter, null, null),
+        var alterationCountByGenes = studyViewMapper.getCnaGenes(StudyViewFilterHelper.build(studyViewFilter, null, null, studyViewFilter.getStudyIds()),
             AlterationFilterHelper.build(alterationFilter));
         assertEquals(2, alterationCountByGenes.size());