diff --git a/src/main/java/org/cbioportal/persistence/StudyViewRepository.java b/src/main/java/org/cbioportal/persistence/StudyViewRepository.java index b5c7748ffe9..3b60e17fcd4 100644 --- a/src/main/java/org/cbioportal/persistence/StudyViewRepository.java +++ b/src/main/java/org/cbioportal/persistence/StudyViewRepository.java @@ -9,6 +9,7 @@ import org.cbioportal.model.CopyNumberCountByGene; import org.cbioportal.model.GenomicDataCountItem; import org.cbioportal.model.GenomicDataCount; +import org.cbioportal.model.MolecularProfile; import org.cbioportal.model.PatientTreatment; import org.cbioportal.model.Sample; import org.cbioportal.model.SampleTreatment; @@ -67,4 +68,10 @@ public interface StudyViewRepository { Map getMutationCounts(StudyViewFilterContext studyViewFilterContext, GenomicDataFilter genomicDataFilter); List getMutationCountsByType(StudyViewFilterContext studyViewFilterContext, List genomicDataFilters); + + List getGenomicDataBinCounts(StudyViewFilterContext studyViewFilterContext, List filteredAttributes); + + List getGenericAssayDataBinCounts(StudyViewFilterContext studyViewFilterContext, List filteredAttributes); + + List getGenericAssayProfiles(); } diff --git a/src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java b/src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java index 14c77b47f04..7359ad50c16 100644 --- a/src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java +++ b/src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java @@ -1,6 +1,7 @@ package org.cbioportal.persistence.helper; import org.cbioportal.model.ClinicalAttribute; +import org.cbioportal.model.MolecularProfile; import org.cbioportal.persistence.enums.ClinicalAttributeDataSource; import org.cbioportal.web.parameter.CategorizedClinicalDataCountFilter; import org.cbioportal.web.parameter.CustomSampleIdentifier; @@ -13,10 +14,12 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; public final class StudyViewFilterHelper { public static StudyViewFilterHelper build(@Nullable StudyViewFilter studyViewFilter, @Nullable Map> clinicalAttributesMap, + @Nullable Map> genericAssayProfilesMap, @Nullable List customDataSamples) { if (Objects.isNull(studyViewFilter)) { studyViewFilter = new StudyViewFilter(); @@ -24,10 +27,13 @@ public static StudyViewFilterHelper build(@Nullable StudyViewFilter studyViewFil if (Objects.isNull(clinicalAttributesMap)) { clinicalAttributesMap = new EnumMap<>(ClinicalAttributeDataSource.class); } + if (Objects.isNull(genericAssayProfilesMap)) { + genericAssayProfilesMap = new EnumMap<>(ClinicalAttributeDataSource.class); + } if (Objects.isNull(customDataSamples)) { customDataSamples = new ArrayList<>(); } - return new StudyViewFilterHelper(studyViewFilter, clinicalAttributesMap, customDataSamples); + return new StudyViewFilterHelper(studyViewFilter, clinicalAttributesMap, genericAssayProfilesMap, customDataSamples); } private final StudyViewFilter studyViewFilter; @@ -37,9 +43,10 @@ public static StudyViewFilterHelper build(@Nullable StudyViewFilter studyViewFil private StudyViewFilterHelper(@NonNull StudyViewFilter studyViewFilter, @NonNull Map> clinicalAttributesMap, + @NonNull Map> genericAssayProfilesMap, @NonNull List customDataSamples) { this.studyViewFilter = studyViewFilter; - this.categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter, clinicalAttributesMap); + this.categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter, clinicalAttributesMap, genericAssayProfilesMap); this.customDataSamples = customDataSamples; this.applyPatientIdFilters = shouldApplyPatientIdFilters(); } @@ -56,46 +63,77 @@ public List customDataSamples() { return this.customDataSamples; } - private CategorizedClinicalDataCountFilter extractClinicalDataCountFilters(final StudyViewFilter studyViewFilter, Map> clinicalAttributesMap) { - - if (studyViewFilter.getClinicalDataFilters() == null || clinicalAttributesMap.isEmpty()) { + private CategorizedClinicalDataCountFilter extractClinicalDataCountFilters(final StudyViewFilter studyViewFilter, Map> clinicalAttributesMap, Map> genericAssayProfilesMap) { + if ((studyViewFilter.getClinicalDataFilters() == null || clinicalAttributesMap.isEmpty()) && + (studyViewFilter.getGenericAssayDataFilters() == null || genericAssayProfilesMap.isEmpty()) && + studyViewFilter.getGenomicDataFilters() == null) + { return CategorizedClinicalDataCountFilter.getBuilder().build(); } - List patientCategoricalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.PATIENT) - .stream().filter(ca -> ca.getDatatype().equals("STRING")) - .map(ClinicalAttribute::getAttrId) - .toList(); + CategorizedClinicalDataCountFilter.Builder builder = CategorizedClinicalDataCountFilter.getBuilder(); + + if (studyViewFilter.getClinicalDataFilters() != null) { + List patientCategoricalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.PATIENT) + .stream().filter(ca -> ca.getDatatype().equals("STRING")) + .map(ClinicalAttribute::getAttrId) + .toList(); - List patientNumericalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.PATIENT) - .stream().filter(ca -> ca.getDatatype().equals("NUMBER")) - .map(ClinicalAttribute::getAttrId) - .toList(); + List patientNumericalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.PATIENT) + .stream().filter(ca -> ca.getDatatype().equals("NUMBER")) + .map(ClinicalAttribute::getAttrId) + .toList(); - List sampleCategoricalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.SAMPLE) - .stream().filter(ca -> ca.getDatatype().equals("STRING")) - .map(ClinicalAttribute::getAttrId) - .toList(); + List sampleCategoricalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.SAMPLE) + .stream().filter(ca -> ca.getDatatype().equals("STRING")) + .map(ClinicalAttribute::getAttrId) + .toList(); - List sampleNumericalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.SAMPLE) - .stream().filter(ca -> ca.getDatatype().equals("NUMBER")) - .map(ClinicalAttribute::getAttrId) - .toList(); + List sampleNumericalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.SAMPLE) + .stream().filter(ca -> ca.getDatatype().equals("NUMBER")) + .map(ClinicalAttribute::getAttrId) + .toList(); + + builder.setPatientCategoricalClinicalDataFilters(studyViewFilter.getClinicalDataFilters() + .stream().filter(clinicalDataFilter -> patientCategoricalAttributes.contains(clinicalDataFilter.getAttributeId())) + .collect(Collectors.toList())) + .setPatientNumericalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() + .filter(clinicalDataFilter -> patientNumericalAttributes.contains(clinicalDataFilter.getAttributeId())) + .collect(Collectors.toList())) + .setSampleCategoricalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() + .filter(clinicalDataFilter -> sampleCategoricalAttributes.contains(clinicalDataFilter.getAttributeId())) + .collect(Collectors.toList())) + .setSampleNumericalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() + .filter(clinicalDataFilter -> sampleNumericalAttributes.contains(clinicalDataFilter.getAttributeId())) + .collect(Collectors.toList())); + } + if (studyViewFilter.getGenomicDataFilters() != null) { + builder.setSampleNumericalGenomicDataFilters(studyViewFilter.getGenomicDataFilters().stream() + .filter(genomicDataFilter -> !genomicDataFilter.getProfileType().equals("cna") && !genomicDataFilter.getProfileType().equals("gistic")) + .collect(Collectors.toList())); + builder.setSampleCategoricalGenomicDataFilters(studyViewFilter.getGenomicDataFilters().stream() + .filter(genomicDataFilter -> genomicDataFilter.getProfileType().equals("cna") || genomicDataFilter.getProfileType().equals("gistic")) + .collect(Collectors.toList())); + } + if (studyViewFilter.getGenericAssayDataFilters() != null) { + // TODO: Support patient level profiles and data filtering + List sampleCategoricalProfileTypes = genericAssayProfilesMap.get(ClinicalAttributeDataSource.SAMPLE) + .stream().filter(profile -> profile.getDatatype().equals("CATEGORICAL") || profile.getDatatype().equals("BINARY")) + .map(profile -> profile.getStableId().replace(profile.getCancerStudyIdentifier() + "_", "")) + .toList(); - return CategorizedClinicalDataCountFilter.getBuilder() - .setPatientCategoricalClinicalDataFilters(studyViewFilter.getClinicalDataFilters() - .stream().filter(clinicalDataFilter -> patientCategoricalAttributes.contains(clinicalDataFilter.getAttributeId())) - .toList()) - .setPatientNumericalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() - .filter(clinicalDataFilter -> patientNumericalAttributes.contains(clinicalDataFilter.getAttributeId())) - .toList()) - .setSampleCategoricalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() - .filter(clinicalDataFilter -> sampleCategoricalAttributes.contains(clinicalDataFilter.getAttributeId())) - .toList()) - .setSampleNumericalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() - .filter(clinicalDataFilter -> sampleNumericalAttributes.contains(clinicalDataFilter.getAttributeId())) - .toList()) - .build(); + List sampleNumericalProfileTypes = genericAssayProfilesMap.get(ClinicalAttributeDataSource.SAMPLE) + .stream().filter(profile -> profile.getDatatype().equals("LIMIT-VALUE")) + .map(profile -> profile.getStableId().replace(profile.getCancerStudyIdentifier() + "_", "")) + .toList(); + builder.setSampleNumericalGenericAssayDataFilters(studyViewFilter.getGenericAssayDataFilters().stream() + .filter(genericAssayDataFilter -> sampleNumericalProfileTypes.contains(genericAssayDataFilter.getProfileType())) + .collect(Collectors.toList())); + builder.setSampleCategoricalGenericAssayDataFilters(studyViewFilter.getGenericAssayDataFilters().stream() + .filter(genericAssayDataFilter -> sampleCategoricalProfileTypes.contains(genericAssayDataFilter.getProfileType())) + .collect(Collectors.toList())); + } + return builder.build(); } public boolean shouldApplyPatientIdFilters() { diff --git a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java index 6dabd46d0df..5bc659e5c5a 100644 --- a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java +++ b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java @@ -12,6 +12,7 @@ import org.cbioportal.model.GenomicDataCount; import org.cbioportal.model.GenomicDataCountItem; import org.cbioportal.model.PatientTreatment; +import org.cbioportal.model.MolecularProfile; import org.cbioportal.model.Sample; import org.cbioportal.model.SampleTreatment; import org.cbioportal.persistence.helper.AlterationFilterHelper; @@ -65,4 +66,9 @@ public interface StudyViewMapper { Map getMutationCounts(StudyViewFilterHelper studyViewFilterHelper, GenomicDataFilter genomicDataFilter); List getMutationCountsByType(StudyViewFilterHelper studyViewFilterHelper, List genomicDataFilters); + + List getGenomicDataBinCounts(StudyViewFilterHelper studyViewFilterHelper, List attributeIds); + List getGenericAssayDataBinCounts(StudyViewFilterHelper studyViewFilterHelper, List attributeIds); + + List getGenericAssayProfiles(); } diff --git a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java index 5bf385f366e..dc5d147e88b 100644 --- a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java +++ b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java @@ -9,6 +9,7 @@ import org.cbioportal.model.GenomicDataCountItem; import org.cbioportal.model.GenomicDataCount; import org.cbioportal.model.CopyNumberCountByGene; +import org.cbioportal.model.MolecularProfile; import org.cbioportal.model.PatientTreatment; import org.cbioportal.model.Sample; import org.cbioportal.model.SampleTreatment; @@ -17,8 +18,10 @@ import org.cbioportal.persistence.enums.ClinicalAttributeDataSource; import org.cbioportal.persistence.helper.AlterationFilterHelper; import org.cbioportal.persistence.helper.StudyViewFilterHelper; +import org.cbioportal.web.parameter.CategorizedClinicalDataCountFilter; import org.cbioportal.web.parameter.ClinicalDataType; import org.cbioportal.web.parameter.GenomicDataFilter; +import org.cbioportal.web.parameter.StudyViewFilter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; @@ -34,6 +37,7 @@ public class StudyViewMyBatisRepository implements StudyViewRepository { private Map> clinicalAttributesMap = new HashMap<>(); + private Map> genericAssayProfilesMap = new HashMap<>(); private static final List FILTERED_CLINICAL_ATTR_VALUES = Collections.emptyList(); @@ -101,7 +105,7 @@ public List getMolecularProfileSampleCounts(StudyViewFilterCon } public StudyViewFilterHelper createStudyViewFilterHelper(StudyViewFilterContext studyViewFilterContext) { - return StudyViewFilterHelper.build(studyViewFilterContext.studyViewFilter(), getClinicalAttributeNameMap(), studyViewFilterContext.customDataFilterSamples()); + return StudyViewFilterHelper.build(studyViewFilterContext.studyViewFilter(), getClinicalAttributeNameMap(), getGenericAssayProfilesMap(), studyViewFilterContext.customDataFilterSamples()); } @Override @@ -109,6 +113,11 @@ public List getClinicalAttributes() { return mapper.getClinicalAttributes(); } + @Override + public List getGenericAssayProfiles() { + return mapper.getGenericAssayProfiles(); + } + @Override public Map getClinicalAttributeDatatypeMap() { if (clinicalAttributesMap.isEmpty()) { @@ -199,12 +208,28 @@ public List getSampleTreatments(StudyViewFilterContext studyVie public int getTotalSampleTreatmentCount(StudyViewFilterContext studyViewFilterContext) { return mapper.getTotalSampleTreatmentCounts(createStudyViewFilterHelper(studyViewFilterContext)); } + + @Override + public List getGenomicDataBinCounts(StudyViewFilterContext studyViewFilterContext, List attributeIds) { + return mapper.getGenomicDataBinCounts(createStudyViewFilterHelper(studyViewFilterContext), attributeIds); + } + + @Override + public List getGenericAssayDataBinCounts(StudyViewFilterContext studyViewFilterContext, List attributeIds) { + return mapper.getGenericAssayDataBinCounts(createStudyViewFilterHelper(studyViewFilterContext), attributeIds); + } private void buildClinicalAttributeNameMap() { clinicalAttributesMap = this.getClinicalAttributes() .stream() .collect(Collectors.groupingBy(ca -> ca.getPatientAttribute() ? ClinicalAttributeDataSource.PATIENT : ClinicalAttributeDataSource.SAMPLE)); } + + private void buildGenericAssayProfilesMap() { + genericAssayProfilesMap = this.getGenericAssayProfiles() + .stream() + .collect(Collectors.groupingBy(ca -> ca.getPatientLevel() ? ClinicalAttributeDataSource.PATIENT : ClinicalAttributeDataSource.SAMPLE)); + } private Map> getClinicalAttributeNameMap() { if (clinicalAttributesMap.isEmpty()) { @@ -212,6 +237,13 @@ private Map> getClinicalAtt } return clinicalAttributesMap; } + + private Map> getGenericAssayProfilesMap() { + if (genericAssayProfilesMap.isEmpty()) { + buildGenericAssayProfilesMap(); + } + return genericAssayProfilesMap; + } @Override public List getCNACounts(StudyViewFilterContext studyViewFilterContext, List genomicDataFilters) { diff --git a/src/main/java/org/cbioportal/service/StudyViewColumnarService.java b/src/main/java/org/cbioportal/service/StudyViewColumnarService.java index 369f33aab41..4a631a67204 100644 --- a/src/main/java/org/cbioportal/service/StudyViewColumnarService.java +++ b/src/main/java/org/cbioportal/service/StudyViewColumnarService.java @@ -45,6 +45,10 @@ public interface StudyViewColumnarService { List getCNACountsByGeneSpecific(StudyViewFilter studyViewFilter, List genomicDataFilters); List getMutationCountsByGeneSpecific(StudyViewFilter studyViewFilter, List genomicDataFilters); + + List getGenomicDataBinCounts(StudyViewFilter studyViewFilter, List filteredAttributes); + + List getGenericAssayDataBinCounts(StudyViewFilter studyViewFilter, List filteredAttributes); List getMutationTypeCountsByGeneSpecific(StudyViewFilter studyViewFilter, List genomicDataFilters); } diff --git a/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java b/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java index c3ebffa4a4f..5c7c1756ed8 100644 --- a/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java +++ b/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java @@ -81,6 +81,30 @@ public SampleTreatmentReport getSampleTreatmentReport(StudyViewFilter studyViewF return treatmentCountReportService.getSampleTreatmentReport(createContext(studyViewFilter)); } + @Override + public List getGenomicDataBinCounts(StudyViewFilter studyViewFilter, List filteredAttributes) { + return studyViewRepository.getGenomicDataBinCounts(createContext(studyViewFilter), filteredAttributes) + .stream().collect(Collectors.groupingBy(ClinicalDataCount::getAttributeId)) + .entrySet().parallelStream().map(e -> { + ClinicalDataCountItem item = new ClinicalDataCountItem(); + item.setAttributeId(e.getKey()); + item.setCounts(e.getValue()); + return item; + }).collect(Collectors.toList()); + } + + @Override + public List getGenericAssayDataBinCounts(StudyViewFilter studyViewFilter, List filteredAttributes) { + return studyViewRepository.getGenericAssayDataBinCounts(createContext(studyViewFilter), filteredAttributes) + .stream().collect(Collectors.groupingBy(ClinicalDataCount::getAttributeId)) + .entrySet().parallelStream().map(e -> { + ClinicalDataCountItem item = new ClinicalDataCountItem(); + item.setAttributeId(e.getKey()); + item.setCounts(e.getValue()); + return item; + }).collect(Collectors.toList()); + } + public List getCnaGenes(StudyViewFilter studyViewFilter) { return alterationCountService.getCnaGenes(createContext(studyViewFilter)); } diff --git a/src/main/java/org/cbioportal/web/columnar/BasicDataBinner.java b/src/main/java/org/cbioportal/web/columnar/BasicDataBinner.java new file mode 100644 index 00000000000..f47beb9f9b6 --- /dev/null +++ b/src/main/java/org/cbioportal/web/columnar/BasicDataBinner.java @@ -0,0 +1,306 @@ +package org.cbioportal.web.columnar; + +import org.cbioportal.model.*; +import org.cbioportal.service.StudyViewColumnarService; +import org.cbioportal.web.columnar.util.NewClinicalDataBinUtil; +import org.cbioportal.web.parameter.*; +import org.cbioportal.web.util.DataBinner; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.stereotype.Component; + +import java.util.*; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; + +@Component +public class BasicDataBinner { + private final StudyViewColumnarService studyViewColumnarService; + private final DataBinner dataBinner; + + @Autowired + public BasicDataBinner( + StudyViewColumnarService studyViewColumnarService, + DataBinner dataBinner + ) { + this.studyViewColumnarService = studyViewColumnarService; + this.dataBinner = dataBinner; + } + + // convert from counts to clinical data + private List convertCountsToData(List clinicalDataCounts) + { + return clinicalDataCounts + .stream() + .map(NewClinicalDataBinUtil::generateClinicalDataFromClinicalDataCount) + .flatMap(Collection::stream) + .toList(); + } + + @Cacheable(cacheResolver = "generalRepositoryCacheResolver", condition = "@cacheEnabledConfig.getEnabled()") + public List getDataBins( + DataBinMethod dataBinMethod, + T dataBinCountFilter, + boolean shouldRemoveSelfFromFilter) { + // get data bin filters based on the type of the filter + // either Genomic data or Generic Assay data or clinical data + List dataBinFilters = fetchDataBinFilters(dataBinCountFilter); + StudyViewFilter studyViewFilter = dataBinCountFilter.getStudyViewFilter(); + + if (shouldRemoveSelfFromFilter && dataBinFilters.size() == 1) { + removeSelfFromFilter(dataBinFilters.get(0), studyViewFilter); + } + + List uniqueKeys = dataBinFilters.stream().map(this::getDataBinFilterUniqueKey).collect(Collectors.toList()); + + // a new StudyView filter to partially filter by study and sample ids only + // we need this additional partial filter because we always need to know the bins generated for the initial state + // which allows us to keep the number of bins and bin ranges consistent even if there are additional data filters. + // we only want to update the counts for each bin, we don't want to regenerate the bins for the filtered data. + // NOTE: partial filter is only needed when dataBinMethod == DataBinMethod.STATIC but that's always the case + // for the frontend implementation. we can't really use dataBinMethod == DataBinMethod.DYNAMIC because of the + // complication it brings to the frontend visualization and filtering + StudyViewFilter partialFilter = new StudyViewFilter(); + partialFilter.setStudyIds(studyViewFilter.getStudyIds()); + partialFilter.setSampleIdentifiers(studyViewFilter.getSampleIdentifiers()); + + // we need to fetch data for the partial filter in order to generate the bins for initial state + // we use the filtered data to calculate the counts for each bin, we do not regenerate bins for the filtered data + List unfilteredClinicalDataCounts; + List filteredClinicalDataCounts; + Map attributeDatatypeMap; + switch (dataBinCountFilter) { + // TODO: first case is to support clinical data, but clinical data is not using this now. We should update controller to use this method later + case ClinicalDataBinCountFilter clinicalDataBinCountFilter -> { + unfilteredClinicalDataCounts = studyViewColumnarService.getClinicalDataCounts(partialFilter, uniqueKeys); + filteredClinicalDataCounts = studyViewColumnarService.getClinicalDataCounts(studyViewFilter, uniqueKeys); + attributeDatatypeMap = studyViewColumnarService.getClinicalAttributeDatatypeMap(); + } + case GenomicDataBinCountFilter genomicDataBinCountFilter -> { + unfilteredClinicalDataCounts = studyViewColumnarService.getGenomicDataBinCounts(partialFilter, uniqueKeys); + filteredClinicalDataCounts = studyViewColumnarService.getGenomicDataBinCounts(studyViewFilter, uniqueKeys); + attributeDatatypeMap = Collections.emptyMap(); + } + case GenericAssayDataBinCountFilter genericAssayDataBinCountFilter -> { + unfilteredClinicalDataCounts = studyViewColumnarService.getGenericAssayDataBinCounts(partialFilter, uniqueKeys); + filteredClinicalDataCounts = studyViewColumnarService.getGenericAssayDataBinCounts(studyViewFilter, uniqueKeys); + attributeDatatypeMap = Collections.emptyMap(); + } + default -> { + unfilteredClinicalDataCounts = Collections.emptyList(); + filteredClinicalDataCounts = Collections.emptyList(); + attributeDatatypeMap = Collections.emptyMap(); + } + } + + // TODO ignoring conflictingPatientAttributeIds for now + List unfilteredClinicalData = convertCountsToData( + unfilteredClinicalDataCounts.stream().flatMap(c -> c.getCounts().stream()).toList() + ); + List filteredClinicalData = convertCountsToData( + filteredClinicalDataCounts.stream().flatMap(c -> c.getCounts().stream()).toList() + ); + + Map> unfilteredClinicalDataByAttributeId = + unfilteredClinicalData.stream().collect(Collectors.groupingBy(Binnable::getAttrId)); + + Map> filteredClinicalDataByAttributeId = + filteredClinicalData.stream().collect(Collectors.groupingBy(Binnable::getAttrId)); + + // Define result variables + List resultDataBins = Collections.emptyList(); + + // TODO: need to update attributeDatatypeMap to include patient level data for Generic Assay Profiles + if (dataBinMethod == DataBinMethod.STATIC) { + if (!unfilteredClinicalData.isEmpty()) { + resultDataBins = calculateStaticDataBins( + dataBinner, + dataBinFilters, + attributeDatatypeMap, + unfilteredClinicalDataByAttributeId, + filteredClinicalDataByAttributeId + ); + } + } + // TODO: need to update attributeDatatypeMap to include patient level data for Generic Assay Profiles + else { // dataBinMethod == DataBinMethod.DYNAMIC + // TODO we should consider removing dynamic binning support + // we never use dynamic binning in the frontend because number of bins and the bin ranges can change + // each time there is a new filter which makes the frontend implementation complicated + if (!filteredClinicalData.isEmpty()) { + resultDataBins = calculateDynamicDataBins( + dataBinner, + dataBinFilters, + attributeDatatypeMap, + filteredClinicalDataByAttributeId + ); + } + } + + return resultDataBins; + } + + private void removeSelfFromFilter(S dataBinFilter, StudyViewFilter studyViewFilter) { + if (studyViewFilter != null) { + if (dataBinFilter instanceof ClinicalDataBinFilter clinicalDataBinFilter && + studyViewFilter.getClinicalDataFilters() != null) { + studyViewFilter.getClinicalDataFilters().removeIf(f -> f.getAttributeId().equals(clinicalDataBinFilter.getAttributeId())); + } else if (dataBinFilter instanceof GenomicDataBinFilter genomicDataBinFilter && + studyViewFilter.getGenomicDataFilters() != null) { + studyViewFilter.getGenomicDataFilters().removeIf(f -> + f.getHugoGeneSymbol().equals(genomicDataBinFilter.getHugoGeneSymbol()) + && f.getProfileType().equals(genomicDataBinFilter.getProfileType()) + ); + } else if (dataBinFilter instanceof GenericAssayDataBinFilter genericAssayDataBinFilter && + studyViewFilter.getGenericAssayDataFilters() != null) { + studyViewFilter.getGenericAssayDataFilters().removeIf(f -> + f.getStableId().equals(genericAssayDataBinFilter.getStableId()) + && f.getProfileType().equals(genericAssayDataBinFilter.getProfileType()) + ); + } + } + } + + private List fetchDataBinFilters(T dataBinCountFilter) { + if (dataBinCountFilter instanceof ClinicalDataBinCountFilter) { + return (List) ((ClinicalDataBinCountFilter) dataBinCountFilter).getAttributes(); + } else if (dataBinCountFilter instanceof GenomicDataBinCountFilter) { + return (List) ((GenomicDataBinCountFilter) dataBinCountFilter).getGenomicDataBinFilters(); + } else if (dataBinCountFilter instanceof GenericAssayDataBinCountFilter) { + return (List) ((GenericAssayDataBinCountFilter) dataBinCountFilter).getGenericAssayDataBinFilters(); + } + return new ArrayList<>(); + } + + private String getDataBinFilterUniqueKey(S dataBinFilter) { + if (dataBinFilter instanceof ClinicalDataBinFilter clinicalDataBinFilter) { + return clinicalDataBinFilter.getAttributeId(); + } else if (dataBinFilter instanceof GenomicDataBinFilter genomicDataBinFilter) { + return genomicDataBinFilter.getHugoGeneSymbol() + genomicDataBinFilter.getProfileType(); + } else if (dataBinFilter instanceof GenericAssayDataBinFilter genericAssayDataBinFilter) { + return genericAssayDataBinFilter.getStableId() + genericAssayDataBinFilter.getProfileType(); + } + return null; + } + + private List calculateStaticDataBins( + DataBinner dataBinner, + List dataBinFilters, + Map attributeDatatypeMap, + Map> unfilteredClinicalDataByAttributeId, + Map> filteredClinicalDataByAttributeId + ) { + List result = new ArrayList<>(); + + for (T dataBinFilter : dataBinFilters) { + // if there is data for requested attribute + // TODO: consider if this is correct to passing in a empty map + if (attributeDatatypeMap.isEmpty() || attributeDatatypeMap.containsKey(getDataBinFilterUniqueKey(dataBinFilter))) { + List dataBins = dataBinner + .calculateClinicalDataBins( + dataBinFilter, + filteredClinicalDataByAttributeId.getOrDefault(getDataBinFilterUniqueKey(dataBinFilter), emptyList()), + unfilteredClinicalDataByAttributeId.getOrDefault(getDataBinFilterUniqueKey(dataBinFilter), emptyList()) + ) + .stream() + .map(dataBin -> (U) transform(dataBinFilter, dataBin)) + .toList(); + + result.addAll(dataBins); + } + } + + return result; + } + + private List calculateDynamicDataBins( + DataBinner dataBinner, + List dataBinFilters, + Map attributeDatatypeMap, + Map> filteredClinicalDataByAttributeId + ) { + List result = new ArrayList<>(); + + for (T dataBinFilter : dataBinFilters) { + // if there is data for requested attribute + // TODO: consider if this is correct to passing in a empty map + if (attributeDatatypeMap.isEmpty() || attributeDatatypeMap.containsKey(getDataBinFilterUniqueKey(dataBinFilter))) { + List dataBins = dataBinner + .calculateDataBins( + dataBinFilter, + filteredClinicalDataByAttributeId.getOrDefault(getDataBinFilterUniqueKey(dataBinFilter), emptyList()) + ) + .stream() + .map(dataBin -> (U) transform(dataBinFilter, dataBin)) + .toList(); + result.addAll(dataBins); + } + } + + return result; + } + + private T transform(S dataBinFilter, DataBin dataBin) { + if (dataBinFilter instanceof ClinicalDataBinFilter clinicalDataBinFilter) { + return (T) dataBinToClinicalDataBin(clinicalDataBinFilter, dataBin); + } else if (dataBinFilter instanceof GenomicDataBinFilter genomicDataBinFilter) { + return (T) dataBintoGenomicDataBin(genomicDataBinFilter, dataBin); + } else if (dataBinFilter instanceof GenericAssayDataBinFilter genericAssayDataBinFilter) { + return (T) dataBintoGenericAssayDataBin(genericAssayDataBinFilter, dataBin); + } + return null; + } + + private ClinicalDataBin dataBinToClinicalDataBin(ClinicalDataBinFilter attribute, DataBin dataBin) { + ClinicalDataBin clinicalDataBin = new ClinicalDataBin(); + clinicalDataBin.setAttributeId(attribute.getAttributeId()); + clinicalDataBin.setCount(dataBin.getCount()); + if (dataBin.getEnd() != null) { + clinicalDataBin.setEnd(dataBin.getEnd()); + } + if (dataBin.getSpecialValue() != null) { + clinicalDataBin.setSpecialValue(dataBin.getSpecialValue()); + } + if (dataBin.getStart() != null) { + clinicalDataBin.setStart(dataBin.getStart()); + } + return clinicalDataBin; + } + + private GenomicDataBin dataBintoGenomicDataBin(GenomicDataBinFilter genomicDataBinFilter, DataBin dataBin) { + GenomicDataBin genomicDataBin = new GenomicDataBin(); + genomicDataBin.setCount(dataBin.getCount()); + genomicDataBin.setHugoGeneSymbol(genomicDataBinFilter.getHugoGeneSymbol()); + genomicDataBin.setProfileType(genomicDataBinFilter.getProfileType()); + if (dataBin.getSpecialValue() != null) { + genomicDataBin.setSpecialValue(dataBin.getSpecialValue()); + } + if (dataBin.getStart() != null) { + genomicDataBin.setStart(dataBin.getStart()); + } + if (dataBin.getEnd() != null) { + genomicDataBin.setEnd(dataBin.getEnd()); + } + return genomicDataBin; + } + + private GenericAssayDataBin dataBintoGenericAssayDataBin(GenericAssayDataBinFilter genericAssayDataBinFilter, + DataBin dataBin) { + GenericAssayDataBin genericAssayDataBin = new GenericAssayDataBin(); + genericAssayDataBin.setCount(dataBin.getCount()); + genericAssayDataBin.setStableId(genericAssayDataBinFilter.getStableId()); + genericAssayDataBin.setProfileType(genericAssayDataBinFilter.getProfileType()); + if (dataBin.getSpecialValue() != null) { + genericAssayDataBin.setSpecialValue(dataBin.getSpecialValue()); + } + if (dataBin.getStart() != null) { + genericAssayDataBin.setStart(dataBin.getStart()); + } + if (dataBin.getEnd() != null) { + genericAssayDataBin.setEnd(dataBin.getEnd()); + } + return genericAssayDataBin; + } + +} diff --git a/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java b/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java index ffeff0ad7b3..fc55e05481b 100644 --- a/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java +++ b/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java @@ -19,6 +19,8 @@ import org.cbioportal.model.ClinicalViolinPlotData; import org.cbioportal.model.CopyNumberCountByGene; import org.cbioportal.model.DensityPlotData; +import org.cbioportal.model.GenericAssayDataBin; +import org.cbioportal.model.GenomicDataBin; import org.cbioportal.model.GenomicDataCount; import org.cbioportal.model.PatientTreatmentReport; import org.cbioportal.model.Sample; @@ -34,6 +36,8 @@ import org.cbioportal.web.parameter.ClinicalDataCountFilter; import org.cbioportal.web.parameter.ClinicalDataFilter; import org.cbioportal.web.parameter.DataBinMethod; +import org.cbioportal.web.parameter.GenericAssayDataBinCountFilter; +import org.cbioportal.web.parameter.GenomicDataBinCountFilter; import org.cbioportal.web.parameter.GenomicDataCountFilter; import org.cbioportal.web.parameter.GenomicDataFilter; import org.cbioportal.web.parameter.MutationOption; @@ -70,6 +74,8 @@ public class StudyViewColumnStoreController { private final StudyViewColumnarService studyViewColumnarService; private final ClinicalDataBinner clinicalDataBinner; + + private final BasicDataBinner basicDataBinner; private final ClinicalDataDensityPlotService clinicalDataDensityPlotService; private final ViolinPlotService violinPlotService; @@ -79,11 +85,13 @@ public class StudyViewColumnStoreController { @Autowired public StudyViewColumnStoreController(StudyViewColumnarService studyViewColumnarService, ClinicalDataBinner clinicalDataBinner, + BasicDataBinner basicDataBinner, ClinicalDataDensityPlotService clinicalDataDensityPlotService, ViolinPlotService violinPlotService ) { this.studyViewColumnarService = studyViewColumnarService; this.clinicalDataBinner = clinicalDataBinner; + this.basicDataBinner = basicDataBinner; this.clinicalDataDensityPlotService = clinicalDataDensityPlotService; this.violinPlotService = violinPlotService; } @@ -524,4 +532,43 @@ public ResponseEntity fetchSampleTreatmentCounts( // // return new ResponseEntity<>(clinicalDataBins, HttpStatus.OK); // } + + @PreAuthorize("hasPermission(#involvedCancerStudies, 'Collection', T(org.cbioportal.utils.security.AccessLevel).READ)") + @PostMapping(value = "/column-store/genomic-data-bin-counts/fetch", + consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + @ApiResponse(responseCode = "200", description = "OK", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = GenomicDataBin.class)))) + public ResponseEntity> fetchGenomicDataBinCounts( + @RequestParam(defaultValue = "DYNAMIC") DataBinMethod dataBinMethod, + @RequestBody(required = false) GenomicDataBinCountFilter genomicDataBinCountFilter, + @RequestAttribute(required = false, value = "involvedCancerStudies") Collection involvedCancerStudies, + @RequestAttribute(required = false, value = "interceptedGenomicDataBinCountFilter") GenomicDataBinCountFilter interceptedGenomicDataBinCountFilter + ) { + List genomicDataBins = basicDataBinner.getDataBins( + dataBinMethod, + interceptedGenomicDataBinCountFilter, + true + ); + return new ResponseEntity<>(genomicDataBins, HttpStatus.OK); + } + + @PreAuthorize("hasPermission(#involvedCancerStudies, 'Collection', T(org.cbioportal.utils.security.AccessLevel).READ)") + @PostMapping(value = "/column-store/generic-assay-data-bin-counts/fetch", + consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + @ApiResponse(responseCode = "200", description = "OK", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = GenericAssayDataBin.class)))) + public ResponseEntity> fetchGenericAssayDataBinCounts( + @RequestParam(defaultValue = "DYNAMIC") DataBinMethod dataBinMethod, + @RequestBody(required = false) GenericAssayDataBinCountFilter genericAssayDataBinCountFilter, + @RequestAttribute(required = false, value = "involvedCancerStudies") Collection involvedCancerStudies, + @RequestAttribute(required = false, value = "interceptedGenericAssayDataBinCountFilter") GenericAssayDataBinCountFilter interceptedGenericAssayDataBinCountFilter + ) { + List genericAssayDataBins = basicDataBinner.getDataBins( + dataBinMethod, + interceptedGenericAssayDataBinCountFilter, + true + ); + return new ResponseEntity<>(genericAssayDataBins, HttpStatus.OK); + } + } diff --git a/src/main/java/org/cbioportal/web/parameter/CategorizedClinicalDataCountFilter.java b/src/main/java/org/cbioportal/web/parameter/CategorizedClinicalDataCountFilter.java index 734d9bb53e7..a1e59701d80 100644 --- a/src/main/java/org/cbioportal/web/parameter/CategorizedClinicalDataCountFilter.java +++ b/src/main/java/org/cbioportal/web/parameter/CategorizedClinicalDataCountFilter.java @@ -11,11 +11,29 @@ public static Builder getBuilder() { private final List sampleCategoricalClinicalDataFilters; private final List patientNumericalClinicalDataFilters; private final List patientCategoricalClinicalDataFilters; + + private final List sampleNumericalGenomicDataFilters; + private final List sampleCategoricalGenomicDataFilters; + private final List patientNumericalGenomicDataFilters; + private final List patientCategoricalGenomicDataFilters; + + private final List sampleNumericalGenericAssayDataFilters; + private final List sampleCategoricalGenericAssayDataFilters; + private final List patientNumericalGenericAssayDataFilters; + private final List patientCategoricalGenericAssayDataFilters; private CategorizedClinicalDataCountFilter(Builder builder) { this.sampleCategoricalClinicalDataFilters = builder.sampleCategoricalClinicalDataFilters; this.sampleNumericalClinicalDataFilters = builder.sampleNumericalClinicalDataFilters; this.patientCategoricalClinicalDataFilters = builder.patientCategoricalClinicalDataFilters; this.patientNumericalClinicalDataFilters = builder.patientNumericalClinicalDataFilters; + this.sampleCategoricalGenomicDataFilters = builder.sampleCategoricalGenomicDataFilters; + this.sampleNumericalGenomicDataFilters = builder.sampleNumericalGenomicDataFilters; + this.patientCategoricalGenomicDataFilters = builder.patientCategoricalGenomicDataFilters; + this.patientNumericalGenomicDataFilters = builder.patientNumericalGenomicDataFilters; + this.sampleCategoricalGenericAssayDataFilters = builder.sampleCategoricalGenericAssayDataFilters; + this.sampleNumericalGenericAssayDataFilters = builder.sampleNumericalGenericAssayDataFilters; + this.patientCategoricalGenericAssayDataFilters = builder.patientCategoricalGenericAssayDataFilters; + this.patientNumericalGenericAssayDataFilters = builder.patientNumericalGenericAssayDataFilters; } public List getSampleNumericalClinicalDataFilters() { @@ -34,15 +52,55 @@ public List getPatientCategoricalClinicalDataFilters() { return patientCategoricalClinicalDataFilters; } + public List getSampleNumericalGenomicDataFilters() { + return sampleNumericalGenomicDataFilters; + } + + public List getSampleCategoricalGenomicDataFilters() { + return sampleCategoricalGenomicDataFilters; + } + + public List getPatientNumericalGenomicDataFilters() { + return patientNumericalGenomicDataFilters; + } + + public List getPatientCategoricalGenomicDataFilters() { + return patientCategoricalGenomicDataFilters; + } + + public List getSampleNumericalGenericAssayDataFilters() { + return sampleNumericalGenericAssayDataFilters; + } + + public List getSampleCategoricalGenericAssayDataFilters() { + return sampleCategoricalGenericAssayDataFilters; + } + + public List getPatientNumericalGenericAssayDataFilters() { + return patientNumericalGenericAssayDataFilters; + } + + public List getPatientCategoricalGenericAssayDataFilters() { + return patientCategoricalGenericAssayDataFilters; + } + public static class Builder { private List sampleNumericalClinicalDataFilters; private List sampleCategoricalClinicalDataFilters; private List patientNumericalClinicalDataFilters; private List patientCategoricalClinicalDataFilters; + private List sampleNumericalGenomicDataFilters; + private List sampleCategoricalGenomicDataFilters; + private List patientNumericalGenomicDataFilters; + private List patientCategoricalGenomicDataFilters; + private List sampleNumericalGenericAssayDataFilters; + private List sampleCategoricalGenericAssayDataFilters; + private List patientNumericalGenericAssayDataFilters; + private List patientCategoricalGenericAssayDataFilters; private Builder(){ - + } public Builder setSampleCategoricalClinicalDataFilters(List sampleCategoricalClinicalDataFilters) { this.sampleCategoricalClinicalDataFilters = sampleCategoricalClinicalDataFilters; @@ -63,6 +121,46 @@ public Builder setPatientNumericalClinicalDataFilters(List p this.patientNumericalClinicalDataFilters = patientNumericalClinicalDataFilters; return this; } + + public Builder setSampleCategoricalGenomicDataFilters(List sampleCategoricalGenomicDataFilters) { + this.sampleCategoricalGenomicDataFilters = sampleCategoricalGenomicDataFilters; + return this; + } + + public Builder setSampleNumericalGenomicDataFilters(List sampleNumericalGenomicDataFilters) { + this.sampleNumericalGenomicDataFilters = sampleNumericalGenomicDataFilters; + return this; + } + + public Builder setPatientCategoricalGenomicDataFilters(List patientCategoricalGenomicDataFilters) { + this.patientCategoricalGenomicDataFilters = patientCategoricalGenomicDataFilters; + return this; + } + + public Builder setPatientNumericalGenomicDataFilters(List patientNumericalGenomicDataFilters) { + this.patientNumericalGenomicDataFilters = patientNumericalGenomicDataFilters; + return this; + } + + public Builder setSampleCategoricalGenericAssayDataFilters(List sampleCategoricalGenericAssayDataFilters) { + this.sampleCategoricalGenericAssayDataFilters = sampleCategoricalGenericAssayDataFilters; + return this; + } + + public Builder setSampleNumericalGenericAssayDataFilters(List sampleNumericalGenericAssayDataFilters) { + this.sampleNumericalGenericAssayDataFilters = sampleNumericalGenericAssayDataFilters; + return this; + } + + public Builder setPatientCategoricalGenericAssayDataFilters(List patientCategoricalGenericAssayDataFilters) { + this.patientCategoricalGenericAssayDataFilters = patientCategoricalGenericAssayDataFilters; + return this; + } + + public Builder setPatientNumericalGenericAssayDataFilters(List patientNumericalGenericAssayDataFilters) { + this.patientNumericalGenericAssayDataFilters = patientNumericalGenericAssayDataFilters; + return this; + } public CategorizedClinicalDataCountFilter build() { return new CategorizedClinicalDataCountFilter(this); diff --git a/src/main/java/org/cbioportal/web/util/StudyViewFilterApplier.java b/src/main/java/org/cbioportal/web/util/StudyViewFilterApplier.java index 0fb4e83dc2c..16e5483d6dd 100644 --- a/src/main/java/org/cbioportal/web/util/StudyViewFilterApplier.java +++ b/src/main/java/org/cbioportal/web/util/StudyViewFilterApplier.java @@ -723,7 +723,6 @@ public List getUniqkeyKeys(List studyIds, List caseIds) public List getDataBins( DataBinMethod dataBinMethod, T dataBinCountFilter) { List dataBinFilters = fetchDataBinFilters(dataBinCountFilter); - StudyViewFilter studyViewFilter = dataBinCountFilter.getStudyViewFilter(); if (dataBinFilters.size() == 1) { @@ -733,6 +732,7 @@ public resultDataBins; List filteredSampleIds = new ArrayList<>(); List filteredStudyIds = new ArrayList<>(); + List filteredData = fetchData(dataBinCountFilter, studyViewFilter, filteredSampleIds, filteredStudyIds); @@ -740,7 +740,7 @@ public > filteredClinicalDataByAttributeId = filteredData.stream() .collect(Collectors.groupingBy(Binnable::getAttrId)); - + if (dataBinMethod == DataBinMethod.STATIC) { StudyViewFilter filter = studyViewFilter == null ? null : new StudyViewFilter(); diff --git a/src/main/resources/db-scripts/clickhouse/clickhouse.sql b/src/main/resources/db-scripts/clickhouse/clickhouse.sql index 9e89e8506f9..ffb18b1979d 100644 --- a/src/main/resources/db-scripts/clickhouse/clickhouse.sql +++ b/src/main/resources/db-scripts/clickhouse/clickhouse.sql @@ -323,10 +323,116 @@ FROM WHERE alteration_value != 'NA') AS subquery JOIN sample_derived sd ON sd.internal_id = subquery.sample_id; +CREATE TABLE IF NOT EXISTS genetic_alteration_numerical_derived +( + sample_unique_id String, + cancer_study_identifier LowCardinality(String), + hugo_gene_symbol String, + profile_type LowCardinality(String), + alteration_value String + ) + ENGINE = MergeTree() + ORDER BY (profile_type, cancer_study_identifier, hugo_gene_symbol, sample_unique_id ); + +INSERT INTO TABLE genetic_alteration_numerical_derived +SELECT + sample_unique_id, + cancer_study_identifier, + hugo_gene_symbol, + profile_type, + alteration_value +FROM + (SELECT + sample_id, + hugo_gene_symbol, + profile_type, + alteration_value + FROM + (SELECT + g.hugo_gene_symbol AS hugo_gene_symbol, + replaceOne(stable_id, concat(cs.cancer_study_identifier, '_'), '') as profile_type, -- Compute profile_type + arrayMap(x -> (x = '' ? NULL : x), splitByString(',', assumeNotNull(trim(trailing ',' from ga.values)))) AS alteration_value, + arrayMap(x -> (x = '' ? NULL : toInt32(x)), splitByString(',', assumeNotNull(trim(trailing ',' from gps.ordered_sample_list)))) AS sample_id + FROM + genetic_profile gp + JOIN cancer_study cs ON cs.cancer_study_id = gp.cancer_study_id + JOIN genetic_profile_samples gps ON gp.genetic_profile_id = gps.genetic_profile_id + JOIN genetic_alteration ga ON gp.genetic_profile_id = ga.genetic_profile_id + JOIN gene g ON ga.genetic_entity_id = g.genetic_entity_id + WHERE + gp.genetic_alteration_type != 'COPY_NUMBER_ALTERATION') + ARRAY JOIN alteration_value, sample_id + ) AS subquery + JOIN sample_derived sd ON sd.internal_id = subquery.sample_id; + +CREATE TABLE IF NOT EXISTS generic_assay_data_derived +( + sample_unique_id String, + genetic_entity_id String, + value String, + generic_assay_type String, + profile_stable_id String, + entity_stable_id String, + datatype String, + patient_level NUMERIC, + profile_type String +) + ENGINE = MergeTree() + ORDER BY (profile_type, entity_stable_id, sample_unique_id); + +INSERT INTO TABLE generic_assay_data_derived +SELECT + sd.sample_unique_id as sample_unique_id, + genetic_entity_id, + value, + generic_assay_type, + profile_stable_id, + entity_stable_id, + datatype, + patient_level, + replaceOne(profile_stable_id, concat(cs.cancer_study_identifier, '_'), '') as profile_type +FROM + (SELECT + sample_id, + genetic_entity_id, + value, + cancer_study_id, + generic_assay_type, + genetic_profile_id, + profile_stable_id, + entity_stable_id, + patient_level, + datatype + FROM + (SELECT + sample_id as sample_unique_id, + gp.cancer_study_id AS cancer_study_id, + ga.genetic_entity_id as genetic_entity_id, + gp.genetic_profile_id as genetic_profile_id, + gp.generic_assay_type as generic_assay_type, + gp.stable_id as profile_stable_id, + ge.stable_id as entity_stable_id, + gp.datatype as datatype, + gp.patient_level as patient_level, + arrayMap(x -> (x = '' ? NULL : x), splitByString(',', assumeNotNull(trim(trailing ',' from ga.values)))) AS value, + arrayMap(x -> (x = '' ? NULL : toInt64(x)), splitByString(',', assumeNotNull(trim(trailing ',' from gps.ordered_sample_list)))) AS sample_id + FROM genetic_profile gp + JOIN genetic_profile_samples gps ON gp.genetic_profile_id = gps.genetic_profile_id + JOIN genetic_alteration ga ON gp.genetic_profile_id = ga.genetic_profile_id + JOIN genetic_entity ge on ga.genetic_entity_id = ge.id + WHERE + gp.generic_assay_type IS NOT NULL + ) + ARRAY JOIN value, sample_id) AS subquery + JOIN cancer_study cs ON cs.cancer_study_id = subquery.cancer_study_id + JOIN sample_derived sd ON sd.internal_id = subquery.sample_id; + OPTIMIZE TABLE sample_to_gene_panel_derived; OPTIMIZE TABLE gene_panel_to_gene_derived; OPTIMIZE TABLE sample_derived; OPTIMIZE TABLE genomic_event_derived; OPTIMIZE TABLE clinical_data_derived; OPTIMIZE TABLE clinical_event_derived; -OPTIMIZE TABLE genetic_alteration_cna_derived; \ No newline at end of file +OPTIMIZE TABLE genetic_alteration_cna_derived; +OPTIMIZE TABLE genetic_alteration_numerical_derived; +OPTIMIZE TABLE generic_assay_data_derived; diff --git a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml index f2ce040188e..9e49b493642 100644 --- a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml +++ b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml @@ -149,19 +149,47 @@ - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -339,6 +367,131 @@ + + + SELECT ${unique_id} + FROM ${table_name} + WHERE hugo_gene_symbol = '${genomicDataFilter.hugoGeneSymbol}' AND + profile_type='${genomicDataFilter.profileType}' + + + + AND + + + + + + AND match(alteration_value, '^>?=?[-+]?[0-9]*[.,]?[0-9]+$') + + + AND match(alteration_value, '^<?=?[-+]?[0-9]*[.,]?[0-9]+$') + + + AND match(alteration_value, '^[-+]?[0-9]*[.,]?[0-9]+$') + + + + + AND abs( + minus( + + + , + ${dataFilterValue.start} + ) + ) < exp(-11) + + + + AND + + + > ${dataFilterValue.start} + + + AND + + + <= ${dataFilterValue.end} + + + + + + + + + + + SELECT ${unique_id} + FROM ${table_name} + WHERE entity_stable_id = '${genericAssayDataFilter.stableId}' AND + profile_type='${genericAssayDataFilter.profileType}' + + + + AND + + + + + + AND match(value, '^>?=?[-+]?[0-9]*[.,]?[0-9]+$') + + + AND match(value, '^<?=?[-+]?[0-9]*[.,]?[0-9]+$') + + + AND match(value, '^[-+]?[0-9]*[.,]?[0-9]+$') + + + + + AND abs( + minus( + + + , + ${dataFilterValue.start} + ) + ) < exp(-11) + + + + AND + + + > ${dataFilterValue.start} + + + AND + + + <= ${dataFilterValue.end} + + + + + + + + + + SELECT ${unique_id} + FROM ${table_name} + WHERE entity_stable_id = '${genericAssayDataFilter.stableId}' AND + profile_type='${genericAssayDataFilter.profileType}' + + + AND ( + + + + ) = '${dataFilterValue.value}' + + + diff --git a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml index 5af781dfeb0..0abcd185147 100644 --- a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml +++ b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml @@ -556,6 +556,69 @@ GROUP BY treatments.treatment; + + + + + +