From 89d572891ba577ea0cb3fcf60ef69705f37219d1 Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Mon, 18 Nov 2024 09:32:52 -0500 Subject: [PATCH 1/2] [ALS-7760] Replicate Old Search in new Data-Dictionary (#53) **[CHORE] GH Actions Fix** - Fixed GitHub Actions to resolve workflow issues. **Testing Enhancements** - Added `@ActiveProfiles("test")` to testing classes to remove spam logs from `DataSourceVerifier` during unit tests. **JSON Parsing Refactor** - Created `JsonBlobParser` for improved JSON parsing. - Refactored `ConceptResultSetUtil` to use `JsonBlobParser` for clearer separation of concerns. **Code Cleanup** - Removed unused imports from `ConceptShell` and `ContinuousConcept` classes to improve code readability. **Legacy Search Feature** - Implemented a new legacy search feature including service, controller, and related model classes. - Updated `ConceptRepository` to support legacy search queries. - Added test cases to ensure functionality. **Testing Improvements** - Added unit tests for `LegacySearchQueryMapper` to validate JSON parsing and string replacement. - Introduced integration tests for `LegacySearchController` to verify search response correctness using a PostgreSQL container. **Initial Configurations** - Added application properties for database configuration and dashboard settings. - Introduced Docker commands for local development with sample weights configuration. - Included `weights.csv` and `dictonaryRequest.http` as sample data for testing API requests. **Search Query Refactor** - Created `LegacySearchRepository` to handle legacy search functionalities. - Moved query logic (`ALLOW_FILTERING_Q`) to `QueryUtility`. - Removed legacy search code from `ConceptRepository` for better separation of concerns. **Filter Processing Refactor** - Introduced `FilterProcessor` to centralize filter processing logic. - Enhanced methods in `MetadataResultSetUtil` (`getDescription`, `getParentName`, `getParentDisplay`) for better validation using `StringUtils`. **Repository Enhancements** - Added `LegacySearchRepositoryTest` to verify legacy search functionalities. - Refactored legacy search logic from `ConceptService` into `LegacySearchRepository`. - Cleaned up unused imports and methods for better maintainability. --- Co-authored-by: Luke Sikina --- dictionaryweights/README.md | 11 ++ .../resources/application-bdc-dev.properties | 9 ++ dictionaryweights/weights.csv | 7 ++ dictonaryReqeust.http | 15 +++ .../dictionary/concept/ConceptRepository.java | 22 +--- .../concept/ConceptResultSetUtil.java | 62 ++-------- .../dictionary/concept/ConceptService.java | 1 + .../concept/model/ConceptShell.java | 1 - .../concept/model/ContinuousConcept.java | 1 - .../datasource/DataSourceVerifier.java | 11 +- .../dictionary/facet/FilterPreProcessor.java | 35 +++--- .../dictionary/filter/FilterProcessor.java | 34 ++++++ .../legacysearch/LegacySearchController.java | 32 +++++ .../legacysearch/LegacySearchQueryMapper.java | 34 ++++++ .../legacysearch/LegacySearchRepository.java | 79 +++++++++++++ .../legacysearch/LegacySearchService.java | 23 ++++ .../legacysearch/MetadataResultSetUtil.java | 109 ++++++++++++++++++ .../legacysearch/SearchResultRowMapper.java | 38 ++++++ .../model/CategoricalMetadata.java | 23 ++++ .../model/ContinuousMetadata.java | 24 ++++ .../legacysearch/model/LegacyResponse.java | 6 + .../legacysearch/model/LegacySearchQuery.java | 7 ++ .../legacysearch/model/Metadata.java | 4 + .../dictionary/legacysearch/model/Result.java | 12 ++ .../legacysearch/model/Results.java | 8 ++ .../legacysearch/model/SearchResult.java | 6 + .../dictionary/util/JsonBlobParser.java | 65 +++++++++++ .../dictionary/util/QueryUtility.java | 19 +++ .../resources/application-bdc-dev.properties | 14 +++ .../concept/ConceptControllerTest.java | 2 + .../concept/ConceptDecoratorServiceTest.java | 3 +- .../concept/ConceptRepositoryTest.java | 2 + .../concept/ConceptResultSetUtilTest.java | 5 +- .../ConceptServiceIntegrationTest.java | 2 + .../concept/ConceptServiceTest.java | 2 + .../dashboard/DashboardConfigTest.java | 2 + .../dashboard/DashboardControllerTest.java | 2 + .../dashboard/DashboardServiceTest.java | 2 + .../dataset/DatasetRepositoryTest.java | 2 - .../dataset/DatasetServiceTest.java | 2 + .../dictionary/facet/FacetControllerTest.java | 2 + .../dictionary/facet/FacetServiceTest.java | 2 + .../facet/FilterPreProcessorTest.java | 2 + .../dictionary/info/InfoControllerTest.java | 2 + ...LegacySearchControllerIntegrationTest.java | 57 +++++++++ .../LegacySearchQueryMapperTest.java | 50 ++++++++ .../LegacySearchRepositoryTest.java | 76 ++++++++++++ 47 files changed, 828 insertions(+), 101 deletions(-) create mode 100644 dictionaryweights/README.md create mode 100644 dictionaryweights/src/main/resources/application-bdc-dev.properties create mode 100644 dictionaryweights/weights.csv create mode 100644 dictonaryReqeust.http create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/FilterProcessor.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchController.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepository.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchService.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/MetadataResultSetUtil.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/SearchResultRowMapper.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/CategoricalMetadata.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/ContinuousMetadata.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacyResponse.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacySearchQuery.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Metadata.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Result.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Results.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/SearchResult.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java create mode 100644 src/main/resources/application-bdc-dev.properties create mode 100644 src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchControllerIntegrationTest.java create mode 100644 src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapperTest.java create mode 100644 src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepositoryTest.java diff --git a/dictionaryweights/README.md b/dictionaryweights/README.md new file mode 100644 index 0000000..39e6352 --- /dev/null +++ b/dictionaryweights/README.md @@ -0,0 +1,11 @@ +## Docker commands for local development +### Docker build +```bash +docker build --no-cache --build-arg SPRING_PROFILE=bdc-dev -t weights:latest . +``` + +### Docker run +You will need a local weights.csv file. +```bash + docker run --rm -t --name dictionary-weights --network=host -v ./weights.csv:/weights.csv weights:latest +``` \ No newline at end of file diff --git a/dictionaryweights/src/main/resources/application-bdc-dev.properties b/dictionaryweights/src/main/resources/application-bdc-dev.properties new file mode 100644 index 0000000..5b79313 --- /dev/null +++ b/dictionaryweights/src/main/resources/application-bdc-dev.properties @@ -0,0 +1,9 @@ +spring.application.name=dictionaryweights +spring.main.web-application-type=none + +spring.datasource.url=jdbc:postgresql://localhost:5432/dictionary_db?currentSchema=dict +spring.datasource.username=username +spring.datasource.password=password +spring.datasource.driver-class-name=org.postgresql.Driver + +weights.filename=/weights.csv \ No newline at end of file diff --git a/dictionaryweights/weights.csv b/dictionaryweights/weights.csv new file mode 100644 index 0000000..d3cc913 --- /dev/null +++ b/dictionaryweights/weights.csv @@ -0,0 +1,7 @@ +concept_node.DISPLAY,2 +concept_node.CONCEPT_PATH,2 +dataset.FULL_NAME,1 +dataset.DESCRIPTION,1 +parent.DISPLAY,1 +grandparent.DISPLAY,1 +concept_node_meta_str,1 \ No newline at end of file diff --git a/dictonaryReqeust.http b/dictonaryReqeust.http new file mode 100644 index 0000000..e474278 --- /dev/null +++ b/dictonaryReqeust.http @@ -0,0 +1,15 @@ +# curl 'https://dev.picsure.biodatacatalyst.nhlbi.nih.gov/picsure/proxy/dictionary-api/concepts?page_number=1&page_size=1' +# -H 'origin: https://dev.picsure.biodatacatalyst.nhlbi.nih.gov' +# -H 'referer: https://dev.picsure.biodatacatalyst.nhlbi.nih.gov/' +# --data-raw '{"facets":[],"search":"","consents":[]}' +POST http://localhost:80/concepts?page_number=0&page_size=100 +Content-Type: application/json + +{"facets":[],"search":"lipid triglyceride"} + +### + +POST http://localhost:80/search +Content-Type: application/json + +{"@type":"GeneralQueryRequest","resourceCredentials":{},"query":{"searchTerm":"breast","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":10000000},"resourceUUID":null} \ No newline at end of file diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java index 410e379..7f449bb 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java @@ -3,6 +3,7 @@ import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; import edu.harvard.dbmi.avillach.dictionary.filter.QueryParamPair; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.SearchResultRowMapper; import edu.harvard.dbmi.avillach.dictionary.util.MapExtractor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -15,24 +16,12 @@ import java.util.Map; import java.util.Optional; +import static edu.harvard.dbmi.avillach.dictionary.util.QueryUtility.ALLOW_FILTERING_Q; + + @Repository public class ConceptRepository { - private static final String ALLOW_FILTERING_Q = """ - WITH allow_filtering AS ( - SELECT - concept_node.concept_node_id AS concept_node_id, - (string_agg(concept_node_meta.value, ' ') NOT LIKE '%' || 'true' || '%') AS allowFiltering - FROM - concept_node - JOIN concept_node_meta ON - concept_node.concept_node_id = concept_node_meta.concept_node_id - AND concept_node_meta.KEY IN (:disallowed_meta_keys) - GROUP BY - concept_node.concept_node_id - ) - """; - private final NamedParameterJdbcTemplate template; private final ConceptRowMapper mapper; private final ConceptFilterQueryGenerator filterGen; @@ -40,7 +29,6 @@ AND concept_node_meta.KEY IN (:disallowed_meta_keys) private final ConceptResultSetExtractor conceptResultSetExtractor; private final List disallowedMetaFields; - @Autowired public ConceptRepository( NamedParameterJdbcTemplate template, ConceptRowMapper mapper, ConceptFilterQueryGenerator filterGen, @@ -240,4 +228,6 @@ WITH RECURSIVE nodes AS ( return Optional.ofNullable(template.query(sql, params, conceptResultSetExtractor)); } + + } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java index 9e8c220..0201691 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java @@ -2,29 +2,31 @@ import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept; import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept; -import org.json.JSONArray; -import org.json.JSONException; +import edu.harvard.dbmi.avillach.dictionary.util.JsonBlobParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.math.BigDecimal; -import java.math.BigInteger; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; @Component public class ConceptResultSetUtil { private static final Logger log = LoggerFactory.getLogger(ConceptResultSetUtil.class); + private final JsonBlobParser jsonBlobParser; + + @Autowired + public ConceptResultSetUtil(JsonBlobParser jsonBlobParser) { + this.jsonBlobParser = jsonBlobParser; + } public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException { return new CategoricalConcept( rs.getString("concept_path"), rs.getString("name"), rs.getString("display"), rs.getString("dataset"), - rs.getString("description"), rs.getString("values") == null ? List.of() : parseValues(rs.getString("values")), + rs.getString("description"), rs.getString("values") == null ? List.of() : jsonBlobParser.parseValues(rs.getString("values")), rs.getBoolean("allowFiltering"), rs.getString("studyAcronym"), null, null ); } @@ -32,53 +34,11 @@ public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException { public ContinuousConcept mapContinuous(ResultSet rs) throws SQLException { return new ContinuousConcept( rs.getString("concept_path"), rs.getString("name"), rs.getString("display"), rs.getString("dataset"), - rs.getString("description"), rs.getBoolean("allowFiltering"), parseMin(rs.getString("values")), - parseMax(rs.getString("values")), rs.getString("studyAcronym"), null + rs.getString("description"), rs.getBoolean("allowFiltering"), jsonBlobParser.parseMin(rs.getString("values")), + jsonBlobParser.parseMax(rs.getString("values")), rs.getString("studyAcronym"), null ); } - public List parseValues(String valuesArr) { - try { - ArrayList vals = new ArrayList<>(); - JSONArray arr = new JSONArray(valuesArr); - for (int i = 0; i < arr.length(); i++) { - vals.add(arr.getString(i)); - } - return vals; - } catch (JSONException ex) { - return List.of(); - } - } - public Float parseMin(String valuesArr) { - return parseFromIndex(valuesArr, 0); - } - private Float parseFromIndex(String valuesArr, int index) { - try { - JSONArray arr = new JSONArray(valuesArr); - if (arr.length() != 2) { - return 0F; - } - Object raw = arr.get(index); - return switch (raw) { - case Double d -> d.floatValue(); - case Integer i -> i.floatValue(); - case String s -> Double.valueOf(s).floatValue(); - case BigDecimal d -> d.floatValue(); - case BigInteger i -> i.floatValue(); - default -> 0f; - }; - } catch (JSONException ex) { - log.warn("Invalid json array for values: ", ex); - return 0F; - } catch (NumberFormatException ex) { - log.warn("Valid json array but invalid val within: ", ex); - return 0F; - } - } - - public Float parseMax(String valuesArr) { - return parseFromIndex(valuesArr, 1); - } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java index edd1e4d..dc35254 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java @@ -70,4 +70,5 @@ public Optional conceptTree(String dataset, String conceptPath, int dep public Optional conceptDetailWithoutAncestors(String dataset, String conceptPath) { return getConcept(dataset, conceptPath, false); } + } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java index 164b953..871c34f 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java @@ -1,7 +1,6 @@ package edu.harvard.dbmi.avillach.dictionary.concept.model; import edu.harvard.dbmi.avillach.dictionary.dataset.Dataset; -import jakarta.annotation.Nullable; import java.util.List; import java.util.Map; diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java index 8b465d3..021cb4f 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java @@ -4,7 +4,6 @@ import edu.harvard.dbmi.avillach.dictionary.dataset.Dataset; import jakarta.annotation.Nullable; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/datasource/DataSourceVerifier.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/datasource/DataSourceVerifier.java index 9e36308..e3fc2a5 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/datasource/DataSourceVerifier.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/datasource/DataSourceVerifier.java @@ -3,15 +3,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Service; import javax.sql.DataSource; import java.sql.Connection; import java.sql.SQLException; -@Service +@Profile("!test") +@Configuration public class DataSourceVerifier { private static final Logger LOG = LoggerFactory.getLogger(DataSourceVerifier.class); @@ -28,11 +30,10 @@ public void verifyDataSourceConnection() { try (Connection connection = dataSource.getConnection()) { if (connection != null) { LOG.info("Datasource connection verified successfully."); - } else { - LOG.info("Failed to obtain a connection from the datasource."); } } catch (SQLException e) { - LOG.info("Error verifying datasource connection: {}", e.getMessage()); + LOG.info("Failed to obtain a connection from the datasource."); + LOG.debug("Error verifying datasource connection: {}", e.getMessage()); } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java index 98357dc..3c6eda1 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java @@ -1,22 +1,28 @@ package edu.harvard.dbmi.avillach.dictionary.facet; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.filter.FilterProcessor; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.MethodParameter; import org.springframework.http.HttpInputMessage; import org.springframework.http.converter.HttpMessageConverter; -import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.ControllerAdvice; import org.springframework.web.servlet.mvc.method.annotation.RequestBodyAdvice; import java.io.IOException; import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.function.Function; @ControllerAdvice public class FilterPreProcessor implements RequestBodyAdvice { + + private final FilterProcessor filterProcessor; + + @Autowired + public FilterPreProcessor(FilterProcessor filterProcessor) { + this.filterProcessor = filterProcessor; + } + + @Override public boolean supports(MethodParameter methodParameter, Type targetType, Class> converterType) { return true; @@ -35,26 +41,13 @@ public Object afterBodyRead( Class> converterType ) { if (body instanceof Filter filter) { - List newFacets = filter.facets(); - List newConsents = filter.consents(); - if (filter.facets() != null) { - newFacets = new ArrayList<>(filter.facets()); - newFacets.sort(Comparator.comparing(Facet::name)); - } - if (filter.consents() != null) { - newConsents = new ArrayList<>(newConsents); - newConsents.sort(Comparator.comparing(Function.identity())); - } - filter = new Filter(newFacets, filter.search(), newConsents); - - if (StringUtils.hasLength(filter.search())) { - filter = new Filter(filter.facets(), filter.search().replaceAll("_", "/"), filter.consents()); - } - return filter; + return filterProcessor.processsFilter(filter); } return body; } + + @Override public Object handleEmptyBody( Object body, HttpInputMessage inputMessage, MethodParameter parameter, Type targetType, diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/FilterProcessor.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/FilterProcessor.java new file mode 100644 index 0000000..f2478ae --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/FilterProcessor.java @@ -0,0 +1,34 @@ +package edu.harvard.dbmi.avillach.dictionary.filter; + +import edu.harvard.dbmi.avillach.dictionary.facet.Facet; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.function.Function; + +@Component +public class FilterProcessor { + + public Filter processsFilter(Filter filter) { + List newFacets = filter.facets(); + List newConsents = filter.consents(); + if (filter.facets() != null) { + newFacets = new ArrayList<>(filter.facets()); + newFacets.sort(Comparator.comparing(Facet::name)); + } + if (filter.consents() != null) { + newConsents = new ArrayList<>(newConsents); + newConsents.sort(Comparator.comparing(Function.identity())); + } + filter = new Filter(newFacets, filter.search(), newConsents); + + if (StringUtils.hasLength(filter.search())) { + filter = new Filter(filter.facets(), filter.search().replaceAll("_", "/"), filter.consents()); + } + return filter; + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchController.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchController.java new file mode 100644 index 0000000..c412200 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchController.java @@ -0,0 +1,32 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacyResponse; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacySearchQuery; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; + +import java.io.IOException; + +@Controller +public class LegacySearchController { + + private final LegacySearchService legacySearchService; + private final LegacySearchQueryMapper legacySearchQueryMapper; + + @Autowired + public LegacySearchController(LegacySearchService legacySearchService, LegacySearchQueryMapper legacySearchQueryMapper) { + this.legacySearchService = legacySearchService; + this.legacySearchQueryMapper = legacySearchQueryMapper; + } + + @RequestMapping(path = "/search") + public ResponseEntity legacySearch(@RequestBody String jsonString) throws IOException { + LegacySearchQuery legacySearchQuery = legacySearchQueryMapper.mapFromJson(jsonString); + return ResponseEntity + .ok(new LegacyResponse(legacySearchService.getSearchResults(legacySearchQuery.filter(), legacySearchQuery.pageable()))); + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java new file mode 100644 index 0000000..001436c --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java @@ -0,0 +1,34 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.filter.FilterProcessor; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacySearchQuery; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.List; + +@Component +public class LegacySearchQueryMapper { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + private final FilterProcessor filterProcessor; + + public LegacySearchQueryMapper(FilterProcessor filterProcessor) { + this.filterProcessor = filterProcessor; + } + + public LegacySearchQuery mapFromJson(String jsonString) throws IOException { + JsonNode rootNode = objectMapper.readTree(jsonString); + JsonNode queryNode = rootNode.get("query"); + + String searchTerm = queryNode.get("searchTerm").asText(); + int limit = queryNode.get("limit").asInt(); + Filter filter = filterProcessor.processsFilter(new Filter(List.of(), searchTerm, List.of())); + return new LegacySearchQuery(filter, PageRequest.of(0, limit)); + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepository.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepository.java new file mode 100644 index 0000000..9ea09c4 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepository.java @@ -0,0 +1,79 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.concept.ConceptFilterQueryGenerator; +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.filter.QueryParamPair; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.util.List; + +import static edu.harvard.dbmi.avillach.dictionary.util.QueryUtility.ALLOW_FILTERING_Q; + +@Repository +public class LegacySearchRepository { + + private final ConceptFilterQueryGenerator filterGen; + private final NamedParameterJdbcTemplate template; + private final List disallowedMetaFields; + private final SearchResultRowMapper searchResultRowMapper; + + @Autowired + public LegacySearchRepository( + ConceptFilterQueryGenerator filterGen, NamedParameterJdbcTemplate template, + @Value("${filtering.unfilterable_concepts}") List disallowedMetaFields, SearchResultRowMapper searchResultRowMapper + ) { + this.filterGen = filterGen; + this.template = template; + this.disallowedMetaFields = disallowedMetaFields; + this.searchResultRowMapper = searchResultRowMapper; + } + + public List getLegacySearchResults(Filter filter, Pageable pageable) { + QueryParamPair filterQ = filterGen.generateFilterQuery(filter, pageable); + String sql = ALLOW_FILTERING_Q + ", " + filterQ.query() + """ + SELECT concept_node.concept_path AS conceptPath, + concept_node.display AS display, + concept_node.name AS name, + concept_node.concept_type AS conceptType, + ds.REF as dataset, + ds.abbreviation AS studyAcronym, + ds.full_name as dsFullName, + continuous_min.VALUE as min, + continuous_max.VALUE as max, + categorical_values.VALUE as values, + allow_filtering.allowFiltering AS allowFiltering, + meta_description.VALUE AS description, + stigmatized.value AS stigmatized, + parent.name AS parentName, + parent.display AS parentDisplay + FROM concept_node + INNER JOIN concepts_filtered_sorted ON concepts_filtered_sorted.concept_node_id = concept_node.concept_node_id + LEFT JOIN dataset AS ds ON concept_node.dataset_id = ds.dataset_id + LEFT JOIN concept_node_meta AS meta_description + ON concept_node.concept_node_id = meta_description.concept_node_id AND + meta_description.KEY = 'description' + LEFT JOIN concept_node_meta AS continuous_min + ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min' + LEFT JOIN concept_node_meta AS continuous_max + ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max' + LEFT JOIN concept_node_meta AS categorical_values + ON concept_node.concept_node_id = categorical_values.concept_node_id AND + categorical_values.KEY = 'values' + LEFT JOIN concept_node_meta AS stigmatized ON concept_node.concept_node_id = stigmatized.concept_node_id AND + stigmatized.KEY = 'stigmatized' + LEFT JOIN concept_node AS parent ON parent.concept_node_id = concept_node.parent_id + LEFT JOIN allow_filtering ON concept_node.concept_node_id = allow_filtering.concept_node_id + ORDER BY concepts_filtered_sorted.rank DESC, concept_node.concept_node_id ASC + """; + MapSqlParameterSource params = filterQ.params().addValue("disallowed_meta_keys", disallowedMetaFields); + + return template.query(sql, params, searchResultRowMapper); + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchService.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchService.java new file mode 100644 index 0000000..e7462c2 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchService.java @@ -0,0 +1,23 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.Results; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; + +@Service +public class LegacySearchService { + + private final LegacySearchRepository legacySearchRepository; + + @Autowired + public LegacySearchService(LegacySearchRepository legacySearchRepository) { + this.legacySearchRepository = legacySearchRepository; + } + + public Results getSearchResults(Filter filter, Pageable pageable) { + return new Results(legacySearchRepository.getLegacySearchResults(filter, pageable)); + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/MetadataResultSetUtil.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/MetadataResultSetUtil.java new file mode 100644 index 0000000..bc8eea2 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/MetadataResultSetUtil.java @@ -0,0 +1,109 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.CategoricalMetadata; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.ContinuousMetadata; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.Result; +import edu.harvard.dbmi.avillach.dictionary.util.JsonBlobParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.ResultSet; +import java.sql.SQLException; + + +@Component +public class MetadataResultSetUtil { + + private final static Logger log = LoggerFactory.getLogger(MetadataResultSetUtil.class); + private final JsonBlobParser jsonBlobParser; + + @Autowired + public MetadataResultSetUtil(JsonBlobParser jsonBlobParser) { + this.jsonBlobParser = jsonBlobParser; + } + + public Result mapContinuousMetadata(ResultSet rs) throws SQLException { + String hashedVarId = hashVarId(rs.getString("conceptPath")); + String description = getDescription(rs); + String parentName = getParentName(rs); + String parentDisplay = getParentDisplay(rs); + + String max = String.valueOf(jsonBlobParser.parseMax(rs.getString("values"))); + String min = String.valueOf(jsonBlobParser.parseMin(rs.getString("values"))); + + ContinuousMetadata metadata = new ContinuousMetadata( + rs.getString("stigmatized"), rs.getString("display"), description, min, rs.getString("conceptPath"), parentName, + rs.getString("conceptPath"), rs.getString("name"), parentDisplay, description, // changed + "{}", "", parentName, max, description, rs.getString("dataset"), hashedVarId, rs.getString("conceptType"), rs.getString("name"), + rs.getString("dataset"), rs.getString("stigmatized"), rs.getString("display"), rs.getString("studyAcronym"), + rs.getString("dsFullName"), parentName, parentDisplay, rs.getString("conceptPath"), min, max + ); + return new Result( + metadata, jsonBlobParser.parseValues(rs.getString("values")), rs.getString("dataset"), parentName, rs.getString("name"), false, + true + ); + } + + public Result mapCategoricalMetadata(ResultSet rs) throws SQLException { + String hashedVarId = hashVarId(rs.getString("conceptPath")); + String description = getDescription(rs); + String parentName = getParentName(rs); + String parentDisplay = getParentDisplay(rs); + + CategoricalMetadata metadata = new CategoricalMetadata( + rs.getString("stigmatized"), rs.getString("display"), description, "", rs.getString("conceptPath"), parentName, + rs.getString("conceptPath"), rs.getString("name"), parentDisplay, description, // changed + "{}", "", parentName, "", description, rs.getString("dataset"), hashedVarId, rs.getString("conceptType"), rs.getString("name"), + rs.getString("dataset"), rs.getString("stigmatized"), rs.getString("display"), rs.getString("studyAcronym"), + rs.getString("dsFullName"), parentName, parentDisplay, rs.getString("conceptPath") + ); + + return new Result( + metadata, jsonBlobParser.parseValues(rs.getString("values")), rs.getString("dataset"), parentName, rs.getString("name"), true, + false + ); + } + + private static String hashVarId(String hpdsPath) { + String hashedVarId = ""; + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] encodedHash = digest.digest(hpdsPath.getBytes(StandardCharsets.UTF_8)); + hashedVarId = bytesToHex(encodedHash); + } catch (NoSuchAlgorithmException e) { + log.error(e.getMessage()); + } + + return hashedVarId; + } + + private static String bytesToHex(byte[] hash) { + StringBuilder hexString = new StringBuilder(2 * hash.length); + for (byte b : hash) { + String hex = Integer.toHexString(0xff & b); + if (hex.length() == 1) { + hexString.append('0'); + } + hexString.append(hex); + } + return hexString.toString(); + } + + private String getParentDisplay(ResultSet rs) throws SQLException { + return StringUtils.hasLength("parentDisplay") ? "" : rs.getString("parentDisplay"); + } + + private String getParentName(ResultSet rs) throws SQLException { + return StringUtils.hasLength(rs.getString("parentName")) ? "All Variables" : rs.getString("parentName"); + } + + private String getDescription(ResultSet rs) throws SQLException { + return StringUtils.hasLength(rs.getString("description")) ? "" : rs.getString("description"); + } +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/SearchResultRowMapper.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/SearchResultRowMapper.java new file mode 100644 index 0000000..cb51309 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/SearchResultRowMapper.java @@ -0,0 +1,38 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptType; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.Result; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Component; + +import java.sql.ResultSet; +import java.sql.SQLException; + +@Component +public class SearchResultRowMapper implements RowMapper { + + private final MetadataResultSetUtil metadataResultSetUtil; + + @Autowired + public SearchResultRowMapper(MetadataResultSetUtil metadataResultSetUtil) { + this.metadataResultSetUtil = metadataResultSetUtil; + } + + @Override + public SearchResult mapRow(ResultSet rs, int rowNum) throws SQLException { + return mapSearchResults(rs); + } + + private SearchResult mapSearchResults(ResultSet rs) throws SQLException { + Result result = switch (ConceptType.toConcept(rs.getString("conceptType"))) { + case Categorical -> this.metadataResultSetUtil.mapCategoricalMetadata(rs); + case Continuous -> this.metadataResultSetUtil.mapContinuousMetadata(rs); + }; + + return new SearchResult(result); + } + + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/CategoricalMetadata.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/CategoricalMetadata.java new file mode 100644 index 0000000..79de460 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/CategoricalMetadata.java @@ -0,0 +1,23 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record CategoricalMetadata( + @JsonProperty("columnmeta_is_stigmatized") String columnmetaIsStigmatized, @JsonProperty("columnmeta_name") String columnmetaName, + @JsonProperty("description") String description, @JsonProperty("columnmeta_min") String columnmetaMin, + @JsonProperty("HPDS_PATH") String hpdsPath, @JsonProperty("derived_group_id") String derivedGroupId, + @JsonProperty("columnmeta_hpds_path") String columnmetaHpdsPath, @JsonProperty("columnmeta_var_id") String columnmetaVarId, + @JsonProperty("columnmeta_var_group_description") String columnmetaVarGroupDescription, + @JsonProperty("derived_var_description") String derivedVarDescription, + @JsonProperty("derived_variable_level_data") String derivedVariableLevelData, @JsonProperty("data_hierarchy") String dataHierarchy, + @JsonProperty("derived_group_description") String derivedGroupDescription, @JsonProperty("columnmeta_max") String columnmetaMax, + @JsonProperty("columnmeta_description") String columnmetaDescription, @JsonProperty("derived_study_id") String derivedStudyId, + @JsonProperty("hashed_var_id") String hashedVarId, @JsonProperty("columnmeta_data_type") String columnmetaDataType, + @JsonProperty("derived_var_id") String derivedVarId, @JsonProperty("columnmeta_study_id") String columnmetaStudyId, + @JsonProperty("is_stigmatized") String isStigmatized, @JsonProperty("derived_var_name") String derivedVarName, + @JsonProperty("derived_study_abv_name") String derivedStudyAbvName, + @JsonProperty("derived_study_description") String derivedStudyDescription, + @JsonProperty("columnmeta_var_group_id") String columnmetaVarGroupId, @JsonProperty("derived_group_name") String derivedGroupName, + @JsonProperty("columnmeta_HPDS_PATH") String columnmetaHpdsPathAlternate +) implements Metadata { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/ContinuousMetadata.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/ContinuousMetadata.java new file mode 100644 index 0000000..a067194 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/ContinuousMetadata.java @@ -0,0 +1,24 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record ContinuousMetadata( + @JsonProperty("columnmeta_is_stigmatized") String columnmetaIsStigmatized, @JsonProperty("columnmeta_name") String columnmetaName, + @JsonProperty("description") String description, @JsonProperty("columnmeta_min") String columnmetaMin, + @JsonProperty("HPDS_PATH") String hpdsPath, @JsonProperty("derived_group_id") String derivedGroupId, + @JsonProperty("columnmeta_hpds_path") String columnmetaHpdsPath, @JsonProperty("columnmeta_var_id") String columnmetaVarId, + @JsonProperty("columnmeta_var_group_description") String columnmetaVarGroupDescription, + @JsonProperty("derived_var_description") String derivedVarDescription, + @JsonProperty("derived_variable_level_data") String derivedVariableLevelData, @JsonProperty("data_hierarchy") String dataHierarchy, + @JsonProperty("derived_group_description") String derivedGroupDescription, @JsonProperty("columnmeta_max") String columnmetaMax, + @JsonProperty("columnmeta_description") String columnmetaDescription, @JsonProperty("derived_study_id") String derivedStudyId, + @JsonProperty("hashed_var_id") String hashedVarId, @JsonProperty("columnmeta_data_type") String columnmetaDataType, + @JsonProperty("derived_var_id") String derivedVarId, @JsonProperty("columnmeta_study_id") String columnmetaStudyId, + @JsonProperty("is_stigmatized") String isStigmatized, @JsonProperty("derived_var_name") String derivedVarName, + @JsonProperty("derived_study_abv_name") String derivedStudyAbvName, + @JsonProperty("derived_study_description") String derivedStudyDescription, + @JsonProperty("columnmeta_var_group_id") String columnmetaVarGroupId, @JsonProperty("derived_group_name") String derivedGroupName, + @JsonProperty("columnmeta_HPDS_PATH") String columnmetaHpdsPathAlternate, @JsonProperty("min") String min, + @JsonProperty("max") String max +) implements Metadata { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacyResponse.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacyResponse.java new file mode 100644 index 0000000..7655c6c --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacyResponse.java @@ -0,0 +1,6 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record LegacyResponse(@JsonProperty("results") Results results) { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacySearchQuery.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacySearchQuery.java new file mode 100644 index 0000000..1147a85 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacySearchQuery.java @@ -0,0 +1,7 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import org.springframework.data.domain.Pageable; + +public record LegacySearchQuery(Filter filter, Pageable pageable) { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Metadata.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Metadata.java new file mode 100644 index 0000000..d3d40d4 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Metadata.java @@ -0,0 +1,4 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +public sealed interface Metadata permits ContinuousMetadata, CategoricalMetadata { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Result.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Result.java new file mode 100644 index 0000000..ba3a51d --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Result.java @@ -0,0 +1,12 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +public record Result( + Metadata metadata, List values, @JsonProperty("studyId") String studyId, @JsonProperty("dtId") String dtId, + @JsonProperty("varId") String varId, @JsonProperty("is_categorical") boolean isCategorical, + @JsonProperty("is_continuous") boolean isContinuous +) { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Results.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Results.java new file mode 100644 index 0000000..fa37331 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Results.java @@ -0,0 +1,8 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +public record Results(@JsonProperty("searchResults") List searchResults) { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/SearchResult.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/SearchResult.java new file mode 100644 index 0000000..d6a7f53 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/SearchResult.java @@ -0,0 +1,6 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record SearchResult(@JsonProperty("result") Result result) { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java new file mode 100644 index 0000000..f976e7c --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java @@ -0,0 +1,65 @@ +package edu.harvard.dbmi.avillach.dictionary.util; + + +import org.json.JSONArray; +import org.json.JSONException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +@Component +public class JsonBlobParser { + + private final static Logger log = LoggerFactory.getLogger(JsonBlobParser.class); + + public List parseValues(String valuesArr) { + try { + ArrayList vals = new ArrayList<>(); + JSONArray arr = new JSONArray(valuesArr); + for (int i = 0; i < arr.length(); i++) { + vals.add(arr.getString(i)); + } + return vals; + } catch (JSONException ex) { + return List.of(); + } + } + + public Float parseMin(String valuesArr) { + return parseFromIndex(valuesArr, 0); + } + + private Float parseFromIndex(String valuesArr, int index) { + try { + JSONArray arr = new JSONArray(valuesArr); + if (arr.length() != 2) { + return 0F; + } + Object raw = arr.get(index); + return switch (raw) { + case Double d -> d.floatValue(); + case Integer i -> i.floatValue(); + case String s -> Double.valueOf(s).floatValue(); + case BigDecimal d -> d.floatValue(); + case BigInteger i -> i.floatValue(); + default -> 0f; + }; + } catch (JSONException ex) { + log.warn("Invalid json array for values: ", ex); + return 0F; + } catch (NumberFormatException ex) { + log.warn("Valid json array but invalid val within: ", ex); + return 0F; + } + } + + public Float parseMax(String valuesArr) { + return parseFromIndex(valuesArr, 1); + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java new file mode 100644 index 0000000..69eedc4 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java @@ -0,0 +1,19 @@ +package edu.harvard.dbmi.avillach.dictionary.util; + +public class QueryUtility { + + public static final String ALLOW_FILTERING_Q = """ + WITH allow_filtering AS ( + SELECT + concept_node.concept_node_id AS concept_node_id, + (string_agg(concept_node_meta.value, ' ') NOT LIKE '%' || 'true' || '%') AS allowFiltering + FROM + concept_node + JOIN concept_node_meta ON + concept_node.concept_node_id = concept_node_meta.concept_node_id + AND concept_node_meta.KEY IN (:disallowed_meta_keys) + GROUP BY + concept_node.concept_node_id + ) + """; +} diff --git a/src/main/resources/application-bdc-dev.properties b/src/main/resources/application-bdc-dev.properties new file mode 100644 index 0000000..219dfbb --- /dev/null +++ b/src/main/resources/application-bdc-dev.properties @@ -0,0 +1,14 @@ +spring.application.name=dictionary +spring.datasource.url=jdbc:postgresql://localhost:5432/dictionary_db?currentSchema=dict +spring.datasource.username=username +spring.datasource.password=password +spring.datasource.driver-class-name=org.postgresql.Driver +server.port=80 + +dashboard.columns={abbreviation:'Abbreviation',name:'Name',clinvars:'Clinical Variables'} +dashboard.column-order=abbreviation,name,clinvars +dashboard.nonmeta-columns=abbreviation,name +dashboard.enable.extra_details=true +dashboard.enable.bdc_hack=true + +filtering.unfilterable_concepts=stigmatized \ No newline at end of file diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java index a5d5fe6..363c180 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java @@ -16,12 +16,14 @@ import org.springframework.data.domain.Pageable; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.Map; import java.util.Optional; @SpringBootTest(properties = {"concept.tree.max_depth=1"}) +@ActiveProfiles("test") class ConceptControllerTest { @MockBean diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptDecoratorServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptDecoratorServiceTest.java index 6800f57..c543f27 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptDecoratorServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptDecoratorServiceTest.java @@ -10,12 +10,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; -import javax.print.attribute.DocAttributeSet; import java.util.Optional; @SpringBootTest +@ActiveProfiles("test") class ConceptDecoratorServiceTest { @MockBean diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java index f371012..8d20516 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java @@ -323,4 +323,6 @@ void shouldGetContConceptWithDecimalNotation() { Assertions.assertEquals(0.57f, concept.min()); Assertions.assertEquals(6.77f, concept.max()); } + + } diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtilTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtilTest.java index 3b1da9e..acd6668 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtilTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtilTest.java @@ -1,17 +1,16 @@ package edu.harvard.dbmi.avillach.dictionary.concept; +import edu.harvard.dbmi.avillach.dictionary.util.JsonBlobParser; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; - class ConceptResultSetUtilTest { @Test void shouldParseValues() { - List actual = new ConceptResultSetUtil().parseValues("[\"Look, I'm valid json\"]"); + List actual = new JsonBlobParser().parseValues("[\"Look, I'm valid json\"]"); List expected = List.of("Look, I'm valid json"); Assertions.assertEquals(expected, actual); diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceIntegrationTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceIntegrationTest.java index e51ac7b..5bef7ba 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceIntegrationTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceIntegrationTest.java @@ -8,6 +8,7 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; import org.testcontainers.containers.PostgreSQLContainer; @@ -21,6 +22,7 @@ @Testcontainers @SpringBootTest +@ActiveProfiles("test") class ConceptServiceIntegrationTest { @Autowired diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java index 0238e0c..610ba94 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java @@ -13,6 +13,7 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.data.domain.Pageable; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.Map; @@ -20,6 +21,7 @@ @SpringBootTest +@ActiveProfiles("test") class ConceptServiceTest { @MockBean diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardConfigTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardConfigTest.java index 397a055..c5e280f 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardConfigTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardConfigTest.java @@ -4,10 +4,12 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; import java.util.List; @SpringBootTest +@ActiveProfiles("test") class DashboardConfigTest { @Autowired diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardControllerTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardControllerTest.java index ec11139..116e284 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardControllerTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardControllerTest.java @@ -8,10 +8,12 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; import java.util.List; @SpringBootTest +@ActiveProfiles("test") class DashboardControllerTest { @MockBean private DashboardService service; diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardServiceTest.java index 1825e21..03255bd 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardServiceTest.java @@ -6,11 +6,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.Map; @SpringBootTest +@ActiveProfiles("test") class DashboardServiceTest { @MockBean DashboardRepository repository; diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetRepositoryTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetRepositoryTest.java index 036a36d..03f169d 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetRepositoryTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetRepositoryTest.java @@ -1,6 +1,5 @@ package edu.harvard.dbmi.avillach.dictionary.dataset; -import edu.harvard.dbmi.avillach.dictionary.facet.FacetRepository; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -15,7 +14,6 @@ import java.util.Map; import java.util.Optional; -import static org.junit.jupiter.api.Assertions.*; @Testcontainers @SpringBootTest diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetServiceTest.java index 22b8f12..8580060 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetServiceTest.java @@ -6,11 +6,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; import java.util.Map; import java.util.Optional; @SpringBootTest +@ActiveProfiles("test") class DatasetServiceTest { @MockBean diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetControllerTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetControllerTest.java index 331f176..96ea6eb 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetControllerTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetControllerTest.java @@ -9,11 +9,13 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.Optional; @SpringBootTest +@ActiveProfiles("test") class FacetControllerTest { @MockBean diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetServiceTest.java index e9c3aab..3b8775f 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetServiceTest.java @@ -8,12 +8,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.Map; import java.util.Optional; @SpringBootTest +@ActiveProfiles("test") class FacetServiceTest { @MockBean private FacetRepository repository; diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessorTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessorTest.java index 6d3d452..70e90a5 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessorTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessorTest.java @@ -8,11 +8,13 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.core.MethodParameter; import org.springframework.http.HttpInputMessage; +import org.springframework.test.context.ActiveProfiles; import org.testcontainers.shaded.com.fasterxml.jackson.databind.type.SimpleType; import java.util.List; @SpringBootTest +@ActiveProfiles("test") class FilterPreProcessorTest { @Autowired diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/info/InfoControllerTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/info/InfoControllerTest.java index d401db4..53345a1 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/info/InfoControllerTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/info/InfoControllerTest.java @@ -6,12 +6,14 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.UUID; @SpringBootTest +@ActiveProfiles("test") class InfoControllerTest { @Autowired diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchControllerIntegrationTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchControllerIntegrationTest.java new file mode 100644 index 0000000..d68a4f4 --- /dev/null +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchControllerIntegrationTest.java @@ -0,0 +1,57 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacyResponse; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.Results; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.MountableFile; + +import java.io.IOException; +import java.util.List; + +@SpringBootTest +@Testcontainers +class LegacySearchControllerIntegrationTest { + + @Autowired + LegacySearchController legacySearchController; + + @Container + static final PostgreSQLContainer databaseContainer = new PostgreSQLContainer<>("postgres:16").withReuse(true) + .withCopyFileToContainer(MountableFile.forClasspathResource("seed.sql"), "/docker-entrypoint-initdb.d/seed.sql"); + + @DynamicPropertySource + static void mySQLProperties(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", databaseContainer::getJdbcUrl); + registry.add("spring.datasource.username", databaseContainer::getUsername); + registry.add("spring.datasource.password", databaseContainer::getPassword); + registry.add("spring.datasource.db", databaseContainer::getDatabaseName); + } + + @Test + void shouldGetLegacyResponseByStudyID() throws IOException { + String jsonString = """ + {"query":{"searchTerm":"phs000007","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":100}} + """; + + ResponseEntity legacyResponseResponseEntity = legacySearchController.legacySearch(jsonString); + System.out.println(legacyResponseResponseEntity); + Assertions.assertEquals(HttpStatus.OK, legacyResponseResponseEntity.getStatusCode()); + LegacyResponse legacyResponseBody = legacyResponseResponseEntity.getBody(); + Assertions.assertNotNull(legacyResponseBody); + Results results = legacyResponseBody.results(); + List searchResults = results.searchResults(); + searchResults.forEach(searchResult -> Assertions.assertEquals("phs000007", searchResult.result().studyId())); + } + +} diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapperTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapperTest.java new file mode 100644 index 0000000..e0661ab --- /dev/null +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapperTest.java @@ -0,0 +1,50 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacySearchQuery; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.domain.Pageable; +import org.springframework.test.context.ActiveProfiles; + +import java.io.IOException; + +@SpringBootTest +@ActiveProfiles("test") +class LegacySearchQueryMapperTest { + + @Autowired + LegacySearchQueryMapper legacySearchQueryMapper; + + @Test + void shouldParseSearchRequest() throws IOException { + String jsonString = """ + {"query":{"searchTerm":"age","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":100}} + """; + + LegacySearchQuery legacySearchQuery = legacySearchQueryMapper.mapFromJson(jsonString); + Filter filter = legacySearchQuery.filter(); + Pageable pageable = legacySearchQuery.pageable(); + + Assertions.assertEquals("age", filter.search()); + Assertions.assertEquals(100, pageable.getPageSize()); + } + + @Test + void shouldReplaceUnderscore() throws IOException { + String jsonString = + """ + {"query":{"searchTerm":"tutorial-biolincc_digitalis","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":100}} + """; + + LegacySearchQuery legacySearchQuery = legacySearchQueryMapper.mapFromJson(jsonString); + Filter filter = legacySearchQuery.filter(); + Pageable pageable = legacySearchQuery.pageable(); + + Assertions.assertEquals("tutorial-biolincc/digitalis", filter.search()); + Assertions.assertEquals(100, pageable.getPageSize()); + } + +} diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepositoryTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepositoryTest.java new file mode 100644 index 0000000..9bb6bab --- /dev/null +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepositoryTest.java @@ -0,0 +1,76 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.concept.ConceptRepository; +import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept; +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.domain.Pageable; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.MountableFile; + +import java.util.List; + +@SpringBootTest +@Testcontainers +public class LegacySearchRepositoryTest { + + @Autowired + LegacySearchRepository subject; + + @Autowired + ConceptRepository conceptService; + + @Container + static final PostgreSQLContainer databaseContainer = new PostgreSQLContainer<>("postgres:16").withReuse(true) + .withCopyFileToContainer(MountableFile.forClasspathResource("seed.sql"), "/docker-entrypoint-initdb.d/seed.sql"); + + @DynamicPropertySource + static void mySQLProperties(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", databaseContainer::getJdbcUrl); + registry.add("spring.datasource.username", databaseContainer::getUsername); + registry.add("spring.datasource.password", databaseContainer::getPassword); + registry.add("spring.datasource.db", databaseContainer::getDatabaseName); + } + + @Test + void shouldGetLegacySearchResults() { + List searchResults = subject.getLegacySearchResults(new Filter(List.of(), "", List.of()), Pageable.unpaged()); + + Assertions.assertEquals(30, searchResults.size()); + } + + @Test + void shouldGetLegacySearchResultsBySearch() { + List searchResults = + subject.getLegacySearchResults(new Filter(List.of(), "phs000007", List.of()), Pageable.unpaged()); + + searchResults.forEach(searchResult -> Assertions.assertEquals("phs000007", searchResult.result().studyId())); + + } + + @Test + void shouldGetLegacySearchResultsByPageSize() { + List searchResults = subject.getLegacySearchResults(new Filter(List.of(), "", List.of()), Pageable.ofSize(5)); + + Assertions.assertEquals(5, searchResults.size()); + } + + @Test + void legacySearchResultShouldGetEqualCountToConceptSearch() { + // This test will ensure modifications made to the conceptSearch will be reflected in the legacy search result. + // They use near equivalent queries and updates made to one should be made to the other. + List searchResults = subject.getLegacySearchResults(new Filter(List.of(), "", List.of()), Pageable.unpaged()); + List concepts = conceptService.getConcepts(new Filter(List.of(), "", List.of()), Pageable.unpaged()); + + Assertions.assertEquals(searchResults.size(), concepts.size()); + } + +} From 5cf4e510890ba69a27d0b2cc26e2332a211cab7a Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Tue, 19 Nov 2024 17:06:50 -0500 Subject: [PATCH 2/2] Refactor legacy search query processing (#57) * Refactor legacy search query processing Refactor LegacySearchQueryMapper to handle logical OR and punctuation in search terms, removing dependency on FilterProcessor. Update ConceptFilterQueryGenerator with generateLegacyFilterQuery and simplify query construction. Adjust LegacySearchRepository to align with new query generation logic. * Add test for handling OR logic in legacy search This commit introduces a new test, `shouldHandleORRequest`, to verify the OR logic within the legacy search functionality. It ensures that the query with an OR condition is processed correctly, and the results are returned as expected without errors. * Refactor search term handling in legacy search. Refactored formatting for compactness in legacySearch method. Added logging in LegacySearchQueryMapper to debug search term construction. Enhanced integration test to verify expanded search results with OR queries. --- dictonaryReqeust.http | 3 +- .../concept/ConceptFilterQueryGenerator.java | 106 +++++++++++++----- .../legacysearch/LegacySearchQueryMapper.java | 41 +++++-- .../legacysearch/LegacySearchRepository.java | 2 +- ...LegacySearchControllerIntegrationTest.java | 34 ++++++ .../LegacySearchQueryMapperTest.java | 34 +++++- 6 files changed, 177 insertions(+), 43 deletions(-) diff --git a/dictonaryReqeust.http b/dictonaryReqeust.http index e474278..ae53188 100644 --- a/dictonaryReqeust.http +++ b/dictonaryReqeust.http @@ -12,4 +12,5 @@ Content-Type: application/json POST http://localhost:80/search Content-Type: application/json -{"@type":"GeneralQueryRequest","resourceCredentials":{},"query":{"searchTerm":"breast","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":10000000},"resourceUUID":null} \ No newline at end of file +{"@type":"GeneralQueryRequest","resourceCredentials":{},"query":{"searchTerm":"throat sore acute #8","includedTags":[], + "excludedTags":[],"returnTags":"true","offset":0,"limit":100000},"resourceUUID":null} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java index 5388c40..fd35437 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java @@ -82,37 +82,7 @@ public QueryParamPair generateFilterQuery(Filter filter, Pageable pageable) { params.addValue("consents", filter.consents()); } clauses.add(createValuelessNodeFilter(filter.search(), filter.consents())); - - - String query = "(\n" + String.join("\n\tINTERSECT\n", clauses) + "\n)"; - String superQuery = """ - WITH q AS ( - %s - ) - %s - SELECT q.concept_node_id AS concept_node_id, max((1 + rank) * coalesce(rank_adjustment, 1)) AS rank - FROM q - LEFT JOIN allow_filtering ON allow_filtering.concept_node_id = q.concept_node_id - GROUP BY q.concept_node_id - ORDER BY max((1 + rank) * coalesce(rank_adjustment, 1)) DESC, q.concept_node_id ASC - """.formatted(query, RANK_ADJUSTMENTS); - // explanation of ORDER BY max((1 + rank) * coalesce(rank_adjustment, 1)) DESC - // you want to sort the best matches first, BUT anything that is marked as unfilterable should be put last - // coalesce will return the first non null value; this solves rows that aren't marked as filterable or not - // I then multiply that by 1 + rank instead of just rank so that a rank value of 0 for an unfilterable var - // is placed below a rank value of 0 for a filterable var - // Finally, I add the concept node id to the sort to keep it stable for ties, otherwise pagination gets weird - - if (pageable.isPaged()) { - superQuery = superQuery + """ - LIMIT :limit - OFFSET :offset - """; - params.addValue("limit", pageable.getPageSize()).addValue("offset", pageable.getOffset()); - } - - superQuery = " concepts_filtered_sorted AS (\n" + superQuery + "\n)"; - + String superQuery = getSuperQuery(pageable, clauses, params); return new QueryParamPair(superQuery, params); } @@ -184,4 +154,78 @@ facet.name IN (:facets_for_category_%s ) AND facet_category.name = :category_%s }).toList(); } + public QueryParamPair generateLegacyFilterQuery(Filter filter, Pageable pageable) { + MapSqlParameterSource params = new MapSqlParameterSource(); + params.addValue("disallowed_meta_keys", disallowedMetaFields); + List clauses = new java.util.ArrayList<>(List.of()); + if (StringUtils.hasLength(filter.search())) { + params.addValue("dynamic_tsquery", filter.search().trim()); + } + clauses.add(createDynamicValuelessNodeFilter(filter.search())); + String superQuery = getSuperQuery(pageable, clauses, params); + + return new QueryParamPair(superQuery, params); + } + + private String createDynamicValuelessNodeFilter(String search) { + String rankQuery = "0 as rank"; + String rankWhere = ""; + if (StringUtils.hasLength(search)) { + rankQuery = "ts_rank(searchable_fields, to_tsquery(:dynamic_tsquery)) AS rank"; + rankWhere = "concept_node.searchable_fields @@ to_tsquery(:dynamic_tsquery) AND"; + } + return """ + SELECT + concept_node.concept_node_id, + %s + FROM + concept_node + LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id + LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min' + LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max' + LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values' + WHERE + %s + %s + ( + continuous_min.value <> '' OR + continuous_max.value <> '' OR + categorical_values.value <> '' + ) + """ + .formatted(rankQuery, rankWhere, ""); + } + + private static String getSuperQuery(Pageable pageable, List clauses, MapSqlParameterSource params) { + String query = "(\n" + String.join("\n\tINTERSECT\n", clauses) + "\n)"; + String superQuery = """ + WITH q AS ( + %s + ) + %s + SELECT q.concept_node_id AS concept_node_id, max((1 + rank) * coalesce(rank_adjustment, 1)) AS rank + FROM q + LEFT JOIN allow_filtering ON allow_filtering.concept_node_id = q.concept_node_id + GROUP BY q.concept_node_id + ORDER BY max((1 + rank) * coalesce(rank_adjustment, 1)) DESC, q.concept_node_id ASC + """.formatted(query, RANK_ADJUSTMENTS); + // explanation of ORDER BY max((1 + rank) * coalesce(rank_adjustment, 1)) DESC + // you want to sort the best matches first, BUT anything that is marked as unfilterable should be put last + // coalesce will return the first non null value; this solves rows that aren't marked as filterable or not + // I then multiply that by 1 + rank instead of just rank so that a rank value of 0 for an unfilterable var + // is placed below a rank value of 0 for a filterable var + // Finally, I add the concept node id to the sort to keep it stable for ties, otherwise pagination gets weird + + if (pageable.isPaged()) { + superQuery = superQuery + """ + LIMIT :limit + OFFSET :offset + """; + params.addValue("limit", pageable.getPageSize()).addValue("offset", pageable.getOffset()); + } + + superQuery = " concepts_filtered_sorted AS (\n" + superQuery + "\n)"; + return superQuery; + } + } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java index 001436c..8a23d3c 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java @@ -3,23 +3,25 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; -import edu.harvard.dbmi.avillach.dictionary.filter.FilterProcessor; import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacySearchQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Component; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; @Component public class LegacySearchQueryMapper { private static final ObjectMapper objectMapper = new ObjectMapper(); - private final FilterProcessor filterProcessor; + private static final Logger log = LoggerFactory.getLogger(LegacySearchQueryMapper.class); - public LegacySearchQueryMapper(FilterProcessor filterProcessor) { - this.filterProcessor = filterProcessor; - } + public LegacySearchQueryMapper() {} public LegacySearchQuery mapFromJson(String jsonString) throws IOException { JsonNode rootNode = objectMapper.readTree(jsonString); @@ -27,8 +29,33 @@ public LegacySearchQuery mapFromJson(String jsonString) throws IOException { String searchTerm = queryNode.get("searchTerm").asText(); int limit = queryNode.get("limit").asInt(); - Filter filter = filterProcessor.processsFilter(new Filter(List.of(), searchTerm, List.of())); - return new LegacySearchQuery(filter, PageRequest.of(0, limit)); + searchTerm = constructTsQuery(searchTerm); + log.debug("Constructed Search Term: {}", searchTerm); + return new LegacySearchQuery(new Filter(List.of(), searchTerm, List.of()), PageRequest.of(0, limit)); + } + + // An attempt to provide OR search that will produce similar results to legacy search-prototype + private String constructTsQuery(String searchTerm) { + // Split on the | to enable or queries + String[] orGroups = searchTerm.split("\\|"); + List orClauses = new ArrayList<>(); + + for (String group : orGroups) { + // To replicate legacy search we will split using its regex [\\s\\p{Punct}]+ + String[] tokens = group.trim().split("[\\s\\p{Punct}]+"); + + // Now we will combine the tokens in this group and '&' them together. + String andClause = Arrays.stream(tokens).filter(token -> !token.isBlank()) // remove empty tokens. + .map(token -> token + ":*") // add the wild card for search + .collect(Collectors.joining(" & ")); + + if (!andClause.isBlank()) { + orClauses.add(andClause); + } + } + + + return String.join(" | ", orClauses); } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepository.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepository.java index 9ea09c4..4ecdea3 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepository.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepository.java @@ -35,7 +35,7 @@ public LegacySearchRepository( } public List getLegacySearchResults(Filter filter, Pageable pageable) { - QueryParamPair filterQ = filterGen.generateFilterQuery(filter, pageable); + QueryParamPair filterQ = filterGen.generateLegacyFilterQuery(filter, pageable); String sql = ALLOW_FILTERING_Q + ", " + filterQ.query() + """ SELECT concept_node.concept_path AS conceptPath, concept_node.display AS display, diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchControllerIntegrationTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchControllerIntegrationTest.java index d68a4f4..b939684 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchControllerIntegrationTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchControllerIntegrationTest.java @@ -54,4 +54,38 @@ void shouldGetLegacyResponseByStudyID() throws IOException { searchResults.forEach(searchResult -> Assertions.assertEquals("phs000007", searchResult.result().studyId())); } + @Test + void shouldHandleORRequest() throws IOException { + String jsonString = """ + {"query":{"searchTerm":"age","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":100}} + """; + + ResponseEntity legacyResponseResponseEntity = legacySearchController.legacySearch(jsonString); + Assertions.assertEquals(HttpStatus.OK, legacyResponseResponseEntity.getStatusCode()); + LegacyResponse legacyResponseBody = legacyResponseResponseEntity.getBody(); + Assertions.assertNotNull(legacyResponseBody); + Results results = legacyResponseBody.results(); + List ageSearchResults = results.searchResults(); + Assertions.assertEquals(4, ageSearchResults.size()); + + jsonString = """ + {"query":{"searchTerm":"physical|age","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":100}} + """; + + legacyResponseResponseEntity = legacySearchController.legacySearch(jsonString); + Assertions.assertEquals(HttpStatus.OK, legacyResponseResponseEntity.getStatusCode()); + legacyResponseBody = legacyResponseResponseEntity.getBody(); + Assertions.assertNotNull(legacyResponseBody); + results = legacyResponseBody.results(); + List physicalORAgeSearchResults = results.searchResults(); + Assertions.assertEquals(5, physicalORAgeSearchResults.size()); + + // Verify that age|physical has expanded the search results + Assertions.assertNotEquals(ageSearchResults.size(), physicalORAgeSearchResults.size()); + + // Verify the OR statement has more results + Assertions.assertTrue(ageSearchResults.size() < physicalORAgeSearchResults.size()); + } + + } diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapperTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapperTest.java index e0661ab..6c8cfa2 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapperTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapperTest.java @@ -28,12 +28,12 @@ void shouldParseSearchRequest() throws IOException { Filter filter = legacySearchQuery.filter(); Pageable pageable = legacySearchQuery.pageable(); - Assertions.assertEquals("age", filter.search()); + Assertions.assertEquals("age:*", filter.search()); Assertions.assertEquals(100, pageable.getPageSize()); } @Test - void shouldReplaceUnderscore() throws IOException { + void shouldHandlePunct() throws IOException { String jsonString = """ {"query":{"searchTerm":"tutorial-biolincc_digitalis","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":100}} @@ -43,7 +43,35 @@ void shouldReplaceUnderscore() throws IOException { Filter filter = legacySearchQuery.filter(); Pageable pageable = legacySearchQuery.pageable(); - Assertions.assertEquals("tutorial-biolincc/digitalis", filter.search()); + Assertions.assertEquals("tutorial:* & biolincc:* & digitalis:*", filter.search()); + Assertions.assertEquals(100, pageable.getPageSize()); + } + + @Test + void shouldHandleOR() throws IOException { + String jsonString = """ + {"query":{"searchTerm":"sex|gender","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":100}} + """; + + LegacySearchQuery legacySearchQuery = legacySearchQueryMapper.mapFromJson(jsonString); + Filter filter = legacySearchQuery.filter(); + Pageable pageable = legacySearchQuery.pageable(); + + Assertions.assertEquals("sex:* | gender:*", filter.search()); + Assertions.assertEquals(100, pageable.getPageSize()); + } + + @Test + void shouldHandleORAndPunct() throws IOException { + String jsonString = """ + {"query":{"searchTerm":"sex|gender age","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":100}} + """; + + LegacySearchQuery legacySearchQuery = legacySearchQueryMapper.mapFromJson(jsonString); + Filter filter = legacySearchQuery.filter(); + Pageable pageable = legacySearchQuery.pageable(); + + Assertions.assertEquals("sex:* | gender:* & age:*", filter.search()); Assertions.assertEquals(100, pageable.getPageSize()); }