diff --git a/dictionaryweights/README.md b/dictionaryweights/README.md new file mode 100644 index 0000000..39e6352 --- /dev/null +++ b/dictionaryweights/README.md @@ -0,0 +1,11 @@ +## Docker commands for local development +### Docker build +```bash +docker build --no-cache --build-arg SPRING_PROFILE=bdc-dev -t weights:latest . +``` + +### Docker run +You will need a local weights.csv file. +```bash + docker run --rm -t --name dictionary-weights --network=host -v ./weights.csv:/weights.csv weights:latest +``` \ No newline at end of file diff --git a/dictionaryweights/src/main/resources/application-bdc-dev.properties b/dictionaryweights/src/main/resources/application-bdc-dev.properties new file mode 100644 index 0000000..5b79313 --- /dev/null +++ b/dictionaryweights/src/main/resources/application-bdc-dev.properties @@ -0,0 +1,9 @@ +spring.application.name=dictionaryweights +spring.main.web-application-type=none + +spring.datasource.url=jdbc:postgresql://localhost:5432/dictionary_db?currentSchema=dict +spring.datasource.username=username +spring.datasource.password=password +spring.datasource.driver-class-name=org.postgresql.Driver + +weights.filename=/weights.csv \ No newline at end of file diff --git a/dictionaryweights/weights.csv b/dictionaryweights/weights.csv new file mode 100644 index 0000000..d3cc913 --- /dev/null +++ b/dictionaryweights/weights.csv @@ -0,0 +1,7 @@ +concept_node.DISPLAY,2 +concept_node.CONCEPT_PATH,2 +dataset.FULL_NAME,1 +dataset.DESCRIPTION,1 +parent.DISPLAY,1 +grandparent.DISPLAY,1 +concept_node_meta_str,1 \ No newline at end of file diff --git a/dictonaryReqeust.http b/dictonaryReqeust.http new file mode 100644 index 0000000..e474278 --- /dev/null +++ b/dictonaryReqeust.http @@ -0,0 +1,15 @@ +# curl 'https://dev.picsure.biodatacatalyst.nhlbi.nih.gov/picsure/proxy/dictionary-api/concepts?page_number=1&page_size=1' +# -H 'origin: https://dev.picsure.biodatacatalyst.nhlbi.nih.gov' +# -H 'referer: https://dev.picsure.biodatacatalyst.nhlbi.nih.gov/' +# --data-raw '{"facets":[],"search":"","consents":[]}' +POST http://localhost:80/concepts?page_number=0&page_size=100 +Content-Type: application/json + +{"facets":[],"search":"lipid triglyceride"} + +### + +POST http://localhost:80/search +Content-Type: application/json + +{"@type":"GeneralQueryRequest","resourceCredentials":{},"query":{"searchTerm":"breast","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":10000000},"resourceUUID":null} \ No newline at end of file diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java index 410e379..7f449bb 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java @@ -3,6 +3,7 @@ import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; import edu.harvard.dbmi.avillach.dictionary.filter.QueryParamPair; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.SearchResultRowMapper; import edu.harvard.dbmi.avillach.dictionary.util.MapExtractor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -15,24 +16,12 @@ import java.util.Map; import java.util.Optional; +import static edu.harvard.dbmi.avillach.dictionary.util.QueryUtility.ALLOW_FILTERING_Q; + + @Repository public class ConceptRepository { - private static final String ALLOW_FILTERING_Q = """ - WITH allow_filtering AS ( - SELECT - concept_node.concept_node_id AS concept_node_id, - (string_agg(concept_node_meta.value, ' ') NOT LIKE '%' || 'true' || '%') AS allowFiltering - FROM - concept_node - JOIN concept_node_meta ON - concept_node.concept_node_id = concept_node_meta.concept_node_id - AND concept_node_meta.KEY IN (:disallowed_meta_keys) - GROUP BY - concept_node.concept_node_id - ) - """; - private final NamedParameterJdbcTemplate template; private final ConceptRowMapper mapper; private final ConceptFilterQueryGenerator filterGen; @@ -40,7 +29,6 @@ AND concept_node_meta.KEY IN (:disallowed_meta_keys) private final ConceptResultSetExtractor conceptResultSetExtractor; private final List disallowedMetaFields; - @Autowired public ConceptRepository( NamedParameterJdbcTemplate template, ConceptRowMapper mapper, ConceptFilterQueryGenerator filterGen, @@ -240,4 +228,6 @@ WITH RECURSIVE nodes AS ( return Optional.ofNullable(template.query(sql, params, conceptResultSetExtractor)); } + + } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java index 9e8c220..0201691 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java @@ -2,29 +2,31 @@ import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept; import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept; -import org.json.JSONArray; -import org.json.JSONException; +import edu.harvard.dbmi.avillach.dictionary.util.JsonBlobParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.math.BigDecimal; -import java.math.BigInteger; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; @Component public class ConceptResultSetUtil { private static final Logger log = LoggerFactory.getLogger(ConceptResultSetUtil.class); + private final JsonBlobParser jsonBlobParser; + + @Autowired + public ConceptResultSetUtil(JsonBlobParser jsonBlobParser) { + this.jsonBlobParser = jsonBlobParser; + } public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException { return new CategoricalConcept( rs.getString("concept_path"), rs.getString("name"), rs.getString("display"), rs.getString("dataset"), - rs.getString("description"), rs.getString("values") == null ? List.of() : parseValues(rs.getString("values")), + rs.getString("description"), rs.getString("values") == null ? List.of() : jsonBlobParser.parseValues(rs.getString("values")), rs.getBoolean("allowFiltering"), rs.getString("studyAcronym"), null, null ); } @@ -32,53 +34,11 @@ public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException { public ContinuousConcept mapContinuous(ResultSet rs) throws SQLException { return new ContinuousConcept( rs.getString("concept_path"), rs.getString("name"), rs.getString("display"), rs.getString("dataset"), - rs.getString("description"), rs.getBoolean("allowFiltering"), parseMin(rs.getString("values")), - parseMax(rs.getString("values")), rs.getString("studyAcronym"), null + rs.getString("description"), rs.getBoolean("allowFiltering"), jsonBlobParser.parseMin(rs.getString("values")), + jsonBlobParser.parseMax(rs.getString("values")), rs.getString("studyAcronym"), null ); } - public List parseValues(String valuesArr) { - try { - ArrayList vals = new ArrayList<>(); - JSONArray arr = new JSONArray(valuesArr); - for (int i = 0; i < arr.length(); i++) { - vals.add(arr.getString(i)); - } - return vals; - } catch (JSONException ex) { - return List.of(); - } - } - public Float parseMin(String valuesArr) { - return parseFromIndex(valuesArr, 0); - } - private Float parseFromIndex(String valuesArr, int index) { - try { - JSONArray arr = new JSONArray(valuesArr); - if (arr.length() != 2) { - return 0F; - } - Object raw = arr.get(index); - return switch (raw) { - case Double d -> d.floatValue(); - case Integer i -> i.floatValue(); - case String s -> Double.valueOf(s).floatValue(); - case BigDecimal d -> d.floatValue(); - case BigInteger i -> i.floatValue(); - default -> 0f; - }; - } catch (JSONException ex) { - log.warn("Invalid json array for values: ", ex); - return 0F; - } catch (NumberFormatException ex) { - log.warn("Valid json array but invalid val within: ", ex); - return 0F; - } - } - - public Float parseMax(String valuesArr) { - return parseFromIndex(valuesArr, 1); - } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java index edd1e4d..dc35254 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java @@ -70,4 +70,5 @@ public Optional conceptTree(String dataset, String conceptPath, int dep public Optional conceptDetailWithoutAncestors(String dataset, String conceptPath) { return getConcept(dataset, conceptPath, false); } + } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java index 164b953..871c34f 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java @@ -1,7 +1,6 @@ package edu.harvard.dbmi.avillach.dictionary.concept.model; import edu.harvard.dbmi.avillach.dictionary.dataset.Dataset; -import jakarta.annotation.Nullable; import java.util.List; import java.util.Map; diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java index 8b465d3..021cb4f 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java @@ -4,7 +4,6 @@ import edu.harvard.dbmi.avillach.dictionary.dataset.Dataset; import jakarta.annotation.Nullable; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/datasource/DataSourceVerifier.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/datasource/DataSourceVerifier.java index 9e36308..e3fc2a5 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/datasource/DataSourceVerifier.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/datasource/DataSourceVerifier.java @@ -3,15 +3,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Service; import javax.sql.DataSource; import java.sql.Connection; import java.sql.SQLException; -@Service +@Profile("!test") +@Configuration public class DataSourceVerifier { private static final Logger LOG = LoggerFactory.getLogger(DataSourceVerifier.class); @@ -28,11 +30,10 @@ public void verifyDataSourceConnection() { try (Connection connection = dataSource.getConnection()) { if (connection != null) { LOG.info("Datasource connection verified successfully."); - } else { - LOG.info("Failed to obtain a connection from the datasource."); } } catch (SQLException e) { - LOG.info("Error verifying datasource connection: {}", e.getMessage()); + LOG.info("Failed to obtain a connection from the datasource."); + LOG.debug("Error verifying datasource connection: {}", e.getMessage()); } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java index 98357dc..3c6eda1 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessor.java @@ -1,22 +1,28 @@ package edu.harvard.dbmi.avillach.dictionary.facet; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.filter.FilterProcessor; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.MethodParameter; import org.springframework.http.HttpInputMessage; import org.springframework.http.converter.HttpMessageConverter; -import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.ControllerAdvice; import org.springframework.web.servlet.mvc.method.annotation.RequestBodyAdvice; import java.io.IOException; import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.function.Function; @ControllerAdvice public class FilterPreProcessor implements RequestBodyAdvice { + + private final FilterProcessor filterProcessor; + + @Autowired + public FilterPreProcessor(FilterProcessor filterProcessor) { + this.filterProcessor = filterProcessor; + } + + @Override public boolean supports(MethodParameter methodParameter, Type targetType, Class> converterType) { return true; @@ -35,26 +41,13 @@ public Object afterBodyRead( Class> converterType ) { if (body instanceof Filter filter) { - List newFacets = filter.facets(); - List newConsents = filter.consents(); - if (filter.facets() != null) { - newFacets = new ArrayList<>(filter.facets()); - newFacets.sort(Comparator.comparing(Facet::name)); - } - if (filter.consents() != null) { - newConsents = new ArrayList<>(newConsents); - newConsents.sort(Comparator.comparing(Function.identity())); - } - filter = new Filter(newFacets, filter.search(), newConsents); - - if (StringUtils.hasLength(filter.search())) { - filter = new Filter(filter.facets(), filter.search().replaceAll("_", "/"), filter.consents()); - } - return filter; + return filterProcessor.processsFilter(filter); } return body; } + + @Override public Object handleEmptyBody( Object body, HttpInputMessage inputMessage, MethodParameter parameter, Type targetType, diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/FilterProcessor.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/FilterProcessor.java new file mode 100644 index 0000000..f2478ae --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/filter/FilterProcessor.java @@ -0,0 +1,34 @@ +package edu.harvard.dbmi.avillach.dictionary.filter; + +import edu.harvard.dbmi.avillach.dictionary.facet.Facet; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.function.Function; + +@Component +public class FilterProcessor { + + public Filter processsFilter(Filter filter) { + List newFacets = filter.facets(); + List newConsents = filter.consents(); + if (filter.facets() != null) { + newFacets = new ArrayList<>(filter.facets()); + newFacets.sort(Comparator.comparing(Facet::name)); + } + if (filter.consents() != null) { + newConsents = new ArrayList<>(newConsents); + newConsents.sort(Comparator.comparing(Function.identity())); + } + filter = new Filter(newFacets, filter.search(), newConsents); + + if (StringUtils.hasLength(filter.search())) { + filter = new Filter(filter.facets(), filter.search().replaceAll("_", "/"), filter.consents()); + } + return filter; + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchController.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchController.java new file mode 100644 index 0000000..c412200 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchController.java @@ -0,0 +1,32 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacyResponse; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacySearchQuery; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; + +import java.io.IOException; + +@Controller +public class LegacySearchController { + + private final LegacySearchService legacySearchService; + private final LegacySearchQueryMapper legacySearchQueryMapper; + + @Autowired + public LegacySearchController(LegacySearchService legacySearchService, LegacySearchQueryMapper legacySearchQueryMapper) { + this.legacySearchService = legacySearchService; + this.legacySearchQueryMapper = legacySearchQueryMapper; + } + + @RequestMapping(path = "/search") + public ResponseEntity legacySearch(@RequestBody String jsonString) throws IOException { + LegacySearchQuery legacySearchQuery = legacySearchQueryMapper.mapFromJson(jsonString); + return ResponseEntity + .ok(new LegacyResponse(legacySearchService.getSearchResults(legacySearchQuery.filter(), legacySearchQuery.pageable()))); + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java new file mode 100644 index 0000000..001436c --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapper.java @@ -0,0 +1,34 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.filter.FilterProcessor; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacySearchQuery; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.List; + +@Component +public class LegacySearchQueryMapper { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + private final FilterProcessor filterProcessor; + + public LegacySearchQueryMapper(FilterProcessor filterProcessor) { + this.filterProcessor = filterProcessor; + } + + public LegacySearchQuery mapFromJson(String jsonString) throws IOException { + JsonNode rootNode = objectMapper.readTree(jsonString); + JsonNode queryNode = rootNode.get("query"); + + String searchTerm = queryNode.get("searchTerm").asText(); + int limit = queryNode.get("limit").asInt(); + Filter filter = filterProcessor.processsFilter(new Filter(List.of(), searchTerm, List.of())); + return new LegacySearchQuery(filter, PageRequest.of(0, limit)); + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepository.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepository.java new file mode 100644 index 0000000..9ea09c4 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepository.java @@ -0,0 +1,79 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.concept.ConceptFilterQueryGenerator; +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.filter.QueryParamPair; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.util.List; + +import static edu.harvard.dbmi.avillach.dictionary.util.QueryUtility.ALLOW_FILTERING_Q; + +@Repository +public class LegacySearchRepository { + + private final ConceptFilterQueryGenerator filterGen; + private final NamedParameterJdbcTemplate template; + private final List disallowedMetaFields; + private final SearchResultRowMapper searchResultRowMapper; + + @Autowired + public LegacySearchRepository( + ConceptFilterQueryGenerator filterGen, NamedParameterJdbcTemplate template, + @Value("${filtering.unfilterable_concepts}") List disallowedMetaFields, SearchResultRowMapper searchResultRowMapper + ) { + this.filterGen = filterGen; + this.template = template; + this.disallowedMetaFields = disallowedMetaFields; + this.searchResultRowMapper = searchResultRowMapper; + } + + public List getLegacySearchResults(Filter filter, Pageable pageable) { + QueryParamPair filterQ = filterGen.generateFilterQuery(filter, pageable); + String sql = ALLOW_FILTERING_Q + ", " + filterQ.query() + """ + SELECT concept_node.concept_path AS conceptPath, + concept_node.display AS display, + concept_node.name AS name, + concept_node.concept_type AS conceptType, + ds.REF as dataset, + ds.abbreviation AS studyAcronym, + ds.full_name as dsFullName, + continuous_min.VALUE as min, + continuous_max.VALUE as max, + categorical_values.VALUE as values, + allow_filtering.allowFiltering AS allowFiltering, + meta_description.VALUE AS description, + stigmatized.value AS stigmatized, + parent.name AS parentName, + parent.display AS parentDisplay + FROM concept_node + INNER JOIN concepts_filtered_sorted ON concepts_filtered_sorted.concept_node_id = concept_node.concept_node_id + LEFT JOIN dataset AS ds ON concept_node.dataset_id = ds.dataset_id + LEFT JOIN concept_node_meta AS meta_description + ON concept_node.concept_node_id = meta_description.concept_node_id AND + meta_description.KEY = 'description' + LEFT JOIN concept_node_meta AS continuous_min + ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min' + LEFT JOIN concept_node_meta AS continuous_max + ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max' + LEFT JOIN concept_node_meta AS categorical_values + ON concept_node.concept_node_id = categorical_values.concept_node_id AND + categorical_values.KEY = 'values' + LEFT JOIN concept_node_meta AS stigmatized ON concept_node.concept_node_id = stigmatized.concept_node_id AND + stigmatized.KEY = 'stigmatized' + LEFT JOIN concept_node AS parent ON parent.concept_node_id = concept_node.parent_id + LEFT JOIN allow_filtering ON concept_node.concept_node_id = allow_filtering.concept_node_id + ORDER BY concepts_filtered_sorted.rank DESC, concept_node.concept_node_id ASC + """; + MapSqlParameterSource params = filterQ.params().addValue("disallowed_meta_keys", disallowedMetaFields); + + return template.query(sql, params, searchResultRowMapper); + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchService.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchService.java new file mode 100644 index 0000000..e7462c2 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchService.java @@ -0,0 +1,23 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.Results; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; + +@Service +public class LegacySearchService { + + private final LegacySearchRepository legacySearchRepository; + + @Autowired + public LegacySearchService(LegacySearchRepository legacySearchRepository) { + this.legacySearchRepository = legacySearchRepository; + } + + public Results getSearchResults(Filter filter, Pageable pageable) { + return new Results(legacySearchRepository.getLegacySearchResults(filter, pageable)); + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/MetadataResultSetUtil.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/MetadataResultSetUtil.java new file mode 100644 index 0000000..bc8eea2 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/MetadataResultSetUtil.java @@ -0,0 +1,109 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.CategoricalMetadata; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.ContinuousMetadata; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.Result; +import edu.harvard.dbmi.avillach.dictionary.util.JsonBlobParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.ResultSet; +import java.sql.SQLException; + + +@Component +public class MetadataResultSetUtil { + + private final static Logger log = LoggerFactory.getLogger(MetadataResultSetUtil.class); + private final JsonBlobParser jsonBlobParser; + + @Autowired + public MetadataResultSetUtil(JsonBlobParser jsonBlobParser) { + this.jsonBlobParser = jsonBlobParser; + } + + public Result mapContinuousMetadata(ResultSet rs) throws SQLException { + String hashedVarId = hashVarId(rs.getString("conceptPath")); + String description = getDescription(rs); + String parentName = getParentName(rs); + String parentDisplay = getParentDisplay(rs); + + String max = String.valueOf(jsonBlobParser.parseMax(rs.getString("values"))); + String min = String.valueOf(jsonBlobParser.parseMin(rs.getString("values"))); + + ContinuousMetadata metadata = new ContinuousMetadata( + rs.getString("stigmatized"), rs.getString("display"), description, min, rs.getString("conceptPath"), parentName, + rs.getString("conceptPath"), rs.getString("name"), parentDisplay, description, // changed + "{}", "", parentName, max, description, rs.getString("dataset"), hashedVarId, rs.getString("conceptType"), rs.getString("name"), + rs.getString("dataset"), rs.getString("stigmatized"), rs.getString("display"), rs.getString("studyAcronym"), + rs.getString("dsFullName"), parentName, parentDisplay, rs.getString("conceptPath"), min, max + ); + return new Result( + metadata, jsonBlobParser.parseValues(rs.getString("values")), rs.getString("dataset"), parentName, rs.getString("name"), false, + true + ); + } + + public Result mapCategoricalMetadata(ResultSet rs) throws SQLException { + String hashedVarId = hashVarId(rs.getString("conceptPath")); + String description = getDescription(rs); + String parentName = getParentName(rs); + String parentDisplay = getParentDisplay(rs); + + CategoricalMetadata metadata = new CategoricalMetadata( + rs.getString("stigmatized"), rs.getString("display"), description, "", rs.getString("conceptPath"), parentName, + rs.getString("conceptPath"), rs.getString("name"), parentDisplay, description, // changed + "{}", "", parentName, "", description, rs.getString("dataset"), hashedVarId, rs.getString("conceptType"), rs.getString("name"), + rs.getString("dataset"), rs.getString("stigmatized"), rs.getString("display"), rs.getString("studyAcronym"), + rs.getString("dsFullName"), parentName, parentDisplay, rs.getString("conceptPath") + ); + + return new Result( + metadata, jsonBlobParser.parseValues(rs.getString("values")), rs.getString("dataset"), parentName, rs.getString("name"), true, + false + ); + } + + private static String hashVarId(String hpdsPath) { + String hashedVarId = ""; + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] encodedHash = digest.digest(hpdsPath.getBytes(StandardCharsets.UTF_8)); + hashedVarId = bytesToHex(encodedHash); + } catch (NoSuchAlgorithmException e) { + log.error(e.getMessage()); + } + + return hashedVarId; + } + + private static String bytesToHex(byte[] hash) { + StringBuilder hexString = new StringBuilder(2 * hash.length); + for (byte b : hash) { + String hex = Integer.toHexString(0xff & b); + if (hex.length() == 1) { + hexString.append('0'); + } + hexString.append(hex); + } + return hexString.toString(); + } + + private String getParentDisplay(ResultSet rs) throws SQLException { + return StringUtils.hasLength("parentDisplay") ? "" : rs.getString("parentDisplay"); + } + + private String getParentName(ResultSet rs) throws SQLException { + return StringUtils.hasLength(rs.getString("parentName")) ? "All Variables" : rs.getString("parentName"); + } + + private String getDescription(ResultSet rs) throws SQLException { + return StringUtils.hasLength(rs.getString("description")) ? "" : rs.getString("description"); + } +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/SearchResultRowMapper.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/SearchResultRowMapper.java new file mode 100644 index 0000000..cb51309 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/SearchResultRowMapper.java @@ -0,0 +1,38 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptType; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.Result; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Component; + +import java.sql.ResultSet; +import java.sql.SQLException; + +@Component +public class SearchResultRowMapper implements RowMapper { + + private final MetadataResultSetUtil metadataResultSetUtil; + + @Autowired + public SearchResultRowMapper(MetadataResultSetUtil metadataResultSetUtil) { + this.metadataResultSetUtil = metadataResultSetUtil; + } + + @Override + public SearchResult mapRow(ResultSet rs, int rowNum) throws SQLException { + return mapSearchResults(rs); + } + + private SearchResult mapSearchResults(ResultSet rs) throws SQLException { + Result result = switch (ConceptType.toConcept(rs.getString("conceptType"))) { + case Categorical -> this.metadataResultSetUtil.mapCategoricalMetadata(rs); + case Continuous -> this.metadataResultSetUtil.mapContinuousMetadata(rs); + }; + + return new SearchResult(result); + } + + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/CategoricalMetadata.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/CategoricalMetadata.java new file mode 100644 index 0000000..79de460 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/CategoricalMetadata.java @@ -0,0 +1,23 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record CategoricalMetadata( + @JsonProperty("columnmeta_is_stigmatized") String columnmetaIsStigmatized, @JsonProperty("columnmeta_name") String columnmetaName, + @JsonProperty("description") String description, @JsonProperty("columnmeta_min") String columnmetaMin, + @JsonProperty("HPDS_PATH") String hpdsPath, @JsonProperty("derived_group_id") String derivedGroupId, + @JsonProperty("columnmeta_hpds_path") String columnmetaHpdsPath, @JsonProperty("columnmeta_var_id") String columnmetaVarId, + @JsonProperty("columnmeta_var_group_description") String columnmetaVarGroupDescription, + @JsonProperty("derived_var_description") String derivedVarDescription, + @JsonProperty("derived_variable_level_data") String derivedVariableLevelData, @JsonProperty("data_hierarchy") String dataHierarchy, + @JsonProperty("derived_group_description") String derivedGroupDescription, @JsonProperty("columnmeta_max") String columnmetaMax, + @JsonProperty("columnmeta_description") String columnmetaDescription, @JsonProperty("derived_study_id") String derivedStudyId, + @JsonProperty("hashed_var_id") String hashedVarId, @JsonProperty("columnmeta_data_type") String columnmetaDataType, + @JsonProperty("derived_var_id") String derivedVarId, @JsonProperty("columnmeta_study_id") String columnmetaStudyId, + @JsonProperty("is_stigmatized") String isStigmatized, @JsonProperty("derived_var_name") String derivedVarName, + @JsonProperty("derived_study_abv_name") String derivedStudyAbvName, + @JsonProperty("derived_study_description") String derivedStudyDescription, + @JsonProperty("columnmeta_var_group_id") String columnmetaVarGroupId, @JsonProperty("derived_group_name") String derivedGroupName, + @JsonProperty("columnmeta_HPDS_PATH") String columnmetaHpdsPathAlternate +) implements Metadata { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/ContinuousMetadata.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/ContinuousMetadata.java new file mode 100644 index 0000000..a067194 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/ContinuousMetadata.java @@ -0,0 +1,24 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record ContinuousMetadata( + @JsonProperty("columnmeta_is_stigmatized") String columnmetaIsStigmatized, @JsonProperty("columnmeta_name") String columnmetaName, + @JsonProperty("description") String description, @JsonProperty("columnmeta_min") String columnmetaMin, + @JsonProperty("HPDS_PATH") String hpdsPath, @JsonProperty("derived_group_id") String derivedGroupId, + @JsonProperty("columnmeta_hpds_path") String columnmetaHpdsPath, @JsonProperty("columnmeta_var_id") String columnmetaVarId, + @JsonProperty("columnmeta_var_group_description") String columnmetaVarGroupDescription, + @JsonProperty("derived_var_description") String derivedVarDescription, + @JsonProperty("derived_variable_level_data") String derivedVariableLevelData, @JsonProperty("data_hierarchy") String dataHierarchy, + @JsonProperty("derived_group_description") String derivedGroupDescription, @JsonProperty("columnmeta_max") String columnmetaMax, + @JsonProperty("columnmeta_description") String columnmetaDescription, @JsonProperty("derived_study_id") String derivedStudyId, + @JsonProperty("hashed_var_id") String hashedVarId, @JsonProperty("columnmeta_data_type") String columnmetaDataType, + @JsonProperty("derived_var_id") String derivedVarId, @JsonProperty("columnmeta_study_id") String columnmetaStudyId, + @JsonProperty("is_stigmatized") String isStigmatized, @JsonProperty("derived_var_name") String derivedVarName, + @JsonProperty("derived_study_abv_name") String derivedStudyAbvName, + @JsonProperty("derived_study_description") String derivedStudyDescription, + @JsonProperty("columnmeta_var_group_id") String columnmetaVarGroupId, @JsonProperty("derived_group_name") String derivedGroupName, + @JsonProperty("columnmeta_HPDS_PATH") String columnmetaHpdsPathAlternate, @JsonProperty("min") String min, + @JsonProperty("max") String max +) implements Metadata { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacyResponse.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacyResponse.java new file mode 100644 index 0000000..7655c6c --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacyResponse.java @@ -0,0 +1,6 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record LegacyResponse(@JsonProperty("results") Results results) { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacySearchQuery.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacySearchQuery.java new file mode 100644 index 0000000..1147a85 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/LegacySearchQuery.java @@ -0,0 +1,7 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import org.springframework.data.domain.Pageable; + +public record LegacySearchQuery(Filter filter, Pageable pageable) { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Metadata.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Metadata.java new file mode 100644 index 0000000..d3d40d4 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Metadata.java @@ -0,0 +1,4 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +public sealed interface Metadata permits ContinuousMetadata, CategoricalMetadata { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Result.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Result.java new file mode 100644 index 0000000..ba3a51d --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Result.java @@ -0,0 +1,12 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +public record Result( + Metadata metadata, List values, @JsonProperty("studyId") String studyId, @JsonProperty("dtId") String dtId, + @JsonProperty("varId") String varId, @JsonProperty("is_categorical") boolean isCategorical, + @JsonProperty("is_continuous") boolean isContinuous +) { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Results.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Results.java new file mode 100644 index 0000000..fa37331 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/Results.java @@ -0,0 +1,8 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +public record Results(@JsonProperty("searchResults") List searchResults) { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/SearchResult.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/SearchResult.java new file mode 100644 index 0000000..d6a7f53 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/model/SearchResult.java @@ -0,0 +1,6 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record SearchResult(@JsonProperty("result") Result result) { +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java new file mode 100644 index 0000000..f976e7c --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java @@ -0,0 +1,65 @@ +package edu.harvard.dbmi.avillach.dictionary.util; + + +import org.json.JSONArray; +import org.json.JSONException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +@Component +public class JsonBlobParser { + + private final static Logger log = LoggerFactory.getLogger(JsonBlobParser.class); + + public List parseValues(String valuesArr) { + try { + ArrayList vals = new ArrayList<>(); + JSONArray arr = new JSONArray(valuesArr); + for (int i = 0; i < arr.length(); i++) { + vals.add(arr.getString(i)); + } + return vals; + } catch (JSONException ex) { + return List.of(); + } + } + + public Float parseMin(String valuesArr) { + return parseFromIndex(valuesArr, 0); + } + + private Float parseFromIndex(String valuesArr, int index) { + try { + JSONArray arr = new JSONArray(valuesArr); + if (arr.length() != 2) { + return 0F; + } + Object raw = arr.get(index); + return switch (raw) { + case Double d -> d.floatValue(); + case Integer i -> i.floatValue(); + case String s -> Double.valueOf(s).floatValue(); + case BigDecimal d -> d.floatValue(); + case BigInteger i -> i.floatValue(); + default -> 0f; + }; + } catch (JSONException ex) { + log.warn("Invalid json array for values: ", ex); + return 0F; + } catch (NumberFormatException ex) { + log.warn("Valid json array but invalid val within: ", ex); + return 0F; + } + } + + public Float parseMax(String valuesArr) { + return parseFromIndex(valuesArr, 1); + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java new file mode 100644 index 0000000..69eedc4 --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/QueryUtility.java @@ -0,0 +1,19 @@ +package edu.harvard.dbmi.avillach.dictionary.util; + +public class QueryUtility { + + public static final String ALLOW_FILTERING_Q = """ + WITH allow_filtering AS ( + SELECT + concept_node.concept_node_id AS concept_node_id, + (string_agg(concept_node_meta.value, ' ') NOT LIKE '%' || 'true' || '%') AS allowFiltering + FROM + concept_node + JOIN concept_node_meta ON + concept_node.concept_node_id = concept_node_meta.concept_node_id + AND concept_node_meta.KEY IN (:disallowed_meta_keys) + GROUP BY + concept_node.concept_node_id + ) + """; +} diff --git a/src/main/resources/application-bdc-dev.properties b/src/main/resources/application-bdc-dev.properties new file mode 100644 index 0000000..219dfbb --- /dev/null +++ b/src/main/resources/application-bdc-dev.properties @@ -0,0 +1,14 @@ +spring.application.name=dictionary +spring.datasource.url=jdbc:postgresql://localhost:5432/dictionary_db?currentSchema=dict +spring.datasource.username=username +spring.datasource.password=password +spring.datasource.driver-class-name=org.postgresql.Driver +server.port=80 + +dashboard.columns={abbreviation:'Abbreviation',name:'Name',clinvars:'Clinical Variables'} +dashboard.column-order=abbreviation,name,clinvars +dashboard.nonmeta-columns=abbreviation,name +dashboard.enable.extra_details=true +dashboard.enable.bdc_hack=true + +filtering.unfilterable_concepts=stigmatized \ No newline at end of file diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java index a5d5fe6..363c180 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java @@ -16,12 +16,14 @@ import org.springframework.data.domain.Pageable; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.Map; import java.util.Optional; @SpringBootTest(properties = {"concept.tree.max_depth=1"}) +@ActiveProfiles("test") class ConceptControllerTest { @MockBean diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptDecoratorServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptDecoratorServiceTest.java index 6800f57..c543f27 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptDecoratorServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptDecoratorServiceTest.java @@ -10,12 +10,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; -import javax.print.attribute.DocAttributeSet; import java.util.Optional; @SpringBootTest +@ActiveProfiles("test") class ConceptDecoratorServiceTest { @MockBean diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java index f371012..8d20516 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java @@ -323,4 +323,6 @@ void shouldGetContConceptWithDecimalNotation() { Assertions.assertEquals(0.57f, concept.min()); Assertions.assertEquals(6.77f, concept.max()); } + + } diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtilTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtilTest.java index 3b1da9e..acd6668 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtilTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtilTest.java @@ -1,17 +1,16 @@ package edu.harvard.dbmi.avillach.dictionary.concept; +import edu.harvard.dbmi.avillach.dictionary.util.JsonBlobParser; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; - class ConceptResultSetUtilTest { @Test void shouldParseValues() { - List actual = new ConceptResultSetUtil().parseValues("[\"Look, I'm valid json\"]"); + List actual = new JsonBlobParser().parseValues("[\"Look, I'm valid json\"]"); List expected = List.of("Look, I'm valid json"); Assertions.assertEquals(expected, actual); diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceIntegrationTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceIntegrationTest.java index e51ac7b..5bef7ba 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceIntegrationTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceIntegrationTest.java @@ -8,6 +8,7 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; import org.testcontainers.containers.PostgreSQLContainer; @@ -21,6 +22,7 @@ @Testcontainers @SpringBootTest +@ActiveProfiles("test") class ConceptServiceIntegrationTest { @Autowired diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java index 0238e0c..610ba94 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java @@ -13,6 +13,7 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.data.domain.Pageable; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.Map; @@ -20,6 +21,7 @@ @SpringBootTest +@ActiveProfiles("test") class ConceptServiceTest { @MockBean diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardConfigTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardConfigTest.java index 397a055..c5e280f 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardConfigTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardConfigTest.java @@ -4,10 +4,12 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; import java.util.List; @SpringBootTest +@ActiveProfiles("test") class DashboardConfigTest { @Autowired diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardControllerTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardControllerTest.java index ec11139..116e284 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardControllerTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardControllerTest.java @@ -8,10 +8,12 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; import java.util.List; @SpringBootTest +@ActiveProfiles("test") class DashboardControllerTest { @MockBean private DashboardService service; diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardServiceTest.java index 1825e21..03255bd 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dashboard/DashboardServiceTest.java @@ -6,11 +6,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.Map; @SpringBootTest +@ActiveProfiles("test") class DashboardServiceTest { @MockBean DashboardRepository repository; diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetRepositoryTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetRepositoryTest.java index 036a36d..03f169d 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetRepositoryTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetRepositoryTest.java @@ -1,6 +1,5 @@ package edu.harvard.dbmi.avillach.dictionary.dataset; -import edu.harvard.dbmi.avillach.dictionary.facet.FacetRepository; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -15,7 +14,6 @@ import java.util.Map; import java.util.Optional; -import static org.junit.jupiter.api.Assertions.*; @Testcontainers @SpringBootTest diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetServiceTest.java index 22b8f12..8580060 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/dataset/DatasetServiceTest.java @@ -6,11 +6,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; import java.util.Map; import java.util.Optional; @SpringBootTest +@ActiveProfiles("test") class DatasetServiceTest { @MockBean diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetControllerTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetControllerTest.java index 331f176..96ea6eb 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetControllerTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetControllerTest.java @@ -9,11 +9,13 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.Optional; @SpringBootTest +@ActiveProfiles("test") class FacetControllerTest { @MockBean diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetServiceTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetServiceTest.java index e9c3aab..3b8775f 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetServiceTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetServiceTest.java @@ -8,12 +8,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.Map; import java.util.Optional; @SpringBootTest +@ActiveProfiles("test") class FacetServiceTest { @MockBean private FacetRepository repository; diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessorTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessorTest.java index 6d3d452..70e90a5 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessorTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FilterPreProcessorTest.java @@ -8,11 +8,13 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.core.MethodParameter; import org.springframework.http.HttpInputMessage; +import org.springframework.test.context.ActiveProfiles; import org.testcontainers.shaded.com.fasterxml.jackson.databind.type.SimpleType; import java.util.List; @SpringBootTest +@ActiveProfiles("test") class FilterPreProcessorTest { @Autowired diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/info/InfoControllerTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/info/InfoControllerTest.java index d401db4..53345a1 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/info/InfoControllerTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/info/InfoControllerTest.java @@ -6,12 +6,14 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; import java.util.List; import java.util.UUID; @SpringBootTest +@ActiveProfiles("test") class InfoControllerTest { @Autowired diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchControllerIntegrationTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchControllerIntegrationTest.java new file mode 100644 index 0000000..d68a4f4 --- /dev/null +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchControllerIntegrationTest.java @@ -0,0 +1,57 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacyResponse; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.Results; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.MountableFile; + +import java.io.IOException; +import java.util.List; + +@SpringBootTest +@Testcontainers +class LegacySearchControllerIntegrationTest { + + @Autowired + LegacySearchController legacySearchController; + + @Container + static final PostgreSQLContainer databaseContainer = new PostgreSQLContainer<>("postgres:16").withReuse(true) + .withCopyFileToContainer(MountableFile.forClasspathResource("seed.sql"), "/docker-entrypoint-initdb.d/seed.sql"); + + @DynamicPropertySource + static void mySQLProperties(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", databaseContainer::getJdbcUrl); + registry.add("spring.datasource.username", databaseContainer::getUsername); + registry.add("spring.datasource.password", databaseContainer::getPassword); + registry.add("spring.datasource.db", databaseContainer::getDatabaseName); + } + + @Test + void shouldGetLegacyResponseByStudyID() throws IOException { + String jsonString = """ + {"query":{"searchTerm":"phs000007","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":100}} + """; + + ResponseEntity legacyResponseResponseEntity = legacySearchController.legacySearch(jsonString); + System.out.println(legacyResponseResponseEntity); + Assertions.assertEquals(HttpStatus.OK, legacyResponseResponseEntity.getStatusCode()); + LegacyResponse legacyResponseBody = legacyResponseResponseEntity.getBody(); + Assertions.assertNotNull(legacyResponseBody); + Results results = legacyResponseBody.results(); + List searchResults = results.searchResults(); + searchResults.forEach(searchResult -> Assertions.assertEquals("phs000007", searchResult.result().studyId())); + } + +} diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapperTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapperTest.java new file mode 100644 index 0000000..e0661ab --- /dev/null +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchQueryMapperTest.java @@ -0,0 +1,50 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.LegacySearchQuery; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.domain.Pageable; +import org.springframework.test.context.ActiveProfiles; + +import java.io.IOException; + +@SpringBootTest +@ActiveProfiles("test") +class LegacySearchQueryMapperTest { + + @Autowired + LegacySearchQueryMapper legacySearchQueryMapper; + + @Test + void shouldParseSearchRequest() throws IOException { + String jsonString = """ + {"query":{"searchTerm":"age","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":100}} + """; + + LegacySearchQuery legacySearchQuery = legacySearchQueryMapper.mapFromJson(jsonString); + Filter filter = legacySearchQuery.filter(); + Pageable pageable = legacySearchQuery.pageable(); + + Assertions.assertEquals("age", filter.search()); + Assertions.assertEquals(100, pageable.getPageSize()); + } + + @Test + void shouldReplaceUnderscore() throws IOException { + String jsonString = + """ + {"query":{"searchTerm":"tutorial-biolincc_digitalis","includedTags":[],"excludedTags":[],"returnTags":"true","offset":0,"limit":100}} + """; + + LegacySearchQuery legacySearchQuery = legacySearchQueryMapper.mapFromJson(jsonString); + Filter filter = legacySearchQuery.filter(); + Pageable pageable = legacySearchQuery.pageable(); + + Assertions.assertEquals("tutorial-biolincc/digitalis", filter.search()); + Assertions.assertEquals(100, pageable.getPageSize()); + } + +} diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepositoryTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepositoryTest.java new file mode 100644 index 0000000..9bb6bab --- /dev/null +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/legacysearch/LegacySearchRepositoryTest.java @@ -0,0 +1,76 @@ +package edu.harvard.dbmi.avillach.dictionary.legacysearch; + +import edu.harvard.dbmi.avillach.dictionary.concept.ConceptRepository; +import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept; +import edu.harvard.dbmi.avillach.dictionary.filter.Filter; +import edu.harvard.dbmi.avillach.dictionary.legacysearch.model.SearchResult; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.domain.Pageable; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.MountableFile; + +import java.util.List; + +@SpringBootTest +@Testcontainers +public class LegacySearchRepositoryTest { + + @Autowired + LegacySearchRepository subject; + + @Autowired + ConceptRepository conceptService; + + @Container + static final PostgreSQLContainer databaseContainer = new PostgreSQLContainer<>("postgres:16").withReuse(true) + .withCopyFileToContainer(MountableFile.forClasspathResource("seed.sql"), "/docker-entrypoint-initdb.d/seed.sql"); + + @DynamicPropertySource + static void mySQLProperties(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", databaseContainer::getJdbcUrl); + registry.add("spring.datasource.username", databaseContainer::getUsername); + registry.add("spring.datasource.password", databaseContainer::getPassword); + registry.add("spring.datasource.db", databaseContainer::getDatabaseName); + } + + @Test + void shouldGetLegacySearchResults() { + List searchResults = subject.getLegacySearchResults(new Filter(List.of(), "", List.of()), Pageable.unpaged()); + + Assertions.assertEquals(30, searchResults.size()); + } + + @Test + void shouldGetLegacySearchResultsBySearch() { + List searchResults = + subject.getLegacySearchResults(new Filter(List.of(), "phs000007", List.of()), Pageable.unpaged()); + + searchResults.forEach(searchResult -> Assertions.assertEquals("phs000007", searchResult.result().studyId())); + + } + + @Test + void shouldGetLegacySearchResultsByPageSize() { + List searchResults = subject.getLegacySearchResults(new Filter(List.of(), "", List.of()), Pageable.ofSize(5)); + + Assertions.assertEquals(5, searchResults.size()); + } + + @Test + void legacySearchResultShouldGetEqualCountToConceptSearch() { + // This test will ensure modifications made to the conceptSearch will be reflected in the legacy search result. + // They use near equivalent queries and updates made to one should be made to the other. + List searchResults = subject.getLegacySearchResults(new Filter(List.of(), "", List.of()), Pageable.unpaged()); + List concepts = conceptService.getConcepts(new Filter(List.of(), "", List.of()), Pageable.unpaged()); + + Assertions.assertEquals(searchResults.size(), concepts.size()); + } + +}