Merge branch 'refs/heads/release' into ALS-7694

dictionaryweights/README.md

@@ -0,0 +1,11 @@
+## Docker commands for local development
+### Docker build
+```bash
+docker build --no-cache --build-arg SPRING_PROFILE=bdc-dev -t weights:latest .
+```
+
+### Docker run
+You will need a local weights.csv file.
+```bash
+    docker run --rm -t --name dictionary-weights --network=host -v ./weights.csv:/weights.csv weights:latest
+```

dictionaryweights/src/main/resources/application-bdc-dev.properties

@@ -0,0 +1,9 @@
+spring.application.name=dictionaryweights
+spring.main.web-application-type=none
+
+spring.datasource.url=jdbc:postgresql://localhost:5432/dictionary_db?currentSchema=dict
+spring.datasource.username=username
+spring.datasource.password=password
+spring.datasource.driver-class-name=org.postgresql.Driver
+
+weights.filename=/weights.csv

dictionaryweights/weights.csv

@@ -0,0 +1,7 @@
+concept_node.DISPLAY,2
+concept_node.CONCEPT_PATH,2
+dataset.FULL_NAME,1
+dataset.DESCRIPTION,1
+parent.DISPLAY,1
+grandparent.DISPLAY,1
+concept_node_meta_str,1

dictonaryReqeust.http

@@ -0,0 +1,16 @@
+# curl 'https://dev.picsure.biodatacatalyst.nhlbi.nih.gov/picsure/proxy/dictionary-api/concepts?page_number=1&page_size=1'
+#  -H 'origin: https://dev.picsure.biodatacatalyst.nhlbi.nih.gov'
+#  -H 'referer: https://dev.picsure.biodatacatalyst.nhlbi.nih.gov/'
+#  --data-raw '{"facets":[],"search":"","consents":[]}'
+POST http://localhost:80/concepts?page_number=0&page_size=100
+Content-Type: application/json
+
+{"facets":[],"search":"lipid triglyceride"}
+
+###
+
+POST http://localhost:80/search
+Content-Type: application/json
+
+{"@type":"GeneralQueryRequest","resourceCredentials":{},"query":{"searchTerm":"throat sore acute #8","includedTags":[],
+  "excludedTags":[],"returnTags":"true","offset":0,"limit":100000},"resourceUUID":null}

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptFilterQueryGenerator.java

@@ -82,37 +82,7 @@ public QueryParamPair generateFilterQuery(Filter filter, Pageable pageable) {
             params.addValue("consents", filter.consents());
         }
         clauses.add(createValuelessNodeFilter(filter.search(), filter.consents()));
-
-
-        String query = "(\n" + String.join("\n\tINTERSECT\n", clauses) + "\n)";
-        String superQuery = """
-            WITH q AS (
-                %s
-            )
-            %s
-            SELECT q.concept_node_id AS concept_node_id, max((1 + rank) * coalesce(rank_adjustment, 1)) AS rank
-            FROM q
-            LEFT JOIN allow_filtering ON allow_filtering.concept_node_id = q.concept_node_id
-            GROUP BY q.concept_node_id
-            ORDER BY max((1 + rank) * coalesce(rank_adjustment, 1)) DESC, q.concept_node_id ASC
-            """.formatted(query, RANK_ADJUSTMENTS);
-        // explanation of ORDER BY max((1 + rank) * coalesce(rank_adjustment, 1)) DESC
-        // you want to sort the best matches first, BUT anything that is marked as unfilterable should be put last
-        // coalesce will return the first non null value; this solves rows that aren't marked as filterable or not
-        // I then multiply that by 1 + rank instead of just rank so that a rank value of 0 for an unfilterable var
-        // is placed below a rank value of 0 for a filterable var
-        // Finally, I add the concept node id to the sort to keep it stable for ties, otherwise pagination gets weird
-
-        if (pageable.isPaged()) {
-            superQuery = superQuery + """
-                LIMIT :limit
-                OFFSET :offset
-                """;
-            params.addValue("limit", pageable.getPageSize()).addValue("offset", pageable.getOffset());
-        }
-
-        superQuery = " concepts_filtered_sorted AS (\n" + superQuery + "\n)";
-
+        String superQuery = getSuperQuery(pageable, clauses, params);
 
         return new QueryParamPair(superQuery, params);
     }
@@ -184,4 +154,78 @@ facet.name IN (:facets_for_category_%s ) AND facet_category.name = :category_%s
         }).toList();
     }
 
+    public QueryParamPair generateLegacyFilterQuery(Filter filter, Pageable pageable) {
+        MapSqlParameterSource params = new MapSqlParameterSource();
+        params.addValue("disallowed_meta_keys", disallowedMetaFields);
+        List<String> clauses = new java.util.ArrayList<>(List.of());
+        if (StringUtils.hasLength(filter.search())) {
+            params.addValue("dynamic_tsquery", filter.search().trim());
+        }
+        clauses.add(createDynamicValuelessNodeFilter(filter.search()));
+        String superQuery = getSuperQuery(pageable, clauses, params);
+
+        return new QueryParamPair(superQuery, params);
+    }
+
+    private String createDynamicValuelessNodeFilter(String search) {
+        String rankQuery = "0 as rank";
+        String rankWhere = "";
+        if (StringUtils.hasLength(search)) {
+            rankQuery = "ts_rank(searchable_fields, to_tsquery(:dynamic_tsquery)) AS rank";
+            rankWhere = "concept_node.searchable_fields @@ to_tsquery(:dynamic_tsquery) AND";
+        }
+        return """
+            SELECT
+                concept_node.concept_node_id,
+                %s
+            FROM
+                concept_node
+                LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id
+                LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min'
+                LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max'
+                LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
+            WHERE
+                %s
+                %s
+                (
+                    continuous_min.value <> '' OR
+                    continuous_max.value <> '' OR
+                    categorical_values.value <> ''
+                )
+            """
+            .formatted(rankQuery, rankWhere, "");
+    }
+
+    private static String getSuperQuery(Pageable pageable, List<String> clauses, MapSqlParameterSource params) {
+        String query = "(\n" + String.join("\n\tINTERSECT\n", clauses) + "\n)";
+        String superQuery = """
+            WITH q AS (
+                %s
+            )
+            %s
+            SELECT q.concept_node_id AS concept_node_id, max((1 + rank) * coalesce(rank_adjustment, 1)) AS rank
+            FROM q
+            LEFT JOIN allow_filtering ON allow_filtering.concept_node_id = q.concept_node_id
+            GROUP BY q.concept_node_id
+            ORDER BY max((1 + rank) * coalesce(rank_adjustment, 1)) DESC, q.concept_node_id ASC
+            """.formatted(query, RANK_ADJUSTMENTS);
+        // explanation of ORDER BY max((1 + rank) * coalesce(rank_adjustment, 1)) DESC
+        // you want to sort the best matches first, BUT anything that is marked as unfilterable should be put last
+        // coalesce will return the first non null value; this solves rows that aren't marked as filterable or not
+        // I then multiply that by 1 + rank instead of just rank so that a rank value of 0 for an unfilterable var
+        // is placed below a rank value of 0 for a filterable var
+        // Finally, I add the concept node id to the sort to keep it stable for ties, otherwise pagination gets weird
+
+        if (pageable.isPaged()) {
+            superQuery = superQuery + """
+                LIMIT :limit
+                OFFSET :offset
+                """;
+            params.addValue("limit", pageable.getPageSize()).addValue("offset", pageable.getOffset());
+        }
+
+        superQuery = " concepts_filtered_sorted AS (\n" + superQuery + "\n)";
+        return superQuery;
+    }
+
 }

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java

@@ -3,6 +3,7 @@
 import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept;
 import edu.harvard.dbmi.avillach.dictionary.filter.Filter;
 import edu.harvard.dbmi.avillach.dictionary.filter.QueryParamPair;
+import edu.harvard.dbmi.avillach.dictionary.legacysearch.SearchResultRowMapper;
 import edu.harvard.dbmi.avillach.dictionary.util.MapExtractor;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -15,32 +16,19 @@
 import java.util.Map;
 import java.util.Optional;
 
+import static edu.harvard.dbmi.avillach.dictionary.util.QueryUtility.ALLOW_FILTERING_Q;
+
+
 @Repository
 public class ConceptRepository {
 
-    private static final String ALLOW_FILTERING_Q = """
-        WITH allow_filtering AS (
-            SELECT
-                concept_node.concept_node_id AS concept_node_id,
-                (string_agg(concept_node_meta.value, ' ') NOT LIKE '%' || 'true' || '%') AS allowFiltering
-            FROM
-                concept_node
-                JOIN concept_node_meta ON
-                    concept_node.concept_node_id = concept_node_meta.concept_node_id
-                    AND concept_node_meta.KEY IN (:disallowed_meta_keys)
-            GROUP BY
-                concept_node.concept_node_id
-        )
-        """;
-
     private final NamedParameterJdbcTemplate template;
     private final ConceptRowMapper mapper;
     private final ConceptFilterQueryGenerator filterGen;
     private final ConceptMetaExtractor conceptMetaExtractor;
     private final ConceptResultSetExtractor conceptResultSetExtractor;
     private final List<String> disallowedMetaFields;
 
-
     @Autowired
     public ConceptRepository(
         NamedParameterJdbcTemplate template, ConceptRowMapper mapper, ConceptFilterQueryGenerator filterGen,
@@ -240,4 +228,6 @@ WITH RECURSIVE nodes AS (
         return Optional.ofNullable(template.query(sql, params, conceptResultSetExtractor));
 
     }
+
+
 }

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java

@@ -2,83 +2,43 @@
 
 import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept;
 import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept;
-import org.json.JSONArray;
-import org.json.JSONException;
+import edu.harvard.dbmi.avillach.dictionary.util.JsonBlobParser;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
-import java.math.BigDecimal;
-import java.math.BigInteger;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.ArrayList;
 import java.util.List;
-import java.util.stream.Collectors;
 
 @Component
 public class ConceptResultSetUtil {
 
     private static final Logger log = LoggerFactory.getLogger(ConceptResultSetUtil.class);
+    private final JsonBlobParser jsonBlobParser;
+
+    @Autowired
+    public ConceptResultSetUtil(JsonBlobParser jsonBlobParser) {
+        this.jsonBlobParser = jsonBlobParser;
+    }
 
     public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException {
         return new CategoricalConcept(
             rs.getString("concept_path"), rs.getString("name"), rs.getString("display"), rs.getString("dataset"),
-            rs.getString("description"), rs.getString("values") == null ? List.of() : parseValues(rs.getString("values")),
+            rs.getString("description"), rs.getString("values") == null ? List.of() : jsonBlobParser.parseValues(rs.getString("values")),
             rs.getBoolean("allowFiltering"), rs.getString("studyAcronym"), null, null
         );
     }
 
     public ContinuousConcept mapContinuous(ResultSet rs) throws SQLException {
         return new ContinuousConcept(
             rs.getString("concept_path"), rs.getString("name"), rs.getString("display"), rs.getString("dataset"),
-            rs.getString("description"), rs.getBoolean("allowFiltering"), parseMin(rs.getString("values")),
-            parseMax(rs.getString("values")), rs.getString("studyAcronym"), null
+            rs.getString("description"), rs.getBoolean("allowFiltering"), jsonBlobParser.parseMin(rs.getString("values")),
+            jsonBlobParser.parseMax(rs.getString("values")), rs.getString("studyAcronym"), null
         );
     }
 
-    public List<String> parseValues(String valuesArr) {
-        try {
-            ArrayList<String> vals = new ArrayList<>();
-            JSONArray arr = new JSONArray(valuesArr);
-            for (int i = 0; i < arr.length(); i++) {
-                vals.add(arr.getString(i));
-            }
-            return vals;
-        } catch (JSONException ex) {
-            return List.of();
-        }
-    }
 
-    public Float parseMin(String valuesArr) {
-        return parseFromIndex(valuesArr, 0);
-    }
 
-    private Float parseFromIndex(String valuesArr, int index) {
-        try {
-            JSONArray arr = new JSONArray(valuesArr);
-            if (arr.length() != 2) {
-                return 0F;
-            }
-            Object raw = arr.get(index);
-            return switch (raw) {
-                case Double d -> d.floatValue();
-                case Integer i -> i.floatValue();
-                case String s -> Double.valueOf(s).floatValue();
-                case BigDecimal d -> d.floatValue();
-                case BigInteger i -> i.floatValue();
-                default -> 0f;
-            };
-        } catch (JSONException ex) {
-            log.warn("Invalid json array for values: ", ex);
-            return 0F;
-        } catch (NumberFormatException ex) {
-            log.warn("Valid json array but invalid val within: ", ex);
-            return 0F;
-        }
-    }
-
-    public Float parseMax(String valuesArr) {
-        return parseFromIndex(valuesArr, 1);
-    }
 }

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java

@@ -70,4 +70,5 @@ public Optional<Concept> conceptTree(String dataset, String conceptPath, int dep
     public Optional<Concept> conceptDetailWithoutAncestors(String dataset, String conceptPath) {
         return getConcept(dataset, conceptPath, false);
     }
+
 }

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java

@@ -1,7 +1,6 @@
 package edu.harvard.dbmi.avillach.dictionary.concept.model;
 
 import edu.harvard.dbmi.avillach.dictionary.dataset.Dataset;
-import jakarta.annotation.Nullable;
 
 import java.util.List;
 import java.util.Map;

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java

@@ -4,7 +4,6 @@
 import edu.harvard.dbmi.avillach.dictionary.dataset.Dataset;
 import jakarta.annotation.Nullable;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;

src/main/java/edu/harvard/dbmi/avillach/dictionary/datasource/DataSourceVerifier.java

@@ -3,15 +3,17 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Profile;
 import org.springframework.context.event.ContextRefreshedEvent;
 import org.springframework.context.event.EventListener;
-import org.springframework.stereotype.Service;
 
 import javax.sql.DataSource;
 import java.sql.Connection;
 import java.sql.SQLException;
 
-@Service
+@Profile("!test")
+@Configuration
 public class DataSourceVerifier {
 
     private static final Logger LOG = LoggerFactory.getLogger(DataSourceVerifier.class);
@@ -28,11 +30,10 @@ public void verifyDataSourceConnection() {
         try (Connection connection = dataSource.getConnection()) {
             if (connection != null) {
                 LOG.info("Datasource connection verified successfully.");
-            } else {
-                LOG.info("Failed to obtain a connection from the datasource.");
             }
         } catch (SQLException e) {
-            LOG.info("Error verifying datasource connection: {}", e.getMessage());
+            LOG.info("Failed to obtain a connection from the datasource.");
+            LOG.debug("Error verifying datasource connection: {}", e.getMessage());
         }
     }