From 5108b827e697c19afd57bc9e99238ef2ecbee663 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 8 Sep 2022 16:40:17 +0100 Subject: [PATCH 01/40] Initial support for screen/project attributes search form --- .../static/idr_gallery/omero_search_form.js | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/idr_gallery/static/idr_gallery/omero_search_form.js b/idr_gallery/static/idr_gallery/omero_search_form.js index 507a4f14..4ad7bec2 100644 --- a/idr_gallery/static/idr_gallery/omero_search_form.js +++ b/idr_gallery/static/idr_gallery/omero_search_form.js @@ -222,12 +222,14 @@ class OmeroSearchForm { // Adds `; - // only show 'image' attributes - let imgKeys = this.resources_data.image; - imgKeys.sort(); - let html = imgKeys - .map((value) => ``) - .join("\n"); + // Show all + let html = Object.entries(this.resources_data).map((resourceValues) => { + let resource = resourceValues[0]; + let values = resourceValues[1]; + values.sort(); + const options = values.map((value) => ``).join("\n"); + return `${options}` + }).join("\n"); $field.html(anyOption + html); } @@ -235,7 +237,7 @@ class OmeroSearchForm { // returns a sort function based on the current query Value // NB: same logic in autocompleteSort() function used on front page queryVal = queryVal.toLowerCase(); - const KNOWN_KEYS = this.resources_data; + const KNOWN_KEYS = [].concat(...Object.values(this.resources_data)); return (a, b) => { // if exact match, show first let aMatch = queryVal == a.Value.toLowerCase(); @@ -244,8 +246,8 @@ class OmeroSearchForm { return aMatch ? -1 : 1; } // show all known Keys before unknown - let aKnown = KNOWN_KEYS.image.includes(a.Key); - let bKnown = KNOWN_KEYS.image.includes(b.Key); + let aKnown = KNOWN_KEYS.includes(a.Key); + let bKnown = KNOWN_KEYS.includes(b.Key); if (aKnown != bKnown) { return aKnown ? -1 : 1; } @@ -269,7 +271,7 @@ class OmeroSearchForm { // Need to know what Attribute is of adjacent if not there; let $select = $(".keyFields", $parent); if ($(`option[value='${key}']`, $select).length == 0) { - $select.append($(``)); + // update this.resources_data and re-render key = $(".keyFields", $orClause).val(); - let data = { value: request.term }; - let url = `${SEARCH_ENGINE_URL}resources/all/searchvalues/`; + let params = { value: request.term }; if (key != "Any") { - data.key = key; + params.key = key; + } + params = new URLSearchParams(params).toString(); + console.log("params", params); + let kvp_url = + `${SEARCH_ENGINE_URL}resources/all/searchvalues/?` + params; + let urls = [kvp_url]; + + if (key == "Any") { + // Need to load data from 2 end-points + let names_url = `${SEARCH_ENGINE_URL}resources/all/names/?value=${request.term}&use_description=true`; + urls.push(names_url); } - // showSpinner(); - $.ajax({ - dataType: "json", - data, - type: "GET", - url: url, - success: function (data) { - // hideSpinner(); - let results = [{ label: "No results found.", value: -1 }]; - // combine 'screen', 'project' and 'image' results - can ignore 'well', 'plate' etc. - let screenHits = data.screen.data.map((obj) => { - return { ...obj, type: "screen" }; - }); - let projectHits = data.project.data.map((obj) => { - return { ...obj, type: "project" }; - }); - let imageHits = data.image.data.map((obj) => { - return { ...obj, type: "image" }; - }); - let data_results = [].concat(screenHits, projectHits, imageHits); - if (data_results.length > 0) { - // only try to show top 100 items... - let max_shown = 100; - let result_count = data_results.length; - // sort to put exact and 'known' matches first - data_results.sort(self.autocompleteSort(request.term)); - results = data_results.slice(0, 100).map((result) => { - let showKey = key === "Any" ? `(${result.Key})` : ""; - let type = result.type; - let count = result[`Number of ${type}s`]; - return { - key: result.Key, - label: `${ - result.Value - } ${showKey} ${count} ${type}${ - count != 1 ? "s" : "" - }`, - value: `${result.Value}`, - dtype: type, - }; - }); - if (result_count > max_shown) { - results.push({ - key: -1, - label: `...and ${ - result_count - max_shown - } more matches not shown`, - value: -1, - }); - } - } - response(results); - }, - error: function (data) { - console.log("ERROR", data); - // hideSpinner(); - response([{ label: "Failed to load", value: -1 }]); - }, + + const promises = urls.map((p) => fetch(p).then((rsp) => rsp.json())); + const responses = await Promise.all(promises); + console.log("responses", responses); + + const data = responses[0]; + + // hideSpinner(); + let results; + // combine 'screen', 'project' and 'image' results - can ignore 'well', 'plate' etc. + let screenHits = data.screen.data.map((obj) => { + return { ...obj, type: "screen" }; + }); + let projectHits = data.project.data.map((obj) => { + return { ...obj, type: "project" }; }); + let imageHits = data.image.data.map((obj) => { + return { ...obj, type: "image" }; + }); + let data_results = [].concat(screenHits, projectHits, imageHits); + // sort to put exact and 'known' matches first + data_results.sort(self.autocompleteSort(request.term)); + + results = data_results.map((result) => { + let showKey = key === "Any" ? `(${result.Key})` : ""; + let type = result.type; + let count = result[`Number of ${type}s`]; + return { + key: result.Key, + label: `${ + result.Value + } ${showKey} ${count} ${type}${ + count != 1 ? "s" : "" + }`, + value: `${result.Value}`, + dtype: type, + }; + }); + if (key == "Any") { + const projectNameHits = mapNames( + responses[1].project, + "project", + request.term + ); + const screenNameHits = mapNames( + responses[1].screen, + "screen", + request.term + ); + const nameHits = projectNameHits.concat(screenNameHits); + console.log("nameHits", nameHits); + // TODO: sort nameHits... + results = nameHits.concat(results); + } + let result_count = results.length; + + const max_shown = 100; + if (result_count > max_shown) { + results = results.slice(0, max_shown); + results.push({ + key: -1, + label: `...and ${ + result_count - max_shown + } more matches not shown`, + value: -1, + }); + } else if (result_count == 0) { + results = [{ label: "No results found.", value: -1 }]; + } + console.log("results", results); + response(results); }, minLength: 1, open: function () {}, From 439a9f4926d055e11441b1647fcb9b76c5db4eb3 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 28 Nov 2022 12:15:01 +0000 Subject: [PATCH 05/40] Fix auto-complete for Name/Description --- .../static/idr_gallery/omero_search_form.js | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/idr_gallery/static/idr_gallery/omero_search_form.js b/idr_gallery/static/idr_gallery/omero_search_form.js index f07f4935..8c398d73 100644 --- a/idr_gallery/static/idr_gallery/omero_search_form.js +++ b/idr_gallery/static/idr_gallery/omero_search_form.js @@ -59,16 +59,31 @@ const FILTER_ICON_SVG = ` C58.1,59.1,81.058,61.387,105.34,61.387c24.283,0,47.24-2.287,65.034-6.449L119.631,116.486z"/> `; +const NAME = "Name (IDR number)" + // projects or screens might match Name or Description. -function mapNames(rsp, type, searchTerm) { +function mapNames(rsp, type, key, searchTerm) { // rsp is a list of [ {id, name, description}, ] searchTerm = searchTerm.toLowerCase(); + + if (key == "Description") { + // results from resources/all/names/?use_description=true will include searches by name + // need to check they really match description + rsp = rsp.filter((resultObj) => { + return resultObj.description.toLowerCase().includes(searchTerm); + }) + } return rsp.map((resultObj) => { let name = resultObj.name; let desc = resultObj.description; - let attribute = name.toLowerCase().includes(searchTerm) - ? "Name (IDR number)" + let attribute = key; + // If we searched for Any, show all results. + // "Attribute" form field will be filled (Name or Desc) if user picks item + if (attribute == "Any") { + attribute = name.toLowerCase().includes(searchTerm) + ? NAME : "Description"; + } let value = name; if (attribute == "Description") { // truncate Description around matching word... @@ -87,11 +102,12 @@ function mapNames(rsp, type, searchTerm) { if (start + targetLength < desc.length) { truncated = truncated + "..."; } - value = truncated; + value = desc; + name = truncated; } return { key: attribute, - label: `${value} (${attribute}) 1 ${type}`, + label: `${name} (${attribute}) 1 ${type}`, value, dtype: type, }; @@ -323,9 +339,12 @@ class OmeroSearchForm { `${SEARCH_ENGINE_URL}resources/all/searchvalues/?` + params; let urls = [kvp_url]; - if (key == "Any") { + if (key == "Any" || key == "Description" || key == NAME) { // Need to load data from 2 end-points - let names_url = `${SEARCH_ENGINE_URL}resources/all/names/?value=${request.term}&use_description=true`; + let names_url = `${SEARCH_ENGINE_URL}resources/all/names/?value=${request.term}`; + if (key == "Any" || key == "Description") { + names_url += `&use_description=true`; + } urls.push(names_url); } @@ -366,15 +385,18 @@ class OmeroSearchForm { dtype: type, }; }); - if (key == "Any") { + // If we searched the 2nd Name/Description endpoint, concat the results... + if (responses[1]) { const projectNameHits = mapNames( responses[1].project, "project", + key, request.term ); const screenNameHits = mapNames( responses[1].screen, "screen", + key, request.term ); const nameHits = projectNameHits.concat(screenNameHits); @@ -691,7 +713,7 @@ class OmeroSearchForm { let self = this; query.query_details.and_filters.push({ // TODO: api key should be 'name' not 'Name (IDR number)' - name: "Name (IDR number)", + name: NAME, value: studyName, operator: "equals", resource: "project", // NB: this works for screens too! From 301eaaf42e57d09c3f4cc13c2e8eb45804b1c46a Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 28 Nov 2022 13:26:48 +0000 Subject: [PATCH 06/40] Search for 'container' instead of project/screen. Show Study in Attribute menu --- idr_gallery/static/idr_gallery/omero_search_form.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/idr_gallery/static/idr_gallery/omero_search_form.js b/idr_gallery/static/idr_gallery/omero_search_form.js index 8c398d73..b549055d 100644 --- a/idr_gallery/static/idr_gallery/omero_search_form.js +++ b/idr_gallery/static/idr_gallery/omero_search_form.js @@ -181,6 +181,9 @@ class OmeroSearchForm { // e.g. find if 'Antibody' key comes from 'image', 'project' etc for (let resource in this.resources_data) { if (this.resources_data[resource].includes(key)) { + if (resource == "project" || resource == "screen") { + resource = "container" + } return resource; } } @@ -277,8 +280,10 @@ class OmeroSearchForm { // Adds `; - // Show all - let html = Object.entries(this.resources_data) + // We combine 'project' and 'screen' into 'Study' + let menu = {'Study': this.resources_data.project.concat(this.resources_data.screen) ,'Image': this.resources_data.image} + + let html = Object.entries(menu) .map((resourceValues) => { let resource = resourceValues[0]; let values = resourceValues[1]; From ad73bf9dfbb2cb618376f8cba90241c4d88e6417 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 28 Nov 2022 13:27:51 +0000 Subject: [PATCH 07/40] prettier fixes --- .../static/idr_gallery/omero_search_form.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/idr_gallery/static/idr_gallery/omero_search_form.js b/idr_gallery/static/idr_gallery/omero_search_form.js index b549055d..54fa4346 100644 --- a/idr_gallery/static/idr_gallery/omero_search_form.js +++ b/idr_gallery/static/idr_gallery/omero_search_form.js @@ -59,7 +59,7 @@ const FILTER_ICON_SVG = ` C58.1,59.1,81.058,61.387,105.34,61.387c24.283,0,47.24-2.287,65.034-6.449L119.631,116.486z"/> `; -const NAME = "Name (IDR number)" +const NAME = "Name (IDR number)"; // projects or screens might match Name or Description. function mapNames(rsp, type, key, searchTerm) { @@ -71,18 +71,18 @@ function mapNames(rsp, type, key, searchTerm) { // need to check they really match description rsp = rsp.filter((resultObj) => { return resultObj.description.toLowerCase().includes(searchTerm); - }) + }); } return rsp.map((resultObj) => { let name = resultObj.name; let desc = resultObj.description; let attribute = key; // If we searched for Any, show all results. - // "Attribute" form field will be filled (Name or Desc) if user picks item + // "Attribute" form field will be filled (Name or Desc) if user picks item if (attribute == "Any") { attribute = name.toLowerCase().includes(searchTerm) - ? NAME - : "Description"; + ? NAME + : "Description"; } let value = name; if (attribute == "Description") { @@ -182,7 +182,7 @@ class OmeroSearchForm { for (let resource in this.resources_data) { if (this.resources_data[resource].includes(key)) { if (resource == "project" || resource == "screen") { - resource = "container" + resource = "container"; } return resource; } @@ -281,7 +281,10 @@ class OmeroSearchForm { let $field = $(".keyFields", $orClause); let anyOption = ``; // We combine 'project' and 'screen' into 'Study' - let menu = {'Study': this.resources_data.project.concat(this.resources_data.screen) ,'Image': this.resources_data.image} + let menu = { + Study: this.resources_data.project.concat(this.resources_data.screen), + Image: this.resources_data.image, + }; let html = Object.entries(menu) .map((resourceValues) => { From d31db877ddc97563bf8bd20ca3e3415492cdf102 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 28 Nov 2022 14:43:22 +0000 Subject: [PATCH 08/40] Don't alert() if no results. Display instead --- idr_gallery/static/idr_gallery/omero_search_form.js | 8 +++----- idr_gallery/templates/idr_gallery/search.html | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/idr_gallery/static/idr_gallery/omero_search_form.js b/idr_gallery/static/idr_gallery/omero_search_form.js index 54fa4346..cd5a3e29 100644 --- a/idr_gallery/static/idr_gallery/omero_search_form.js +++ b/idr_gallery/static/idr_gallery/omero_search_form.js @@ -466,6 +466,9 @@ class OmeroSearchForm { let $select = $(".keyFields", $parent); if ($(`option[value='${key}']`, $select).length == 0) { // update this.resources_data and re-render in place of "name" key +const NAME_IDR_NUMBER = "Name (IDR number)"; // projects or screens might match Name or Description. function mapNames(rsp, type, key, searchTerm) { @@ -82,7 +84,7 @@ function mapNames(rsp, type, key, searchTerm) { // "Attribute" form field will be filled (Name or Desc) if user picks item if (attribute == "Any") { attribute = name.toLowerCase().includes(searchTerm) - ? NAME + ? NAME_KEY : "Description"; } let value = name; @@ -174,6 +176,12 @@ class OmeroSearchForm { if (this.resources_data.error != undefined) { alert(this.resources_data.error); } + // Remove key "Name (IDR number)", replace with "name" + if (this.resources_data["project"].includes(NAME_IDR_NUMBER)) { + this.resources_data["project"] = this.resources_data["project"].filter(k => k != NAME_IDR_NUMBER); + this.resources_data["project"].push(NAME_KEY); + this.resources_data["project"].sort(); + } return this.resources_data; } @@ -287,13 +295,21 @@ class OmeroSearchForm { Image: this.resources_data.image, }; + const getDisplayValue = (value) => { + // UI shows "Name (IDR number)" instead of "name" + if (value == NAME_KEY) { + return NAME_IDR_NUMBER + } + return value + } + let html = Object.entries(menu) .map((resourceValues) => { let resource = resourceValues[0]; let values = resourceValues[1]; - values.sort(); + values.sort((a, b) => a.toLowerCase() < b.toLowerCase() ? -1 : 1); const options = values - .map((value) => ``) + .map((value) => ``) .join("\n"); return `${options}`; }) @@ -347,7 +363,7 @@ class OmeroSearchForm { `${SEARCH_ENGINE_URL}resources/all/searchvalues/?` + params; let urls = [kvp_url]; - if (key == "Any" || key == "Description" || key == NAME) { + if (key == "Any" || key == "Description" || key == NAME_KEY) { // Need to load data from 2 end-points let names_url = `${SEARCH_ENGINE_URL}resources/all/names/?value=${request.term}`; // NB: Don't show auto-complete for Description yet (not supported for search yet) @@ -724,8 +740,7 @@ class OmeroSearchForm { let query = this.getPreviousSearchQuery(); let self = this; query.query_details.and_filters.push({ - // TODO: api key should be 'name' not 'Name (IDR number)' - name: NAME, + name: NAME_KEY, value: studyName, operator: "equals", resource: "project", // NB: this works for screens too! From 8ea2b7c2f6eebd3940f93c9dd8b2f56c5d8e9764 Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 7 Dec 2022 12:12:24 +0000 Subject: [PATCH 13/40] Move autocomplete functionality to reusable function --- .../static/idr_gallery/omero_search_form.js | 251 +++++++++--------- 1 file changed, 129 insertions(+), 122 deletions(-) diff --git a/idr_gallery/static/idr_gallery/omero_search_form.js b/idr_gallery/static/idr_gallery/omero_search_form.js index ddfe486f..e8f09120 100644 --- a/idr_gallery/static/idr_gallery/omero_search_form.js +++ b/idr_gallery/static/idr_gallery/omero_search_form.js @@ -117,6 +117,121 @@ function mapNames(rsp, type, key, searchTerm) { }); } +function autocompleteSort(queryVal, knownKeys = []) { + // returns a sort function based on the current query Value + // knownKeys is list of common keys e.g. ["Gene Symbol", "Antibody"] etc. + + queryVal = queryVal.toLowerCase(); + // const KNOWN_KEYS = [].concat(...Object.values(this.resources_data)); + return (a, b) => { + // if exact match, show first + let aMatch = queryVal == a.Value.toLowerCase(); + let bMatch = queryVal == b.Value.toLowerCase(); + if (aMatch != bMatch) { + return aMatch ? -1 : 1; + } + // show all known Keys before unknown + let aKnown = knownKeys.includes(a.Key); + let bKnown = knownKeys.includes(b.Key); + if (aKnown != bKnown) { + return aKnown ? -1 : 1; + } + // Show highest Image counts first + let aCount = a["Number of images"]; + let bCount = b["Number of images"]; + return aCount > bCount ? -1 : aCount < bCount ? 1 : 0; + }; +} + +async function getAutoCompleteResults(key, query, knownKeys) { + let params = { value: query }; + if (key != "Any") { + params.key = key; + } + params = new URLSearchParams(params).toString(); + let kvp_url = `${SEARCH_ENGINE_URL}resources/all/searchvalues/?` + params; + let urls = [kvp_url]; + + if (key == "Any" || key == "Description" || key == NAME_KEY) { + // Need to load data from 2 end-points + let names_url = `${SEARCH_ENGINE_URL}resources/all/names/?value=${query}`; + // NB: Don't show auto-complete for Description yet (not supported for search yet) + // if (key == "Any" || key == "Description") { + // names_url += `&use_description=true`; + // } + urls.push(names_url); + } + + const promises = urls.map((p) => fetch(p).then((rsp) => rsp.json())); + const responses = await Promise.all(promises); + + const data = responses[0]; + + // hideSpinner(); + let results; + // combine 'screen', 'project' and 'image' results - can ignore 'well', 'plate' etc. + let screenHits = data.screen.data.map((obj) => { + return { ...obj, type: "screen" }; + }); + let projectHits = data.project.data.map((obj) => { + return { ...obj, type: "project" }; + }); + let imageHits = data.image.data.map((obj) => { + return { ...obj, type: "image" }; + }); + let data_results = [].concat(screenHits, projectHits, imageHits); + // sort to put exact and 'known' matches first + data_results.sort(autocompleteSort(query, knownKeys)); + + results = data_results.map((result) => { + let showKey = key === "Any" ? `(${result.Key})` : ""; + let type = result.type; + let count = result[`Number of ${type}s`]; + return { + key: result.Key, + label: `${ + result.Value + } ${showKey} ${count} ${type}${ + count != 1 ? "s" : "" + }`, + value: `${result.Value}`, + dtype: type, + }; + }); + // If we searched the 2nd Name/Description endpoint, concat the results... + if (responses[1]) { + const projectNameHits = mapNames( + responses[1].project, + "project", + key, + query + ); + const screenNameHits = mapNames(responses[1].screen, "screen", key, query); + const nameHits = projectNameHits.concat(screenNameHits); + // TODO: sort nameHits... + results = nameHits.concat(results); + } + + // filter to remove annotation.csv KV pairs + results = results.filter((item) => !item.value.includes("annotation.csv")); + + let result_count = results.length; + + const max_shown = 100; + if (result_count > max_shown) { + results = results.slice(0, max_shown); + results.push({ + key: -1, + label: `...and ${result_count - max_shown} more matches not shown`, + value: -1, + }); + } else if (result_count == 0) { + results = [{ label: "No results found.", value: -1 }]; + } + + return results; +} + const SPINNER_SVG = ``; class OmeroSearchForm { constructor(formId, SEARCH_ENGINE_URL, resultsId) { @@ -178,7 +293,9 @@ class OmeroSearchForm { } // Remove key "Name (IDR number)", replace with "name" if (this.resources_data["project"].includes(NAME_IDR_NUMBER)) { - this.resources_data["project"] = this.resources_data["project"].filter(k => k != NAME_IDR_NUMBER); + this.resources_data["project"] = this.resources_data["project"].filter( + (k) => k != NAME_IDR_NUMBER + ); this.resources_data["project"].push(NAME_KEY); this.resources_data["project"].sort(); } @@ -298,18 +415,21 @@ class OmeroSearchForm { const getDisplayValue = (value) => { // UI shows "Name (IDR number)" instead of "name" if (value == NAME_KEY) { - return NAME_IDR_NUMBER + return NAME_IDR_NUMBER; } - return value - } + return value; + }; let html = Object.entries(menu) .map((resourceValues) => { let resource = resourceValues[0]; let values = resourceValues[1]; - values.sort((a, b) => a.toLowerCase() < b.toLowerCase() ? -1 : 1); + values.sort((a, b) => (a.toLowerCase() < b.toLowerCase() ? -1 : 1)); const options = values - .map((value) => ``) + .map( + (value) => + `` + ) .join("\n"); return `${options}`; }) @@ -317,34 +437,10 @@ class OmeroSearchForm { $field.html(anyOption + html); } - autocompleteSort(queryVal) { - // returns a sort function based on the current query Value - // NB: same logic in autocompleteSort() function used on front page - queryVal = queryVal.toLowerCase(); - const KNOWN_KEYS = [].concat(...Object.values(this.resources_data)); - return (a, b) => { - // if exact match, show first - let aMatch = queryVal == a.Value.toLowerCase(); - let bMatch = queryVal == b.Value.toLowerCase(); - if (aMatch != bMatch) { - return aMatch ? -1 : 1; - } - // show all known Keys before unknown - let aKnown = KNOWN_KEYS.includes(a.Key); - let bKnown = KNOWN_KEYS.includes(b.Key); - if (aKnown != bKnown) { - return aKnown ? -1 : 1; - } - // Show highest Image counts first - let aCount = a["Number of images"]; - let bCount = b["Number of images"]; - return aCount > bCount ? -1 : aCount < bCount ? 1 : 0; - }; - } - initAutoComplete($orClause) { let self = this; let $this = $(".valueFields", $orClause); + const knownKeys = [].concat(...Object.values(this.resources_data)); // key is updated when user starts typing, also used to handle response and select let key; $this @@ -354,98 +450,9 @@ class OmeroSearchForm { source: async function (request, response) { // Need to know what Attribute is of adjacent in place of "name" key const NAME_IDR_NUMBER = "Name (IDR number)"; +const displayTypes = { + "image": "image", + "project": "experiment", + "screen": "screen" +} + // projects or screens might match Name or Description. function mapNames(rsp, type, key, searchTerm) { // rsp is a list of [ {id, name, description}, ] @@ -108,9 +114,10 @@ function mapNames(rsp, type, key, searchTerm) { value = desc; name = truncated; } + return { key: attribute, - label: `${name} (${attribute}) 1 ${type}`, + label: `${name} (${attribute}) 1 ${displayTypes[type]}`, value, dtype: type, }; From fc3b853744820af8ee0dc1370654d70b0854ce7b Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 7 Dec 2022 13:54:35 +0000 Subject: [PATCH 16/40] Use result.count for autocomplete display --- idr_gallery/static/idr_gallery/omero_search_form.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idr_gallery/static/idr_gallery/omero_search_form.js b/idr_gallery/static/idr_gallery/omero_search_form.js index 0c8bcfd4..2267cbb2 100644 --- a/idr_gallery/static/idr_gallery/omero_search_form.js +++ b/idr_gallery/static/idr_gallery/omero_search_form.js @@ -191,7 +191,7 @@ async function getAutoCompleteResults(key, query, knownKeys) { results = data_results.map((result) => { let showKey = key === "Any" ? `(${result.Key})` : ""; let type = result.type; - let count = result[`Number of ${type}s`]; + let count = result.count; return { key: result.Key, label: `${ From 8ce5b62560f7ff101589faff7044c7cf4676c8f9 Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 7 Dec 2022 13:55:50 +0000 Subject: [PATCH 17/40] Generate auto-complete results grouped by key - console.log only --- .../static/idr_gallery/omero_search_form.js | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/idr_gallery/static/idr_gallery/omero_search_form.js b/idr_gallery/static/idr_gallery/omero_search_form.js index 2267cbb2..b6aff220 100644 --- a/idr_gallery/static/idr_gallery/omero_search_form.js +++ b/idr_gallery/static/idr_gallery/omero_search_form.js @@ -64,10 +64,10 @@ const NAME_KEY = "name"; const NAME_IDR_NUMBER = "Name (IDR number)"; const displayTypes = { - "image": "image", - "project": "experiment", - "screen": "screen" -} + image: "image", + project: "experiment", + screen: "screen", +}; // projects or screens might match Name or Description. function mapNames(rsp, type, key, searchTerm) { @@ -220,6 +220,27 @@ async function getAutoCompleteResults(key, query, knownKeys) { // filter to remove annotation.csv KV pairs results = results.filter((item) => !item.value.includes("annotation.csv")); + // Generate Summary of [{key: "name", hits: 5, type: container}, {key: "Gene Symbol", hits: 1000, type: image}} + let keyCounts = {}; + data_results.forEach((result) => { + if (!keyCounts[result.Key]) { + keyCounts[result.Key] = { + key: result.Key, + count: 0, + type: result.type, + matches: [], + }; + } + keyCounts[result.Key].count += result.count; + keyCounts[result.Key].matches.push(result); + }); + let keyCountsList = Object.values(keyCounts); + keyCountsList.sort((a, b) => + a.count < b.count ? 1 : a.count > b.count ? -1 : a.key > b.key ? 1 : -1 + ); + // TODO: we don't use this summary yet... Display to user somehow?? + console.log("keyCountsList", keyCountsList); + let result_count = results.length; const max_shown = 100; From 9743a5d8eac7a07a1ba6bd904e2d19f63a0de15a Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 7 Dec 2022 14:26:55 +0000 Subject: [PATCH 18/40] Handle redirect for non-mapr queries on studies --- idr_gallery/views.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/idr_gallery/views.py b/idr_gallery/views.py index b7aa96d1..9deef3a5 100644 --- a/idr_gallery/views.py +++ b/idr_gallery/views.py @@ -62,8 +62,16 @@ def index(request, super_category=None, conn=None, **kwargs): key=keyval[0], value=keyval[1], operator="equals") - # otherwise show filter studies page - template = "idr_gallery/mapr_search.html" + # handle e.g. ?query=Publication%20Authors:smith + # ?key=Publication+Authors&value=Smith&operator=contains&resource=container + keyval = query.split(":", 1) + # search for studies ("containers") and use "contains" + # to match previous behaviour + return redirect_with_params('idr_gallery_search', + key=keyval[0], + value=keyval[1], + resource="container", + operator="contains") else: template = "idr_gallery/search.html" context = {'template': template} From 52ed6abb828828a7065a2f12f63ff5ff01639c58 Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 7 Dec 2022 14:49:03 +0000 Subject: [PATCH 19/40] Use resource:container for loading study images --- idr_gallery/static/idr_gallery/omero_search_form.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idr_gallery/static/idr_gallery/omero_search_form.js b/idr_gallery/static/idr_gallery/omero_search_form.js index b6aff220..d0790c5c 100644 --- a/idr_gallery/static/idr_gallery/omero_search_form.js +++ b/idr_gallery/static/idr_gallery/omero_search_form.js @@ -776,7 +776,7 @@ class OmeroSearchForm { name: NAME_KEY, value: studyName, operator: "equals", - resource: "project", // NB: this works for screens too! + resource: "container", }); // if pagination data object exists, we are loading next pages... const pagination = $studyRow.data("pagination"); From 5e68658c898a85588439dee58b94758e67a2e40c Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 8 Dec 2022 16:30:21 +0000 Subject: [PATCH 20/40] Use 'contains' for Image-attribute redirects --- idr_gallery/views.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/idr_gallery/views.py b/idr_gallery/views.py index 9deef3a5..1b97ffe0 100644 --- a/idr_gallery/views.py +++ b/idr_gallery/views.py @@ -57,11 +57,12 @@ def index(request, super_category=None, conn=None, **kwargs): if query.startswith("mapr_"): keyval = find_mapr_key_value(request, query) if keyval is not None: - # /search/?key=Gene+Symbol&value=pax6&operator=equals + # /search/?key=Gene+Symbol&value=pax6&operator=contains + # Use "contains" to be consistent with studies search below return redirect_with_params('idr_gallery_search', key=keyval[0], value=keyval[1], - operator="equals") + operator="contains") # handle e.g. ?query=Publication%20Authors:smith # ?key=Publication+Authors&value=Smith&operator=contains&resource=container keyval = query.split(":", 1) From 9119e95299b358ffd3f35db7cec442eb0ab38c1f Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 9 Dec 2022 10:12:26 +0000 Subject: [PATCH 21/40] Switch operator order in UI, contains first default --- idr_gallery/static/idr_gallery/omero_search_form.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/idr_gallery/static/idr_gallery/omero_search_form.js b/idr_gallery/static/idr_gallery/omero_search_form.js index d0790c5c..e44e5e96 100644 --- a/idr_gallery/static/idr_gallery/omero_search_form.js +++ b/idr_gallery/static/idr_gallery/omero_search_form.js @@ -10,8 +10,8 @@ const AND_CLAUSE_HTML = `
From 3e38c34556094a86e9455bec8d16a68f9062e35f Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 9 Dec 2022 10:13:48 +0000 Subject: [PATCH 22/40] Show operator in auto-complete. Set operator if not 'Any' key --- .../static/idr_gallery/omero_search_form.js | 55 +++++++++++++++---- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/idr_gallery/static/idr_gallery/omero_search_form.js b/idr_gallery/static/idr_gallery/omero_search_form.js index e44e5e96..505844cc 100644 --- a/idr_gallery/static/idr_gallery/omero_search_form.js +++ b/idr_gallery/static/idr_gallery/omero_search_form.js @@ -70,7 +70,7 @@ const displayTypes = { }; // projects or screens might match Name or Description. -function mapNames(rsp, type, key, searchTerm) { +function mapNames(rsp, type, key, searchTerm, operator) { // rsp is a list of [ {id, name, description}, ] searchTerm = searchTerm.toLowerCase(); @@ -117,7 +117,7 @@ function mapNames(rsp, type, key, searchTerm) { return { key: attribute, - label: `${name} (${attribute}) 1 ${displayTypes[type]}`, + label: `${attribute} ${operator} ${name} (1 ${displayTypes[type]})`, value, dtype: type, }; @@ -148,7 +148,7 @@ function autocompleteSort(queryVal, knownKeys = []) { }; } -async function getAutoCompleteResults(key, query, knownKeys) { +async function getAutoCompleteResults(key, query, knownKeys, operator) { let params = { value: query }; if (key != "Any") { params.key = key; @@ -189,16 +189,15 @@ async function getAutoCompleteResults(key, query, knownKeys) { data_results.sort(autocompleteSort(query, knownKeys)); results = data_results.map((result) => { - let showKey = key === "Any" ? `(${result.Key})` : ""; let type = result.type; let count = result.count; return { key: result.Key, - label: `${ + label: `${result.Key} ${operator} ${ result.Value - } ${showKey} ${count} ${type}${ + } (${count} ${type}${ count != 1 ? "s" : "" - }`, + })`, value: `${result.Value}`, dtype: type, }; @@ -209,7 +208,8 @@ async function getAutoCompleteResults(key, query, knownKeys) { responses[1].project, "project", key, - query + query, + operator ); const screenNameHits = mapNames(responses[1].screen, "screen", key, query); const nameHits = projectNameHits.concat(screenNameHits); @@ -255,6 +255,22 @@ async function getAutoCompleteResults(key, query, knownKeys) { results = [{ label: "No results found.", value: -1 }]; } + // If not "Any", add an option to search for contains the currently typed query + if (key != "Any") { + let total = keyCounts[key].count; + let type = keyCounts[key].type; + const allOption = { + key: key, + label: `${key} contains ${query} ${total} ${type}${ + total != 1 ? "s" : "" + }`, + value: query, + dtype: type, + operator: "contains", + }; + results.unshift(allOption); + } + return results; } @@ -476,9 +492,18 @@ class OmeroSearchForm { source: async function (request, response) { // Need to know what Attribute is of adjacent if not there; + let $select = $(".condition", $parent); + $select.val(operator); + } + displayHideRemoveButtons() { let $btns = $(".remove_row", this.$form); $btns.each(function (index, btn) { From 4aaa8e6c1593165ef4e9100f423f5a56d98e5209 Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 9 Dec 2022 13:11:21 +0000 Subject: [PATCH 23/40] Front page filter studies by Name, not by annotation.csv --- idr_gallery/static/idr_gallery/model.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/idr_gallery/static/idr_gallery/model.js b/idr_gallery/static/idr_gallery/model.js index 7bf839b1..9a93c823 100644 --- a/idr_gallery/static/idr_gallery/model.js +++ b/idr_gallery/static/idr_gallery/model.js @@ -379,7 +379,7 @@ class StudiesModel { } filterStudiesAnyText(text) { - // Search for studies with text in their keys, values, or description. + // Search for studies with text in their keys, values, name or description. // Returns a list of matching studies. Each study is returned along with kvps that match text // [study, [{key: value}, {Description: this study is great}]] @@ -393,6 +393,13 @@ class StudiesModel { let keyValuePairs = []; if (study.mapValues) { keyValuePairs = [...study.mapValues]; + + // Don't want to find "annotation.csv" KVPs + keyValuePairs = keyValuePairs.filter( + (kvp) => !kvp[1].includes("annotation.csv") + ); + + keyValuePairs.push(["Name", study.Name]); } keyValuePairs.push(["Description", study.StudyDescription]); let match = keyValuePairs.some((kvp) => regex.test(kvp[1])); From b63d6c49e9c3896c249c67d2ee65d3218d0b065f Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 14 Dec 2022 13:38:40 +0000 Subject: [PATCH 24/40] Fix result.count when summarising autocomplete for 'name' --- .../static/idr_gallery/omero_search_form.js | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/idr_gallery/static/idr_gallery/omero_search_form.js b/idr_gallery/static/idr_gallery/omero_search_form.js index 505844cc..145e890d 100644 --- a/idr_gallery/static/idr_gallery/omero_search_form.js +++ b/idr_gallery/static/idr_gallery/omero_search_form.js @@ -75,13 +75,18 @@ function mapNames(rsp, type, key, searchTerm, operator) { searchTerm = searchTerm.toLowerCase(); // use_description not enabled yet (see below) - // if (key == "Description") { + // if (key == "description") { // // results from resources/all/names/?use_description=true will include searches by name // // need to check they really match description. // rsp = rsp.filter((resultObj) => { // return resultObj.description.toLowerCase().includes(searchTerm); // }); // } + // Need to filter out containers without images + rsp = rsp.filter((resultObj) => { + return !(resultObj.no_images === 0); + }); + return rsp.map((resultObj) => { let name = resultObj.name; let desc = resultObj.description; @@ -91,10 +96,10 @@ function mapNames(rsp, type, key, searchTerm, operator) { if (attribute == "Any") { attribute = name.toLowerCase().includes(searchTerm) ? NAME_KEY - : "Description"; + : "description"; } let value = name; - if (attribute == "Description") { + if (attribute == "description") { // truncate Description around matching word... let start = desc.toLowerCase().indexOf(searchTerm); let targetLength = 80; @@ -119,6 +124,7 @@ function mapNames(rsp, type, key, searchTerm, operator) { key: attribute, label: `${attribute} ${operator} ${name} (1 ${displayTypes[type]})`, value, + count: 1, dtype: type, }; }); @@ -157,11 +163,11 @@ async function getAutoCompleteResults(key, query, knownKeys, operator) { let kvp_url = `${SEARCH_ENGINE_URL}resources/all/searchvalues/?` + params; let urls = [kvp_url]; - if (key == "Any" || key == "Description" || key == NAME_KEY) { + if (key == "Any" || key == "description" || key == NAME_KEY) { // Need to load data from 2 end-points let names_url = `${SEARCH_ENGINE_URL}resources/all/names/?value=${query}`; - // NB: Don't show auto-complete for Description yet (not supported for search yet) - // if (key == "Any" || key == "Description") { + // NB: Don't show auto-complete for Description yet - issues with 'equals' search + // if (key == "Any" || key == "description") { // names_url += `&use_description=true`; // } urls.push(names_url); @@ -200,6 +206,7 @@ async function getAutoCompleteResults(key, query, knownKeys, operator) { })`, value: `${result.Value}`, dtype: type, + count, }; }); // If we searched the 2nd Name/Description endpoint, concat the results... @@ -220,19 +227,20 @@ async function getAutoCompleteResults(key, query, knownKeys, operator) { // filter to remove annotation.csv KV pairs results = results.filter((item) => !item.value.includes("annotation.csv")); - // Generate Summary of [{key: "name", hits: 5, type: container}, {key: "Gene Symbol", hits: 1000, type: image}} + // Generate Summary of [{key: "name", count: 5, type: container, matches:[]} } let keyCounts = {}; - data_results.forEach((result) => { - if (!keyCounts[result.Key]) { - keyCounts[result.Key] = { - key: result.Key, + results.forEach((result) => { + let key = result.key; + if (!keyCounts[key]) { + keyCounts[key] = { + key: key, count: 0, - type: result.type, + type: result.dtype, matches: [], }; } - keyCounts[result.Key].count += result.count; - keyCounts[result.Key].matches.push(result); + keyCounts[key].count += result.count; + keyCounts[key].matches.push(result); }); let keyCountsList = Object.values(keyCounts); keyCountsList.sort((a, b) => @@ -241,8 +249,8 @@ async function getAutoCompleteResults(key, query, knownKeys, operator) { // TODO: we don't use this summary yet... Display to user somehow?? console.log("keyCountsList", keyCountsList); + // truncate list let result_count = results.length; - const max_shown = 100; if (result_count > max_shown) { results = results.slice(0, max_shown); @@ -256,7 +264,7 @@ async function getAutoCompleteResults(key, query, knownKeys, operator) { } // If not "Any", add an option to search for contains the currently typed query - if (key != "Any") { + if (key != "Any" && keyCounts[key]) { let total = keyCounts[key].count; let type = keyCounts[key].type; const allOption = { From da5ea639c6b2a4a27296b10baead703a12a6b94f Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 10 Jan 2023 11:11:39 +0000 Subject: [PATCH 25/40] Handle redirect Name to name --- idr_gallery/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/idr_gallery/views.py b/idr_gallery/views.py index 1b97ffe0..771fe523 100644 --- a/idr_gallery/views.py +++ b/idr_gallery/views.py @@ -68,8 +68,10 @@ def index(request, super_category=None, conn=None, **kwargs): keyval = query.split(":", 1) # search for studies ("containers") and use "contains" # to match previous behaviour + # NB: 'Name' needs to be 'name' for search-engine + key = "name" if keyval[0] == "Name" else keyval[0] return redirect_with_params('idr_gallery_search', - key=keyval[0], + key=key, value=keyval[1], resource="container", operator="contains") From 0f7b81f40b7023f1e3745537d209ac7e18212c4d Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 10 Jan 2023 11:45:55 +0000 Subject: [PATCH 26/40] Use arrow function to avoid prettier conflict local prettier is formatting as }.bind(this)()); but this fails on github actions --- idr_gallery/static/idr_gallery/omero_search_form.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/idr_gallery/static/idr_gallery/omero_search_form.js b/idr_gallery/static/idr_gallery/omero_search_form.js index 145e890d..15dc8a4f 100644 --- a/idr_gallery/static/idr_gallery/omero_search_form.js +++ b/idr_gallery/static/idr_gallery/omero_search_form.js @@ -306,11 +306,11 @@ class OmeroSearchForm { // TODO: wait for loadResources() // then build form... - (async function () { + (async () => { await this.loadResources(); this.addAnd(); this.trigger("ready"); - }.bind(this)()); + })(); } // pub/sub methods. see https://github.com/cowboy/jquery-tiny-pubsub From 5d8c998ab9650d172f3af3b146eb461d0ef79cea Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 24 Jan 2023 16:43:32 +0000 Subject: [PATCH 27/40] Fix projects -> experiments in auto-complete --- idr_gallery/static/idr_gallery/omero_search_form.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/idr_gallery/static/idr_gallery/omero_search_form.js b/idr_gallery/static/idr_gallery/omero_search_form.js index 15dc8a4f..cf71993f 100644 --- a/idr_gallery/static/idr_gallery/omero_search_form.js +++ b/idr_gallery/static/idr_gallery/omero_search_form.js @@ -201,7 +201,7 @@ async function getAutoCompleteResults(key, query, knownKeys, operator) { key: result.Key, label: `${result.Key} ${operator} ${ result.Value - } (${count} ${type}${ + } (${count} ${displayTypes[type]}${ count != 1 ? "s" : "" })`, value: `${result.Value}`, @@ -269,7 +269,8 @@ async function getAutoCompleteResults(key, query, knownKeys, operator) { let type = keyCounts[key].type; const allOption = { key: key, - label: `${key} contains ${query} ${total} ${type}${ + label: `${key} contains + ${query} ${total} ${displayTypes[type]}${ total != 1 ? "s" : "" }`, value: query, From 8b02b160d05342a96d7929e0230235826d454994 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 24 Jan 2023 16:53:09 +0000 Subject: [PATCH 28/40] prettier fix --- idr_gallery/static/idr_gallery/omero_search_form.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/idr_gallery/static/idr_gallery/omero_search_form.js b/idr_gallery/static/idr_gallery/omero_search_form.js index cf71993f..b8237184 100644 --- a/idr_gallery/static/idr_gallery/omero_search_form.js +++ b/idr_gallery/static/idr_gallery/omero_search_form.js @@ -270,9 +270,9 @@ async function getAutoCompleteResults(key, query, knownKeys, operator) { const allOption = { key: key, label: `${key} contains - ${query} ${total} ${displayTypes[type]}${ - total != 1 ? "s" : "" - }`, + ${query} ${total} ${ + displayTypes[type] + }${total != 1 ? "s" : ""}`, value: query, dtype: type, operator: "contains", From 4b6b3d28b0d7fd724f453524861d6ef70fcf9734 Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 25 Jan 2023 13:21:46 +0000 Subject: [PATCH 29/40] Replace 'study' with 'experiment/screen' in autocomplete and results --- .../static/idr_gallery/omero_search_form.js | 15 ++++++++------- idr_gallery/templates/idr_gallery/search.html | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/idr_gallery/static/idr_gallery/omero_search_form.js b/idr_gallery/static/idr_gallery/omero_search_form.js index b8237184..8c5ca600 100644 --- a/idr_gallery/static/idr_gallery/omero_search_form.js +++ b/idr_gallery/static/idr_gallery/omero_search_form.js @@ -63,7 +63,7 @@ const NAME_KEY = "name"; // display this on the keyFields