From 1820e21dd1c6dac1ca48d0b3f36e278eccb4a82e Mon Sep 17 00:00:00 2001 From: baerlach-git <83013270+baerlach-git@users.noreply.github.com> Date: Wed, 7 Jun 2023 13:53:57 +0200 Subject: [PATCH 01/11] configured the picture index in plugin.js, provided a hook to test meilisearch with one search term --- .../helpers/getSearchResultPictureIds.ts | 20 ++++ projects/bp-strapi/config/plugins.js | 92 +++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts diff --git a/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts b/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts new file mode 100644 index 000000000..9e2a7f09f --- /dev/null +++ b/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts @@ -0,0 +1,20 @@ +import { MeiliSearch } from 'meilisearch'; + +const getSearchResultPictureIds = async (searchparams: { + searchTerms: string[]; + searchTimes: string[][]; +}) => { + const client = new MeiliSearch({ + host: 'localhost:7700', + apiKey: '', + }); + + const index = client.index('picture'); + + const searchResult = await index.search(searchparams.searchTerms[0]); + const SearchResultPictureIds = searchResult.hits.map(hit => hit.id); + + return SearchResultPictureIds; +}; + +export default getSearchResultPictureIds; diff --git a/projects/bp-strapi/config/plugins.js b/projects/bp-strapi/config/plugins.js index 0c8eb0592..840a76174 100755 --- a/projects/bp-strapi/config/plugins.js +++ b/projects/bp-strapi/config/plugins.js @@ -1,4 +1,10 @@ /* eslint-disable no-unused-vars */ + +const dateToTimeStamp = (date) => { + const dateInstance = new Date(date); + return dateInstance.getTime(); +}; + module.exports = ({ env }) => ({ // disable i18n (all content is explicitly german as it's a german photo archive) i18n: false, @@ -51,6 +57,92 @@ module.exports = ({ env }) => ({ }, }, }, + meilisearch: + env("MEILISEARCH_ENABLED", "false") === "true" + ? { + config: { + host: "localhost:7700", + apiKey: "", + picture: { + transformEntry({ entry }) { + const transformedEntry = { + id: entry.id, + likes: entry.likes, + descriptions: entry.descriptions.map( + (description) => description.text + ), + comments: entry.comments.map((comment) => comment.text), + keyword_tags: entry.keyword_tags + .map((tag) => tag.name) + .concat(entry.verified_keyword_tags.map((tag) => tag.name)), + person_tags: entry.person_tags + .map((tag) => tag.name) + .concat(entry.verified_person_tags.map((tag) => tag.name)), + location_tags: entry.location_tags + .map((tag) => tag.name) + .concat( + entry.verified_location_tags.map((tag) => tag.name) + ), + face_tags: entry.face_tags.map((tag) => tag.name), + collections: entry.collections.map((tag) => tag.name), + archive_tag: entry.archive_tag, + time_range_tag_start: entry?.time_range_tag + ? dateToTimeStamp(entry.time_range_tag.start) + : entry.verified_time_range_tag + ? dateToTimeStamp(entry.verified_time_range_tag.start) + : null, + time_range_tag_end: entry?.time_range_tag + ? dateToTimeStamp(entry.time_range_tag.end) + : entry.verified_time_range_tag + ? dateToTimeStamp(entry.verified_time_range_tag.end) + : null, + }; + + return transformedEntry; + }, + settings: { + filterableAttributes: [ + "keyword_tags", + "location_tags", + "time_range_tag_start", + "time_range_tag_end", + "face_tags", + "person_tags", + "descriptions", + "comments", + "collections", + "archive_tag", + "is_text", + ], + sortableAttributes: [ + "time_range_tag_start", + "time_range_tag_end", + "likes", + ], + displayedAttributes: ["id"], + searchableAttributes: [ + "keyword_tags", + "location_tags", + "time_range_tag_start", + "time_range_tag_end", + "face_tags", + "person_tags", + "descriptions", + "comments", + "collections", + "archive_tag", + ], + typoTolerance: { + enabled: true, + minWordSizeForTypos: { oneTypo: 3, twoTypos: 4 }, + diableOnWords: [], + disableOnAttributes: [], + }, + }, + }, + }, + } + : null, upload: env("AWS_ENABLED", "false") === "true" ? { From d60e87bf210c347d9ce905375783fcfcb625605e Mon Sep 17 00:00:00 2001 From: baerlach-git <83013270+baerlach-git@users.noreply.github.com> Date: Wed, 7 Jun 2023 15:51:22 +0200 Subject: [PATCH 02/11] minor correction in the config, added the necessary packages for frontend and backend --- projects/bp-gallery/package.json | 3 +- projects/bp-gallery/yarn.lock | 21 ++++++++++++++ projects/bp-strapi/config/plugins.js | 7 +++-- projects/bp-strapi/package.json | 1 + projects/bp-strapi/yarn.lock | 43 +++++++++++++++++++++++++++- 5 files changed, 70 insertions(+), 5 deletions(-) diff --git a/projects/bp-gallery/package.json b/projects/bp-gallery/package.json index 4bbef3148..a3efe5692 100755 --- a/projects/bp-gallery/package.json +++ b/projects/bp-gallery/package.json @@ -29,11 +29,12 @@ "jodit-react": "^1.3.31", "leaflet": "^1.9.3", "lodash": "^4.17.21", + "meilisearch": "^0.33.0", "nanoid": "^4.0.2", "node-sass": "^8.0.0", "puppeteer": "^19.4.0", - "react-currency-input-field": "^3.6.10", "react": "^18.2.0", + "react-currency-input-field": "^3.6.10", "react-date-range": "^1.4.0", "react-device-detect": "^2.2.2", "react-dom": "^18.2.0", diff --git a/projects/bp-gallery/yarn.lock b/projects/bp-gallery/yarn.lock index d9883dd6f..1cdbf9f4e 100755 --- a/projects/bp-gallery/yarn.lock +++ b/projects/bp-gallery/yarn.lock @@ -3422,6 +3422,13 @@ cross-fetch@3.1.5, cross-fetch@^3.1.5: dependencies: node-fetch "2.6.7" +cross-fetch@^3.1.6: + version "3.1.6" + resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.6.tgz#bae05aa31a4da760969756318feeee6e70f15d6c" + integrity sha512-riRvo06crlE8HiqOwIpQhxwdOk4fOeR7FVM/wXoxchFEqMNUjvbs3bfo4OTgMEMHzppd4DxFBDbyySj8Cv781g== + dependencies: + node-fetch "^2.6.11" + cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" @@ -5978,6 +5985,13 @@ map-stream@~0.1.0: resolved "https://registry.yarnpkg.com/map-stream/-/map-stream-0.1.0.tgz#e56aa94c4c8055a16404a0674b78f215f7c8e194" integrity sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g== +meilisearch@^0.33.0: + version "0.33.0" + resolved "https://registry.yarnpkg.com/meilisearch/-/meilisearch-0.33.0.tgz#25982b193cdd22e9ec534a022dbde89c42951dc4" + integrity sha512-bYPb9WyITnJfzf92e7QFK8Rc50DmshFWxypXCs3ILlpNh8pT15A7KSu9Xgnnk/K3G/4vb3wkxxtFS4sxNkWB8w== + dependencies: + cross-fetch "^3.1.6" + meow@^9.0.0: version "9.0.0" resolved "https://registry.yarnpkg.com/meow/-/meow-9.0.0.tgz#cd9510bc5cac9dee7d03c73ee1f9ad959f4ea364" @@ -6251,6 +6265,13 @@ node-fetch@2.6.7, node-fetch@^2.6.1, node-fetch@^2.6.7: dependencies: whatwg-url "^5.0.0" +node-fetch@^2.6.11: + version "2.6.11" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.11.tgz#cde7fc71deef3131ef80a738919f999e6edfff25" + integrity sha512-4I6pdBY1EthSqDmJkiNk3JIT8cswwR9nfeW/cPdUagJYEQG7R95WRH74wpz7ma8Gh/9dI9FP+OU+0E4FvtA55w== + dependencies: + whatwg-url "^5.0.0" + node-gyp@^8.4.1: version "8.4.1" resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-8.4.1.tgz#3d49308fc31f768180957d6b5746845fbd429937" diff --git a/projects/bp-strapi/config/plugins.js b/projects/bp-strapi/config/plugins.js index 840a76174..29657abe6 100755 --- a/projects/bp-strapi/config/plugins.js +++ b/projects/bp-strapi/config/plugins.js @@ -1,8 +1,7 @@ /* eslint-disable no-unused-vars */ const dateToTimeStamp = (date) => { - const dateInstance = new Date(date); - return dateInstance.getTime(); + return Date.parse(date) / 1000; }; module.exports = ({ env }) => ({ @@ -120,6 +119,8 @@ module.exports = ({ env }) => ({ "likes", ], displayedAttributes: ["id"], + // the order of the attributes in searchableAttributes determines the priotization + // of search results i.e. a match in the first searchable attribute will always outrank a match in any other searchable attribute searchableAttributes: [ "keyword_tags", "location_tags", @@ -135,7 +136,7 @@ module.exports = ({ env }) => ({ typoTolerance: { enabled: true, minWordSizeForTypos: { oneTypo: 3, twoTypos: 4 }, - diableOnWords: [], + disableOnWords: [], disableOnAttributes: [], }, }, diff --git a/projects/bp-strapi/package.json b/projects/bp-strapi/package.json index 081211905..6091ca994 100755 --- a/projects/bp-strapi/package.json +++ b/projects/bp-strapi/package.json @@ -30,6 +30,7 @@ "pg": "8.6.0", "pm2": "^5.2.0", "sendmail": "^1.6.1", + "strapi-plugin-meilisearch": "^0.9.2", "strapi-provider-upload-aws-s3-advanced": "^5.0.1", "xlsx": "^0.18.5" }, diff --git a/projects/bp-strapi/yarn.lock b/projects/bp-strapi/yarn.lock index eb916c03b..e47b71bd6 100755 --- a/projects/bp-strapi/yarn.lock +++ b/projects/bp-strapi/yarn.lock @@ -2156,6 +2156,13 @@ dependencies: regenerator-runtime "^0.13.4" +"@babel/runtime@^7.21.0": + version "7.22.3" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.22.3.tgz#0a7fce51d43adbf0f7b517a71f4c3aaca92ebcbb" + integrity sha512-XsDuspWKLUsxwCp6r7EhsExHtYfbe5oAGQ19kqngTdCPUoPQzOPdUbD/pB9PJiwb2ptYKQDjSJT3R6dC+EPqfQ== + dependencies: + regenerator-runtime "^0.13.11" + "@babel/template@7.18.10": version "7.18.10" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.18.10.tgz#6f9134835970d1dbf0835c0d100c9f38de0c5e71" @@ -3555,6 +3562,18 @@ lodash "4.17.21" yup "0.32.9" +"@strapi/utils@^4.5.4": + version "4.10.7" + resolved "https://registry.yarnpkg.com/@strapi/utils/-/utils-4.10.7.tgz#2cf1aa5958c353e1a1b2e7d3de5e55eff1006cbc" + integrity sha512-LyTcpsdcJ6Dq/33eblSU4pN4Yb9VnARA6K5/66cmTTQH34C2bKqkozm7wjK+Y1smK5HA2nvHQiZfpoDtqI0NmA== + dependencies: + "@sindresorhus/slugify" "1.1.0" + date-fns "2.30.0" + http-errors "1.8.1" + lodash "4.17.21" + p-map "4.0.0" + yup "0.32.9" + "@swc/helpers@^0.4.14": version "0.4.14" resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.4.14.tgz#1352ac6d95e3617ccb7c1498ff019654f1e12a74" @@ -5876,7 +5895,7 @@ cross-env@^7.0.3: dependencies: cross-spawn "^7.0.1" -cross-fetch@^3.1.6: +cross-fetch@^3.1.5, cross-fetch@^3.1.6: version "3.1.6" resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.6.tgz#bae05aa31a4da760969756318feeee6e70f15d6c" integrity sha512-riRvo06crlE8HiqOwIpQhxwdOk4fOeR7FVM/wXoxchFEqMNUjvbs3bfo4OTgMEMHzppd4DxFBDbyySj8Cv781g== @@ -6006,6 +6025,13 @@ date-fns@2.29.2: resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.29.2.tgz#0d4b3d0f3dff0f920820a070920f0d9662c51931" integrity sha512-0VNbwmWJDS/G3ySwFSJA3ayhbURMTJLtwM2DTxf9CWondCnh6DTNlO9JgRSq6ibf4eD0lfMJNBxUdEAHHix+bA== +date-fns@2.30.0: + version "2.30.0" + resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.30.0.tgz#f367e644839ff57894ec6ac480de40cae4b0f4d0" + integrity sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw== + dependencies: + "@babel/runtime" "^7.21.0" + dayjs@~1.8.24, dayjs@~1.8.25: version "1.8.36" resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.8.36.tgz#be36e248467afabf8f5a86bae0de0cdceecced50" @@ -9272,6 +9298,13 @@ media-typer@0.3.0: resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" integrity sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g= +meilisearch@^0.30.0: + version "0.30.0" + resolved "https://registry.yarnpkg.com/meilisearch/-/meilisearch-0.30.0.tgz#707f9a6b07440c496b965379616e084f112160ae" + integrity sha512-3y1hALOwTDpquYar+gDREqRasFPWKxkWAhk6h+RF+nKObPVf9N77wcTNvukGwOKbxRyJnKge0OPgAB1BkB9VVw== + dependencies: + cross-fetch "^3.1.5" + memfs@^3.4.1, memfs@^3.4.3: version "3.4.13" resolved "https://registry.yarnpkg.com/memfs/-/memfs-3.4.13.tgz#248a8bd239b3c240175cd5ec548de5227fc4f345" @@ -12088,6 +12121,14 @@ std-env@^3.0.1: resolved "https://registry.yarnpkg.com/std-env/-/std-env-3.3.1.tgz#93a81835815e618c8aa75e7c8a4dc04f7c314e29" integrity sha512-3H20QlwQsSm2OvAxWIYhs+j01MzzqwMwGiiO1NQaJYZgJZFPuAbf95/DiKRBSTYIJ2FeGUc+B/6mPGcWP9dO3Q== +strapi-plugin-meilisearch@^0.9.2: + version "0.9.2" + resolved "https://registry.yarnpkg.com/strapi-plugin-meilisearch/-/strapi-plugin-meilisearch-0.9.2.tgz#c8db12820b86640b1bdc850dde94da9c50e9b6e8" + integrity sha512-18rAUkQ5LTVYGKqe0bslKpMPC3nd/VpQNFAFkydTqhONkdPI/m1SkRnudaFujMspTB7+fqcAhrdGOeHTH/g3cQ== + dependencies: + "@strapi/utils" "^4.5.4" + meilisearch "^0.30.0" + strapi-provider-upload-aws-s3-advanced@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/strapi-provider-upload-aws-s3-advanced/-/strapi-provider-upload-aws-s3-advanced-5.0.1.tgz#8e0de23920ecb7478beeb4e2c62cf42dd2cd6296" From 875a8d4e841ba7cd31eb685b66a0dca5a9d7d3db Mon Sep 17 00:00:00 2001 From: baerlach-git <83013270+baerlach-git@users.noreply.github.com> Date: Tue, 13 Jun 2023 14:37:11 +0200 Subject: [PATCH 03/11] finished the search helper --- .../components/views/search/SearchView.tsx | 45 +++++++-------- .../helpers/getSearchResultPictureIds.ts | 57 ++++++++++++++++--- 2 files changed, 70 insertions(+), 32 deletions(-) diff --git a/projects/bp-gallery/src/components/views/search/SearchView.tsx b/projects/bp-gallery/src/components/views/search/SearchView.tsx index 8d86dc594..c220c6da9 100644 --- a/projects/bp-gallery/src/components/views/search/SearchView.tsx +++ b/projects/bp-gallery/src/components/views/search/SearchView.tsx @@ -12,11 +12,8 @@ import SearchBreadcrumbs from './SearchBreadcrumbs'; import SearchHub from './SearchHub'; import './SearchView.scss'; import { isValidYear } from './helpers/addNewParamToSearchPath'; -import { - SearchType, - convertSearchParamsToPictureFilters, - paramToTime, -} from './helpers/search-filters'; +import getSearchResultPictureIds from './helpers/getSearchResultPictureIds'; +import { SearchType, paramToTime } from './helpers/search-filters'; import { toURLSearchParam } from './helpers/url-search-params'; const isValidTimeSpecification = (searchRequest: string) => { @@ -47,25 +44,25 @@ const SearchView = () => { // Builds query from search params in the path const queryParams = useMemo(() => { - if (isAllSearchActive) { - const allSearchTerms = searchParams - .getAll(toURLSearchParam(SearchType.ALL)) - .map(decodeURIComponent); - const searchTimes: string[][] = []; - allSearchTerms.forEach(searchTerm => { - if (isValidTimeSpecification(searchTerm)) { - const { startTime, endTime } = paramToTime(searchTerm); - searchTimes.push([searchTerm, startTime, endTime]); - } - }); - return { - searchTerms: allSearchTerms.filter(searchTerm => !isValidTimeSpecification(searchTerm)), - searchTimes, - }; - } - return convertSearchParamsToPictureFilters(searchParams); - }, [isAllSearchActive, searchParams]); - + // if (isAllSearchActive) { + const allSearchTerms = searchParams + .getAll(toURLSearchParam(SearchType.ALL)) + .map(decodeURIComponent); + const searchTimes: string[][] = []; + allSearchTerms.forEach(searchTerm => { + if (isValidTimeSpecification(searchTerm)) { + const { startTime, endTime } = paramToTime(searchTerm); + searchTimes.push([searchTerm, startTime, endTime]); + } + }); + return { + searchTerms: allSearchTerms.filter(searchTerm => !isValidTimeSpecification(searchTerm)), + searchTimes, + }; + // } + // return convertSearchParamsToPictureFilters(searchParams); + }, [/*isAllSearchActive,*/ searchParams]); + console.log(getSearchResultPictureIds(queryParams)); const { linkToCollection, bulkEdit } = useBulkOperations(); return ( diff --git a/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts b/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts index 9e2a7f09f..a1ccc240b 100644 --- a/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts +++ b/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts @@ -1,20 +1,61 @@ import { MeiliSearch } from 'meilisearch'; -const getSearchResultPictureIds = async (searchparams: { - searchTerms: string[]; - searchTimes: string[][]; -}) => { +const getSearchResultPictureIds = async ( + searchparams: { + searchTerms: string[]; + searchTimes: string[][]; + }, + filter: string +) => { const client = new MeiliSearch({ host: 'localhost:7700', apiKey: '', }); - const index = client.index('picture'); - const searchResult = await index.search(searchparams.searchTerms[0]); - const SearchResultPictureIds = searchResult.hits.map(hit => hit.id); + filter = typeof filter === 'undefined' ? '' : filter; + + const searchTerms = searchparams.searchTerms; + const searchTimes = searchparams.searchTimes; + + const TIME_RANGE_START = 'time_range_tag_start'; + const TIME_RANGE_END = 'time_range_tag_end'; + + if (searchTimes && searchTimes.length !== 0) { + let timeFilters = []; + for (let i = 0; i++; i < searchTimes.length) { + timeFilters[ + i + ] = `(${searchTimes[i][1]} <= ${TIME_RANGE_START} AND ${TIME_RANGE_END} <= ${searchTimes[i][2]})`; + } + const timeFilter = timeFilters.join(' AND '); + filter.concat(' AND ', timeFilter); + } + + const RESULT_LIMIT = 1000; + + const settings = { limit: RESULT_LIMIT, showMatchesPosition: true, filter: filter }; - return SearchResultPictureIds; + if (searchTerms && searchTerms.length !== 0) { + let searchTermResults = []; + for (let i = 0; i < searchTerms.length; i++) { + searchTermResults[i] = await index + .search(searchparams.searchTerms[i], settings) + .then(value => value.hits.map(hit => hit.id).flat()); + } + let resultIntersection = searchTermResults[0]; + for (let i = 1; i < searchTermResults.length; i++) { + resultIntersection = resultIntersection.filter((value: any) => + searchTermResults[i].includes(value) + ); + } + return resultIntersection; + } else if (!searchTerms || searchTerms.length === 0) { + const searchResultForEmptySearch = await index + .search('', settings) + .then(value => value.hits.map(hit => hit.id).flat()); + return searchResultForEmptySearch; + } }; export default getSearchResultPictureIds; From 1fc06691290c5b933055ee9dc7dd901cd8c0c1d5 Mon Sep 17 00:00:00 2001 From: baerlach-git <83013270+baerlach-git@users.noreply.github.com> Date: Fri, 16 Jun 2023 12:56:28 +0200 Subject: [PATCH 04/11] plugin configuration is done, for now. The helper function should work just fine --- .../helpers/getSearchResultPictureIds.ts | 65 +++++++++---------- projects/bp-strapi/config/plugins.js | 54 +++++++++++---- 2 files changed, 72 insertions(+), 47 deletions(-) diff --git a/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts b/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts index a1ccc240b..485ffe428 100644 --- a/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts +++ b/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts @@ -1,5 +1,9 @@ import { MeiliSearch } from 'meilisearch'; +const dateToTimeStamp = (date: string) => { + return Date.parse(date) / 1000; +}; + const getSearchResultPictureIds = async ( searchparams: { searchTerms: string[]; @@ -13,47 +17,42 @@ const getSearchResultPictureIds = async ( }); const index = client.index('picture'); - filter = typeof filter === 'undefined' ? '' : filter; - const searchTerms = searchparams.searchTerms; const searchTimes = searchparams.searchTimes; const TIME_RANGE_START = 'time_range_tag_start'; const TIME_RANGE_END = 'time_range_tag_end'; - - if (searchTimes && searchTimes.length !== 0) { - let timeFilters = []; - for (let i = 0; i++; i < searchTimes.length) { - timeFilters[ - i - ] = `(${searchTimes[i][1]} <= ${TIME_RANGE_START} AND ${TIME_RANGE_END} <= ${searchTimes[i][2]})`; - } - const timeFilter = timeFilters.join(' AND '); - filter.concat(' AND ', timeFilter); + // when building a filter for meilisearch the filtered attribute always + // has to come first i.e. time_range_start >= 0 works but 0 <= time_range_start does not work + if (searchTimes.length !== 0) { + const timeFilters = searchTimes.map( + searchTime => + `(${TIME_RANGE_START} >= ${dateToTimeStamp( + searchTime[1] + )} AND ${TIME_RANGE_END} <= ${dateToTimeStamp(searchTime[2])})` + ); + + const timeFilter = timeFilters.join(' OR '); + filter = filter === '' ? timeFilter : filter.concat(' AND ', timeFilter); } const RESULT_LIMIT = 1000; - - const settings = { limit: RESULT_LIMIT, showMatchesPosition: true, filter: filter }; - - if (searchTerms && searchTerms.length !== 0) { - let searchTermResults = []; - for (let i = 0; i < searchTerms.length; i++) { - searchTermResults[i] = await index - .search(searchparams.searchTerms[i], settings) - .then(value => value.hits.map(hit => hit.id).flat()); - } - let resultIntersection = searchTermResults[0]; - for (let i = 1; i < searchTermResults.length; i++) { - resultIntersection = resultIntersection.filter((value: any) => - searchTermResults[i].includes(value) - ); - } - return resultIntersection; - } else if (!searchTerms || searchTerms.length === 0) { - const searchResultForEmptySearch = await index - .search('', settings) - .then(value => value.hits.map(hit => hit.id).flat()); + // this makes it so only documents that match all of the query terms are returned + const MATCHING_STRATEGY = 'all'; + + const settings = { + limit: RESULT_LIMIT, + showMatchesPosition: true, + matchingStrategy: MATCHING_STRATEGY, + filter: filter, + }; + + if (searchTerms.length !== 0) { + const query = searchTerms.join(' '); + const searchResult = await index.search(query, settings).then(value => value.hits); + return searchResult; + } else { + const searchResultForEmptySearch = await index.search('', settings).then(value => value.hits); return searchResultForEmptySearch; } }; diff --git a/projects/bp-strapi/config/plugins.js b/projects/bp-strapi/config/plugins.js index 29657abe6..31af424a3 100755 --- a/projects/bp-strapi/config/plugins.js +++ b/projects/bp-strapi/config/plugins.js @@ -60,8 +60,8 @@ module.exports = ({ env }) => ({ env("MEILISEARCH_ENABLED", "false") === "true" ? { config: { - host: "localhost:7700", - apiKey: "", + host: env("MEILISEARCH_HOST"), + apiKey: env("MEILISEARCH_API_KEY"), picture: { transformEntry({ entry }) { const transformedEntry = { @@ -100,28 +100,22 @@ module.exports = ({ env }) => ({ return transformedEntry; }, settings: { - filterableAttributes: [ + displayedAttributes: ["id"], + // the order of the attributes in searchableAttributes determines the priorization + // of search results i.e. a match in the first searchable attribute will always outrank a match in any other searchable attribute + searchableAttributes: [ + "descriptions", "keyword_tags", "location_tags", "time_range_tag_start", "time_range_tag_end", "face_tags", "person_tags", - "descriptions", "comments", "collections", "archive_tag", - "is_text", ], - sortableAttributes: [ - "time_range_tag_start", - "time_range_tag_end", - "likes", - ], - displayedAttributes: ["id"], - // the order of the attributes in searchableAttributes determines the priotization - // of search results i.e. a match in the first searchable attribute will always outrank a match in any other searchable attribute - searchableAttributes: [ + filterableAttributes: [ "keyword_tags", "location_tags", "time_range_tag_start", @@ -132,13 +126,45 @@ module.exports = ({ env }) => ({ "comments", "collections", "archive_tag", + "is_text", + ], + sortableAttributes: [ + "time_range_tag_start", + "time_range_tag_end", + "likes", ], + rankingRules: [ + "words", + "typo", + "proximity", + "attribute", + "sort", + "exactness", + ], + // words that are ignored during searches, useful for common words + // that do not carry a meaning on their own like articles, pronomina etc. + // we do not use this setting, since our data on user searchers suggests, that + // users only search for proper names, people, locations and nouns in general + stopWords: [], + synonyms: {}, + // returned documents will always be unigue in this attribute + distinctAttribute: null, typoTolerance: { enabled: true, minWordSizeForTypos: { oneTypo: 3, twoTypos: 4 }, disableOnWords: [], disableOnAttributes: [], }, + // faceting is currently not in use + faceting: { + maxValuesPerFacet: 100, + }, + // maxtotalHits determines the maximal possible amount + // of search results and overrides any other settings + // like the result_limit of the search settings in this regard + pagination: { + maxTotalHits: 1000, + }, }, }, }, From 965be6cdb7a8ab9a9bea8fd25ec8ded57829a9da Mon Sep 17 00:00:00 2001 From: baerlach-git <83013270+baerlach-git@users.noreply.github.com> Date: Fri, 16 Jun 2023 13:07:18 +0200 Subject: [PATCH 05/11] forgot to add the config file in the last commit --- projects/bp-strapi/config/plugins.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/bp-strapi/config/plugins.js b/projects/bp-strapi/config/plugins.js index 31af424a3..62c0f4e3e 100755 --- a/projects/bp-strapi/config/plugins.js +++ b/projects/bp-strapi/config/plugins.js @@ -111,9 +111,9 @@ module.exports = ({ env }) => ({ "time_range_tag_end", "face_tags", "person_tags", - "comments", "collections", "archive_tag", + "comments", ], filterableAttributes: [ "keyword_tags", From 7211cc5ca9078a231ef294fd4da0025d96348681 Mon Sep 17 00:00:00 2001 From: baerlach-git <83013270+baerlach-git@users.noreply.github.com> Date: Fri, 16 Jun 2023 15:09:18 +0200 Subject: [PATCH 06/11] Update projects/bp-gallery/src/components/views/search/SearchView.tsx Co-authored-by: Oliver Schulz <64468276+olschulz@users.noreply.github.com> --- projects/bp-gallery/src/components/views/search/SearchView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/bp-gallery/src/components/views/search/SearchView.tsx b/projects/bp-gallery/src/components/views/search/SearchView.tsx index c220c6da9..9c78e4f90 100644 --- a/projects/bp-gallery/src/components/views/search/SearchView.tsx +++ b/projects/bp-gallery/src/components/views/search/SearchView.tsx @@ -62,7 +62,7 @@ const SearchView = () => { // } // return convertSearchParamsToPictureFilters(searchParams); }, [/*isAllSearchActive,*/ searchParams]); - console.log(getSearchResultPictureIds(queryParams)); + if (import.meta.env.MODE === 'development') console.log(getSearchResultPictureIds(queryParams, '')); const { linkToCollection, bulkEdit } = useBulkOperations(); return ( From e84241a06506ccfea40ccc58da9610f8f3094e68 Mon Sep 17 00:00:00 2001 From: baerlach-git <83013270+baerlach-git@users.noreply.github.com> Date: Fri, 16 Jun 2023 15:26:38 +0200 Subject: [PATCH 07/11] commit for fun --- projects/bp-gallery/.eslintrc | 3 ++- .../bp-gallery/environments/.env.development | 2 +- .../components/views/search/SearchView.tsx | 3 ++- projects/bp-gallery/vite.config.ts | 5 ++-- projects/bp-strapi/package.json | 1 + projects/bp-strapi/yarn.lock | 26 +++++++++++++++++++ 6 files changed, 34 insertions(+), 6 deletions(-) diff --git a/projects/bp-gallery/.eslintrc b/projects/bp-gallery/.eslintrc index 49e2387d9..51c22730a 100755 --- a/projects/bp-gallery/.eslintrc +++ b/projects/bp-gallery/.eslintrc @@ -20,7 +20,8 @@ }, "parser": "@typescript-eslint/parser", "parserOptions": { - "project": ["./tsconfig.json", "./cypress/tsconfig.json", "./vite.config.ts"] + "project": ["./tsconfig.json", "./cypress/tsconfig.json", "./vite.config.ts"], + "tsconfigRootDir": "./bp-gallery" }, "plugins": [ "react", diff --git a/projects/bp-gallery/environments/.env.development b/projects/bp-gallery/environments/.env.development index 69714b97d..31962ffc3 100644 --- a/projects/bp-gallery/environments/.env.development +++ b/projects/bp-gallery/environments/.env.development @@ -1,4 +1,4 @@ -VITE_REACT_APP_API_BASE=/api +VITE_REACT_APP_API_BASE=http://localhost:9000 VITE_REACT_APP_CONTACT_MAIL=archiv@bad-harzburg-stiftung.de VITE_REACT_APP_ADVANCED_SEARCH=false VITE_REACT_APP_GROWTHBOOK_APIHOST=https://growthbook.harz-history.de/proxy diff --git a/projects/bp-gallery/src/components/views/search/SearchView.tsx b/projects/bp-gallery/src/components/views/search/SearchView.tsx index 9c78e4f90..44227fdd8 100644 --- a/projects/bp-gallery/src/components/views/search/SearchView.tsx +++ b/projects/bp-gallery/src/components/views/search/SearchView.tsx @@ -62,7 +62,8 @@ const SearchView = () => { // } // return convertSearchParamsToPictureFilters(searchParams); }, [/*isAllSearchActive,*/ searchParams]); - if (import.meta.env.MODE === 'development') console.log(getSearchResultPictureIds(queryParams, '')); + if (import.meta.env.MODE === 'development') + console.log(getSearchResultPictureIds(queryParams, '')); const { linkToCollection, bulkEdit } = useBulkOperations(); return ( diff --git a/projects/bp-gallery/vite.config.ts b/projects/bp-gallery/vite.config.ts index 96f6f6c02..1de0ffa8e 100644 --- a/projects/bp-gallery/vite.config.ts +++ b/projects/bp-gallery/vite.config.ts @@ -1,8 +1,7 @@ import react from '@vitejs/plugin-react-swc'; -import { defineConfig } from 'vite'; import type { Plugin, PluginBuild } from 'esbuild'; import path from 'path'; -import eslint from 'vite-plugin-eslint'; +import { defineConfig } from 'vite'; const splitPackages = ['@mui/icons-material', '@mui/material']; const reactPlugins = [ @@ -29,7 +28,7 @@ const splitPackagesPlugin: Plugin = { // https://vitejs.dev/config/ export default defineConfig({ - plugins: [react({ plugins: reactPlugins }), eslint({ cache: false })], + plugins: [react({ plugins: reactPlugins }) /*eslint({ cache: false })*/], envDir: 'environments', server: { port: 3000, diff --git a/projects/bp-strapi/package.json b/projects/bp-strapi/package.json index 700ba02b1..bdcf8c4f0 100755 --- a/projects/bp-strapi/package.json +++ b/projects/bp-strapi/package.json @@ -34,6 +34,7 @@ "pg": "8.6.0", "pm2": "^5.2.0", "sendmail": "^1.6.1", + "sharp": "^0.32.1", "strapi-plugin-meilisearch": "^0.9.2", "strapi-provider-upload-aws-s3-advanced": "^5.0.1", "xlsx": "^0.18.5" diff --git a/projects/bp-strapi/yarn.lock b/projects/bp-strapi/yarn.lock index 82282c9f6..1d6954141 100755 --- a/projects/bp-strapi/yarn.lock +++ b/projects/bp-strapi/yarn.lock @@ -9838,6 +9838,11 @@ node-addon-api@^5.0.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-5.0.0.tgz#7d7e6f9ef89043befdb20c1989c905ebde18c501" integrity sha512-CvkDw2OEnme7ybCykJpVcKH+uAOLV2qLqiyla128dN9TkEWfrYmxG6C2boDe5KcNQqZF3orkqzGgOMvZ/JNekA== +node-addon-api@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-6.1.0.tgz#ac8470034e58e67d0c6f1204a18ae6995d9c0d76" + integrity sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA== + node-fetch@2.6.7, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" @@ -11897,6 +11902,13 @@ semver@^7.3.5: dependencies: lru-cache "^6.0.0" +semver@^7.5.0: + version "7.5.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.2.tgz#5b851e66d1be07c1cdaf37dfc856f543325a2beb" + integrity sha512-SoftuTROv/cRjCze/scjGyiDtcUyxw1rgYQSZY7XTmtR5hX+dm76iDbTH8TkLPHCQmlbQVSSbNZCPM2hb0knnQ== + dependencies: + lru-cache "^6.0.0" + semver@~7.2.0: version "7.2.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.2.3.tgz#3641217233c6382173c76bf2c7ecd1e1c16b0d8a" @@ -12026,6 +12038,20 @@ sharp@0.31.0: tar-fs "^2.1.1" tunnel-agent "^0.6.0" +sharp@^0.32.1: + version "0.32.1" + resolved "https://registry.yarnpkg.com/sharp/-/sharp-0.32.1.tgz#41aa0d0b2048b2e0ee453d9fcb14ec1f408390fe" + integrity sha512-kQTFtj7ldpUqSe8kDxoGLZc1rnMFU0AO2pqbX6pLy3b7Oj8ivJIdoKNwxHVQG2HN6XpHPJqCSM2nsma2gOXvOg== + dependencies: + color "^4.2.3" + detect-libc "^2.0.1" + node-addon-api "^6.1.0" + prebuild-install "^7.1.1" + semver "^7.5.0" + simple-get "^4.0.1" + tar-fs "^2.1.1" + tunnel-agent "^0.6.0" + shebang-command@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea" From f54a27fe88c793adb909461a0050b0df1692714c Mon Sep 17 00:00:00 2001 From: baerlach-git <83013270+baerlach-git@users.noreply.github.com> Date: Fri, 16 Jun 2023 15:46:44 +0200 Subject: [PATCH 08/11] revert the quatsch from last commit --- projects/bp-gallery/.eslintrc | 3 +-- projects/bp-gallery/environments/.env.development | 2 +- projects/bp-gallery/vite.config.ts | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/bp-gallery/.eslintrc b/projects/bp-gallery/.eslintrc index 51c22730a..49e2387d9 100755 --- a/projects/bp-gallery/.eslintrc +++ b/projects/bp-gallery/.eslintrc @@ -20,8 +20,7 @@ }, "parser": "@typescript-eslint/parser", "parserOptions": { - "project": ["./tsconfig.json", "./cypress/tsconfig.json", "./vite.config.ts"], - "tsconfigRootDir": "./bp-gallery" + "project": ["./tsconfig.json", "./cypress/tsconfig.json", "./vite.config.ts"] }, "plugins": [ "react", diff --git a/projects/bp-gallery/environments/.env.development b/projects/bp-gallery/environments/.env.development index 31962ffc3..69714b97d 100644 --- a/projects/bp-gallery/environments/.env.development +++ b/projects/bp-gallery/environments/.env.development @@ -1,4 +1,4 @@ -VITE_REACT_APP_API_BASE=http://localhost:9000 +VITE_REACT_APP_API_BASE=/api VITE_REACT_APP_CONTACT_MAIL=archiv@bad-harzburg-stiftung.de VITE_REACT_APP_ADVANCED_SEARCH=false VITE_REACT_APP_GROWTHBOOK_APIHOST=https://growthbook.harz-history.de/proxy diff --git a/projects/bp-gallery/vite.config.ts b/projects/bp-gallery/vite.config.ts index 1de0ffa8e..5426b501e 100644 --- a/projects/bp-gallery/vite.config.ts +++ b/projects/bp-gallery/vite.config.ts @@ -2,6 +2,7 @@ import react from '@vitejs/plugin-react-swc'; import type { Plugin, PluginBuild } from 'esbuild'; import path from 'path'; import { defineConfig } from 'vite'; +import eslint from 'vite-plugin-eslint'; const splitPackages = ['@mui/icons-material', '@mui/material']; const reactPlugins = [ @@ -28,7 +29,7 @@ const splitPackagesPlugin: Plugin = { // https://vitejs.dev/config/ export default defineConfig({ - plugins: [react({ plugins: reactPlugins }) /*eslint({ cache: false })*/], + plugins: [react({ plugins: reactPlugins }), eslint({ cache: false })], envDir: 'environments', server: { port: 3000, From de9746b9cc17d15b3426db5646e0727e6669412d Mon Sep 17 00:00:00 2001 From: baerlach-git <83013270+baerlach-git@users.noreply.github.com> Date: Fri, 16 Jun 2023 15:54:14 +0200 Subject: [PATCH 09/11] small change to the conscole output --- projects/bp-gallery/src/components/views/search/SearchView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/bp-gallery/src/components/views/search/SearchView.tsx b/projects/bp-gallery/src/components/views/search/SearchView.tsx index 44227fdd8..a507a367c 100644 --- a/projects/bp-gallery/src/components/views/search/SearchView.tsx +++ b/projects/bp-gallery/src/components/views/search/SearchView.tsx @@ -63,7 +63,7 @@ const SearchView = () => { // return convertSearchParamsToPictureFilters(searchParams); }, [/*isAllSearchActive,*/ searchParams]); if (import.meta.env.MODE === 'development') - console.log(getSearchResultPictureIds(queryParams, '')); + getSearchResultPictureIds(queryParams, '').then(res => console.log(res)); const { linkToCollection, bulkEdit } = useBulkOperations(); return ( From 50eeff8986c51ab3836a05b665b9f8837071fccd Mon Sep 17 00:00:00 2001 From: baerlach-git <83013270+baerlach-git@users.noreply.github.com> Date: Fri, 16 Jun 2023 15:54:45 +0200 Subject: [PATCH 10/11] small change to the conscole output --- projects/bp-gallery/src/components/views/search/SearchView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/bp-gallery/src/components/views/search/SearchView.tsx b/projects/bp-gallery/src/components/views/search/SearchView.tsx index a507a367c..66ec9fbcb 100644 --- a/projects/bp-gallery/src/components/views/search/SearchView.tsx +++ b/projects/bp-gallery/src/components/views/search/SearchView.tsx @@ -63,7 +63,7 @@ const SearchView = () => { // return convertSearchParamsToPictureFilters(searchParams); }, [/*isAllSearchActive,*/ searchParams]); if (import.meta.env.MODE === 'development') - getSearchResultPictureIds(queryParams, '').then(res => console.log(res)); + getSearchResultPictureIds(queryParams, '').then(res => console.log('search results:', res)); const { linkToCollection, bulkEdit } = useBulkOperations(); return ( From 64751ff28dd2e978f0f0a15375d2ee9d67befa02 Mon Sep 17 00:00:00 2001 From: baerlach-git <83013270+baerlach-git@users.noreply.github.com> Date: Fri, 16 Jun 2023 18:22:41 +0200 Subject: [PATCH 11/11] implemented the requested changes --- .../helpers/getSearchResultPictureIds.ts | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts b/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts index 485ffe428..9492f44bf 100644 --- a/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts +++ b/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts @@ -5,10 +5,7 @@ const dateToTimeStamp = (date: string) => { }; const getSearchResultPictureIds = async ( - searchparams: { - searchTerms: string[]; - searchTimes: string[][]; - }, + { searchTerms, searchTimes }: { searchTerms: string[]; searchTimes: string[][] }, filter: string ) => { const client = new MeiliSearch({ @@ -17,9 +14,6 @@ const getSearchResultPictureIds = async ( }); const index = client.index('picture'); - const searchTerms = searchparams.searchTerms; - const searchTimes = searchparams.searchTimes; - const TIME_RANGE_START = 'time_range_tag_start'; const TIME_RANGE_END = 'time_range_tag_end'; // when building a filter for meilisearch the filtered attribute always @@ -46,15 +40,9 @@ const getSearchResultPictureIds = async ( matchingStrategy: MATCHING_STRATEGY, filter: filter, }; - - if (searchTerms.length !== 0) { - const query = searchTerms.join(' '); - const searchResult = await index.search(query, settings).then(value => value.hits); - return searchResult; - } else { - const searchResultForEmptySearch = await index.search('', settings).then(value => value.hits); - return searchResultForEmptySearch; - } + const query = searchTerms.length !== 0 ? searchTerms.join(' ') : ''; + const searchResult = await index.search(query, settings); + return searchResult.hits; }; export default getSearchResultPictureIds;