Skip to content

Commit

Permalink
feature: Add full text search support
Browse files Browse the repository at this point in the history
  • Loading branch information
MohamedBassem committed Mar 1, 2024
1 parent 75d315d commit 8d7a9c7
Show file tree
Hide file tree
Showing 17 changed files with 388 additions and 12 deletions.
5 changes: 5 additions & 0 deletions .env.sample
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@

# OPENAI_API_KEY=

############### Search ##############

# MEILI_ADDR=
# MEILI_MASTER_KEY=

############## Auth ##############
# Authentik for auth

Expand Down
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ A self-hostable bookmark-everything app with a touch of AI for the data hoarders
- 🔖 Chrome plugin for quick bookmarking.
- 📱 iOS shortcut for bookmarking content from the phone. A minimal mobile app might come later.
- 💾 Self-hostable first.
- 🔎 Full text search of all the content stored.
- [Planned] Archiving the content for offline reading.
- [Planned] Full text search of all the content stored.
- [Planned] Store raw images.

**⚠️ This app is under heavy development and it's far from stable.**
Expand All @@ -37,6 +37,8 @@ The app is configured with env variables.
| DATABASE_URL | Not set | The path for the sqlite database. |
| REDIS_HOST | localhost | The address of redis used by background jobs |
| REDIS_POST | 6379 | The port of redis used by background jobs |
| MEILI_ADDR | Not set | The address of meilisearch. If not set, Search will be disabled. |
| MEILI_MASTER_KEY | Not set | The master key configured for meili. Not needed in development. |

## Security Considerations

Expand Down
7 changes: 7 additions & 0 deletions docker/docker-compose.dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ services:
- 3000:3000
environment:
REDIS_HOST: redis
MEILI_ADDR: http://meilisearch:7700
DATABASE_URL: "/data/db.db"
command:
- pnpm
Expand All @@ -22,6 +23,10 @@ services:
image: redis:7.2-alpine
volumes:
- redis:/data
meilisearch:
image: getmeili/meilisearch:v1.6
volumes:
- meilisearch:/meili_data
workers:
build:
dockerfile: Dockerfile.dev
Expand All @@ -31,6 +36,7 @@ services:
working_dir: /app
environment:
REDIS_HOST: redis
MEILI_ADDR: http://meilisearch:7700
DATABASE_URL: "/data/db.db"
# OPENAI_API_KEY: ...
command:
Expand All @@ -55,4 +61,5 @@ services:

volumes:
redis:
meilisearch:
data:
7 changes: 7 additions & 0 deletions docker/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,23 @@ services:
- 3000:3000
environment:
REDIS_HOST: redis
MEILI_ADDR: http://meilisearch:7700
DATABASE_URL: "/data/db.db"
redis:
image: redis:7.2-alpine
volumes:
- redis:/data
meilisearch:
image: getmeili/meilisearch:v1.6
volumes:
- meilisearch:/meili_data
workers:
image: ghcr.io/mohamedbassem/hoarder-workers:latest
volumes:
- data:/data
environment:
REDIS_HOST: redis
MEILI_ADDR: http://meilisearch:7700
DATABASE_URL: "/data/db.db"
# OPENAI_API_KEY: ...
depends_on:
Expand All @@ -27,4 +33,5 @@ services:

volumes:
redis:
meilisearch:
data:
6 changes: 6 additions & 0 deletions packages/shared/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,12 @@ const serverConfig = {
browserExecutablePath: process.env.BROWSER_EXECUTABLE_PATH, // If not set, the system's browser will be used
browserUserDataDir: process.env.BROWSER_USER_DATA_DIR,
},
meilisearch: process.env.MEILI_ADDR
? {
address: process.env.MEILI_ADDR || "http://127.0.0.1:7700",
key: process.env.MEILI_MASTER_KEY || "",
}
: undefined,
logLevel: process.env.LOG_LEVEL || "debug",
demoMode: (process.env.DEMO_MODE ?? "false") == "true",
};
Expand Down
1 change: 1 addition & 0 deletions packages/shared/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
"version": "0.1.0",
"private": true,
"dependencies": {
"meilisearch": "^0.37.0",
"winston": "^3.11.0",
"zod": "^3.22.4"
},
Expand Down
14 changes: 14 additions & 0 deletions packages/shared/queues.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,17 @@ export type ZOpenAIRequest = z.infer<typeof zOpenAIRequestSchema>;
export const OpenAIQueue = new Queue<ZOpenAIRequest, void>("openai_queue", {
connection: queueConnectionDetails,
});

// Search Indexing Worker
export const zSearchIndexingRequestSchema = z.object({
bookmarkId: z.string(),
});
export type ZSearchIndexingRequest = z.infer<
typeof zSearchIndexingRequestSchema
>;
export const SearchIndexingQueue = new Queue<ZSearchIndexingRequest, void>(
"searching_indexing",
{
connection: queueConnectionDetails,
},
);
50 changes: 50 additions & 0 deletions packages/shared/search.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import { MeiliSearch, Index } from "meilisearch";
import serverConfig from "./config";
import { z } from "zod";

export const zBookmarkIdxSchema = z.object({
id: z.string(),
userId: z.string(),
url: z.string().nullish(),
title: z.string().nullish(),
description: z.string().nullish(),
content: z.string().nullish(),
tags: z.array(z.string()).default([]),
});

export type ZBookmarkIdx = z.infer<typeof zBookmarkIdxSchema>;

let searchClient: MeiliSearch | undefined;

if (serverConfig.meilisearch) {
searchClient = new MeiliSearch({
host: serverConfig.meilisearch.address,
apiKey: serverConfig.meilisearch.key,
});
}

const BOOKMARKS_IDX_NAME = "bookmarks";

let idxClient: Index<ZBookmarkIdx> | undefined;

export async function getSearchIdxClient(): Promise<Index<ZBookmarkIdx> | null> {
if (idxClient) {
return idxClient;
}
if (!searchClient) {
return null;
}

const indicies = await searchClient.getIndexes();
let idxFound = indicies.results.find((i) => i.uid == BOOKMARKS_IDX_NAME);
if (!idxFound) {
const idx = await searchClient.createIndex(BOOKMARKS_IDX_NAME, {
primaryKey: "id",
});
await searchClient.waitForTask(idx.taskUid);
idxFound = await searchClient.getIndex<ZBookmarkIdx>(BOOKMARKS_IDX_NAME);
const taskId = await idxFound.updateFilterableAttributes(["id", "userId"]);
await searchClient.waitForTask(taskId.taskUid);
}
return idxFound;
}
18 changes: 17 additions & 1 deletion packages/web/app/dashboard/components/Sidebar.tsx
Original file line number Diff line number Diff line change
@@ -1,11 +1,20 @@
import { Archive, Star, Tag, Home, PackageOpen, Settings } from "lucide-react";
import {
Archive,
Star,
Tag,
Home,
PackageOpen,
Settings,
Search,
} from "lucide-react";
import { redirect } from "next/navigation";
import SidebarItem from "./SidebarItem";
import { getServerAuthSession } from "@/server/auth";
import Link from "next/link";
import SidebarProfileOptions from "./SidebarProfileOptions";
import { Separator } from "@/components/ui/separator";
import AllLists from "./AllLists";
import serverConfig from "@hoarder/shared/config";

export default async function Sidebar() {
const session = await getServerAuthSession();
Expand Down Expand Up @@ -34,6 +43,13 @@ export default async function Sidebar() {
name="Favourites"
path="/dashboard/bookmarks/favourites"
/>
{serverConfig.meilisearch && (
<SidebarItem
logo={<Search />}
name="Search"
path="/dashboard/search"
/>
)}
<SidebarItem
logo={<Archive />}
name="Archive"
Expand Down
85 changes: 85 additions & 0 deletions packages/web/app/dashboard/search/page.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
"use client";

import { api } from "@/lib/trpc";
import { usePathname, useRouter, useSearchParams } from "next/navigation";
import BookmarksGrid from "../bookmarks/components/BookmarksGrid";
import { Input } from "@/components/ui/input";
import Loading from "../bookmarks/loading";
import { keepPreviousData } from "@tanstack/react-query";
import { Search } from "lucide-react";
import { ActionButton } from "@/components/ui/action-button";
import { useRef } from "react";

export default function SearchPage() {
const router = useRouter();
const pathname = usePathname();
const searchParams = useSearchParams();
const searchQuery = searchParams.get("q") || "";

const { data, isPending, isPlaceholderData, error } =
api.bookmarks.searchBookmarks.useQuery(
{
text: searchQuery,
},
{
placeholderData: keepPreviousData,
},
);

if (error) {
throw error;
}

const inputRef: React.MutableRefObject<HTMLInputElement | null> =
useRef<HTMLInputElement | null>(null);

let timeoutId: NodeJS.Timeout | undefined;

// Debounce user input
const doSearch = () => {
if (!inputRef.current) {
return;
}
router.replace(`${pathname}?q=${inputRef.current.value}`);
};

const onInputChange = () => {
if (timeoutId) {
clearTimeout(timeoutId);
}
timeoutId = setTimeout(() => {
doSearch();
}, 200);
};

return (
<div className="container flex flex-col gap-3 p-4">
<div className="flex gap-2">
<Input
ref={inputRef}
placeholder="Search"
defaultValue={searchQuery}
onChange={onInputChange}
/>
<ActionButton
loading={isPending || isPlaceholderData}
onClick={doSearch}
>
<span className="flex gap-2">
<Search />
<span className="my-auto">Search</span>
</span>
</ActionButton>
</div>
<hr />
{data ? (
<BookmarksGrid
query={{ ids: data.bookmarks.map((b) => b.id) }}
bookmarks={data.bookmarks}
/>
) : (
<Loading />
)}
</div>
);
}
1 change: 1 addition & 0 deletions packages/web/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
"drizzle-orm": "^0.29.4",
"install": "^0.13.0",
"lucide-react": "^0.322.0",
"meilisearch": "^0.37.0",
"next": "14.1.0",
"next-auth": "^4.24.5",
"prettier": "^3.2.5",
Expand Down
51 changes: 50 additions & 1 deletion packages/web/server/api/routers/bookmarks.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { z } from "zod";
import { Context, authedProcedure, router } from "../trpc";
import { getSearchIdxClient } from "@hoarder/shared/search";
import {
ZBookmark,
ZBookmarkContent,
Expand All @@ -17,7 +18,11 @@ import {
bookmarks,
tagsOnBookmarks,
} from "@hoarder/db/schema";
import { LinkCrawlerQueue, OpenAIQueue } from "@hoarder/shared/queues";
import {
LinkCrawlerQueue,
OpenAIQueue,
SearchIndexingQueue,
} from "@hoarder/shared/queues";
import { TRPCError, experimental_trpcMiddleware } from "@trpc/server";
import { and, desc, eq, inArray } from "drizzle-orm";
import { ZBookmarkTags } from "@/lib/types/api/tags";
Expand Down Expand Up @@ -172,6 +177,7 @@ export const bookmarksAppRouter = router({
break;
}
}
SearchIndexingQueue.add("search_indexing", { bookmarkId: bookmark.id });
return bookmark;
}),

Expand Down Expand Up @@ -280,6 +286,49 @@ export const bookmarksAppRouter = router({

return toZodSchema(bookmark);
}),
searchBookmarks: authedProcedure
.input(
z.object({
text: z.string(),
}),
)
.output(zGetBookmarksResponseSchema)
.query(async ({ input, ctx }) => {
const client = await getSearchIdxClient();
if (!client) {
throw new TRPCError({
code: "INTERNAL_SERVER_ERROR",
message: "Search functionality is not configured",
});
}
const resp = await client.search(input.text, {
filter: [`userId = '${ctx.user.id}'`],
});

if (resp.hits.length == 0) {
return { bookmarks: [] };
}
const results = await ctx.db.query.bookmarks.findMany({
where: and(
eq(bookmarks.userId, ctx.user.id),
inArray(
bookmarks.id,
resp.hits.map((h) => h.id),
),
),
with: {
tagsOnBookmarks: {
with: {
tag: true,
},
},
link: true,
text: true,
},
});

return { bookmarks: results.map(toZodSchema) };
}),
getBookmarks: authedProcedure
.input(zGetBookmarksRequestSchema)
.output(zGetBookmarksResponseSchema)
Expand Down
Loading

0 comments on commit 8d7a9c7

Please sign in to comment.