From df79ca29e503fa4c64a8af129a2c822ff9b0df33 Mon Sep 17 00:00:00 2001
From: Ian Eaves <ian.k.eaves@gmail.com>
Date: Fri, 10 Nov 2023 18:45:25 -0600
Subject: [PATCH] Feature/gra 900 server side api capabilities (#763)

*grAI
---
 grai-server/app/Dockerfile                    |   1 +
 .../app/connections/adapters/schemas.py       |  13 +-
 grai-server/app/grAI/chat_implementations.py  | 317 ++++++++++++++++--
 grai-server/app/grAI/consumers.py             |   2 +-
 grai-server/app/lineage/managers.py           |  67 +++-
 .../migrations/0018_vector_embeddings.bak     |  41 +++
 grai-server/app/lineage/models.py             |  40 ++-
 grai-server/app/lineage/tasks.py              |  68 ++++
 grai-server/app/poetry.lock                   | 263 +++++++++++----
 grai-server/app/pyproject.toml                |   2 +
 grai-server/app/the_guide/celery.py           |   9 +-
 grai-server/app/the_guide/settings/base.py    |  67 ++--
 grai-server/docker-compose.yml                |   6 +-
 13 files changed, 752 insertions(+), 144 deletions(-)
 create mode 100644 grai-server/app/lineage/migrations/0018_vector_embeddings.bak
 create mode 100644 grai-server/app/lineage/tasks.py

diff --git a/grai-server/app/Dockerfile b/grai-server/app/Dockerfile
index 43a6d082a..58c92386d 100755
--- a/grai-server/app/Dockerfile
+++ b/grai-server/app/Dockerfile
@@ -71,6 +71,7 @@ RUN apt update \
     && ACCEPT_EULA=Y apt-get install -y msodbcsql18 \
     && rm -rf /var/lib/apt/lists/*
 
+
 ENTRYPOINT ["/usr/src/app/entrypoint.sh"]
 
 # --------- final images --------------- #
diff --git a/grai-server/app/connections/adapters/schemas.py b/grai-server/app/connections/adapters/schemas.py
index efaba0767..f90dad9ce 100644
--- a/grai-server/app/connections/adapters/schemas.py
+++ b/grai-server/app/connections/adapters/schemas.py
@@ -1,7 +1,7 @@
 import pprint
 from typing import Any, List, Literal, Optional, Sequence, Type
 from uuid import UUID
-
+from django.db.models.query import QuerySet
 from django.db.models import Q
 from grai_schemas.v1.edge import EdgeSpec, EdgeV1, SourcedEdgeSpec, SourcedEdgeV1
 from grai_schemas.v1.node import (
@@ -216,3 +216,14 @@ def edge_model_to_edge_v1_schema(model: Edge, schema_type: Literal["EdgeV1"]) ->
     model_dict["destination"] = NodeNamedID(**model.destination.__dict__)
     model_dict["destination"] = {"id": model_dict.pop("destination_id")}
     return EdgeV1.from_spec(model_dict)
+
+
+@model_to_schema.register
+def sequence_model_to_sequence_v1_schema(models: list | tuple, schema_type: str) -> list | tuple:
+    iter = (model_to_schema(model, schema_type) for model in models)
+    return type(models)(iter)
+
+
+@model_to_schema.register
+def queryset_to_sequence_v1_schema(models: QuerySet, schema_type: str) -> list:
+    return [model_to_schema(model, schema_type) for model in models]
diff --git a/grai-server/app/grAI/chat_implementations.py b/grai-server/app/grAI/chat_implementations.py
index 2595a0f8f..74dd66040 100644
--- a/grai-server/app/grAI/chat_implementations.py
+++ b/grai-server/app/grAI/chat_implementations.py
@@ -2,15 +2,81 @@
 import uuid
 from abc import ABC, abstractmethod
 from functools import partial
-from typing import Callable
+from typing import Callable, Any, Literal, Union, Annotated
 
 import openai
 from django.conf import settings
 from django.core.cache import cache
 from grai_schemas.serializers import GraiYamlSerializer
+from pydantic import BaseModel, Field
+from lineage.models import Node, Edge
+from grAI.models import Message
+from django.db.models import Q
+from functools import reduce
+import operator
+import logging
+from connections.adapters.schemas import model_to_schema
+import tiktoken
 from pydantic import BaseModel
 
-from grAI.models import Message
+logging.basicConfig(level=logging.DEBUG)
+
+
+MAX_RETURN_LIMIT = 20
+
+RoleType = Union[Literal["user"], Literal["system"], Literal["assistant"]]
+
+
+class BaseMessage(BaseModel):
+    role: str
+    content: str
+    token_length: int | None = None
+
+    def representation(self):
+        return {"role": self.role, "content": self.content}
+
+
+class UserMessage(BaseMessage):
+    role: Literal["user"] = "user"
+
+
+class SystemMessage(BaseMessage):
+    role: Literal["system"] = "system"
+
+
+class AIMessage(BaseMessage):
+    role: Literal["assistant"] = "assistant"
+
+
+class FunctionMessage(BaseMessage):
+    role: Literal["function"] = "function"
+    name: str
+
+    def representation(self):
+        return {"role": self.role, "content": self.content, "name": self.name}
+
+
+class ChatMessage(BaseModel):
+    message: Union[UserMessage, SystemMessage, AIMessage, FunctionMessage]
+
+
+class ChatMessages(BaseModel):
+    messages: list[BaseMessage]
+
+    def to_gpt(self):
+        return [message.representation() for message in self.messages]
+
+    def __getitem__(self, index):
+        return self.messages[index]
+
+    def __len__(self):
+        return len(self.messages)
+
+    def append(self, item):
+        self.messages.append(item)
+
+    def extend(self, items):
+        self.messages.extend(items)
 
 
 class API(ABC):
@@ -19,7 +85,7 @@ class API(ABC):
     id: str
 
     @abstractmethod
-    def call(self, **kwargs):
+    def call(self, **kwargs) -> (Any, str):
         pass
 
     def serialize(self, result) -> str:
@@ -29,12 +95,166 @@ def serialize(self, result) -> str:
         return GraiYamlSerializer.dump(result)
 
     def response(self, **kwargs) -> str:
-        return self.serialize(self.call(**kwargs))
+        logging.info(f"Calling {self.id} with {kwargs}")
+        obj, message = self.call(**kwargs)
+
+        logging.info(f"Building Response message for {self.id} with {kwargs}")
+        if message is None:
+            return self.serialize(obj)
+        else:
+            return f"{self.serialize(obj)}\n{message}"
 
     def gpt_definition(self) -> dict:
         return {"name": self.id, "description": self.description, "parameters": self.schema_model.schema()}
 
 
+class NodeIdentifier(BaseModel):
+    name: str = Field(description="The name of the node to query for")
+    namespace: str = Field(description="The namespace of the node to query for")
+
+
+class NodeLookup(BaseModel):
+    nodes: list[NodeIdentifier] = Field(description="A list of nodes to lookup")
+
+
+class NodeLookupAPI(API):
+    id = "node_lookup"
+    description = (
+        "A function to lookup metadata about one or more nodes. Results will be returned in the order requested."
+    )
+    schema_model = NodeLookup
+
+    def __init__(self, workspace: str | uuid.UUID):
+        self.workspace = workspace
+        self.query_limit = MAX_RETURN_LIMIT
+
+    def response_message(self, result_set: list[Node]) -> str | None:
+        total_results = len(result_set)
+        if total_results > self.query_limit:
+            message = f"Returned {self.query_limit} of {total_results} results. You might need to narrow your search."
+        elif total_results == 0:
+            message = "No results found matching these query conditions."
+        else:
+            message = None
+
+        return message
+
+    def call(self, **kwargs) -> (list[Node], str | None):
+        try:
+            validation = self.schema_model(**kwargs)
+        except:
+            return [], "Invalid input. Please check your input and try again."
+        q_objects = (Q(**node.dict(exclude_none=True)) for node in validation.nodes)
+        query = reduce(operator.or_, q_objects)
+        result_set = Node.objects.filter(workspace=self.workspace).filter(query).order_by("-created_at").all()
+        response_items = model_to_schema(result_set[: self.query_limit], "NodeV1")
+        return response_items, self.response_message(result_set)
+
+
+class FuzzyMatchQuery(BaseModel):
+    string: str = Field(description="The fuzzy string used to search amongst node names")
+
+
+class FuzzyMatchNodesAPI(API):
+    id = "node_fuzzy_lookup"
+    description = "Performs a fuzzy search for nodes matching a name regardless of namespace"
+    schema_model = FuzzyMatchQuery
+
+    def __init__(self, workspace: str | uuid.UUID):
+        self.workspace = workspace
+        self.query_limit = MAX_RETURN_LIMIT
+
+    def response_message(self, result_set: list[Node]) -> str | None:
+        total_results = len(result_set)
+        if total_results > self.query_limit:
+            message = f"Returned {self.query_limit} of {total_results} results. You might need to narrow your search."
+        elif total_results == 0:
+            message = "No results found matching these query conditions."
+        else:
+            message = None
+
+        return message
+
+    def call(self, string: str) -> (list, str | None):
+        result_set = (
+            Node.objects.filter(workspace=self.workspace).filter(name__contains=string).order_by("-created_at").all()
+        )
+        response_items = [{"name": node.name, "namespace": node.namespace} for node in result_set]
+
+        return response_items, self.response_message(result_set)
+
+
+class EdgeLookupSchema(BaseModel):
+    source: uuid.UUID | None = Field(description="The primary key of the source node on an edge", default=None)
+    destination: uuid.UUID | None = Field(
+        description="The primary key of the destination node on an edge", default=None
+    )
+
+
+class MultiEdgeLookup(BaseModel):
+    edges: list[EdgeLookupSchema] = Field(
+        description="List of edges to lookup. Edges can be uniquely identified by a (name, namespace) tuple, or by a (source, destination) tuple of the nodes the edge connects"
+    )
+
+
+class EdgeLookupAPI(API):
+    id = "edge_lookup"
+    description = """
+    This function Supports looking up edges from a data lineage graph. For example, a query with name=Test but no
+    namespace value will return all edges explicitly named "Test" regardless of namespace.
+    Edges are uniquely identified both by their (name, namespace), and by the (source, destination) nodes they connect.
+    """
+    schema_model = MultiEdgeLookup
+
+    def __init__(self, workspace: str | uuid.UUID):
+        self.workspace = workspace
+        self.query_limit = MAX_RETURN_LIMIT
+
+    def response_message(self, result_set: list[Edge]) -> str | None:
+        total_results = len(result_set)
+        if total_results > self.query_limit:
+            message = f"Returned {self.query_limit} of {total_results} results. You might need to narrow your search."
+        elif total_results == 0:
+            message = "No results found matching these query conditions."
+        else:
+            message = None
+
+        return message
+
+    def call(self, **kwargs) -> (list[Edge], str | None):
+        validation = self.schema_model(**kwargs)
+        q_objects = (Q(**node.dict(exclude_none=True)) for node in validation.edges)
+        query = reduce(operator.or_, q_objects)
+        result_set = Edge.objects.filter(workspace=self.workspace).filter(query).all()[: self.query_limit]
+        return model_to_schema(result_set[: self.query_limit], "EdgeV1"), self.response_message(result_set)
+
+
+class EdgeFuzzyLookupSchema(BaseModel):
+    name__contains: str | None = Field(
+        description="The name of the edge to lookup perform a fuzzy search on", default=None
+    )
+    namespace__contains: str | None = Field(
+        description="The namespace of the edge to lookup perform a fuzzy search on", default=None
+    )
+    is_active: bool | None = Field(description="Whether or not the edge is active", default=True)
+
+
+class MultiFuzzyEdgeLookup(BaseModel):
+    edges: list[EdgeLookupSchema] = Field(
+        description="List of edges to lookup. Edges can be uniquely identified by a (name, namespace) tuple, or by a (source, destination) tuple of the nodes the edge connects"
+    )
+
+
+class EdgeFuzzyLookupAPI(EdgeLookupAPI):
+    id = "edge_fuzzy_lookup"
+    description = """
+    This function Supports looking up edges from a data lineage graph. For example, a query with name__contains=test
+    but no namespace value will return all edges whose names contain the substring "test" regardless of namespace.
+    Edges are uniquely identified both by their (name, namespace), and by the (source, destination) nodes they connect.
+    """
+    schema_model = MultiFuzzyEdgeLookup
+
+
 class NodeEdgeSerializer:
     def __init__(self, nodes, edges):
         self.nodes = nodes
@@ -67,6 +287,11 @@ def serialize(self, result):
         return result
 
 
+class FakeEncoder:
+    def encode(self, text):
+        return [1, 2, 3, 4]
+
+
 class BaseConversation:
     def __init__(
         self,
@@ -81,6 +306,7 @@ def __init__(
             functions = []
 
         self.model_type = model_type
+        self.encoder = FakeEncoder()  # tiktoken.encoding_for_model(self.model_type)
         self.chat_id = chat_id
         self.cache_id = f"grAI:chat_id:{chat_id}"
         self.system_context = prompt
@@ -88,37 +314,46 @@ def __init__(
         self.api_functions = {func.id: func for func in functions}
         self.verbose = verbose
 
-        self.prompt_message = {"role": "system", "content": self.system_context}
+        self.prompt_message = SystemMessage(content=self.system_context)
         self.hydrate_chat()
 
     @property
-    def cached_messages(self) -> list:
-        return cache.get(self.cache_id)
+    def cached_messages(self) -> list[BaseMessage]:
+        messages = [ChatMessage(message=message).message for message in cache.get(self.cache_id)]
+        return messages
 
     @cached_messages.setter
-    def cached_messages(self, values):
-        cache.set(self.cache_id, values)
+    def cached_messages(self, values: list[BaseMessage]):
+        cache.set(self.cache_id, [v.dict() for v in values])
 
     def hydrate_chat(self):
+        logging.info(f"Hydrating chat history for conversations: {self.chat_id}")
         messages = cache.get(self.cache_id, None)
 
         if messages is None:
-            model_messages = Message.objects.filter(chat_id=self.chat_id).order_by("-created_at").all()
-            messages = [{"role": m.role, "content": m.message} for m in model_messages]
-            self.cached_messages = messages
+            logging.info(f"Loading chat history for chat {self.chat_id} from database")
+            messages_iter = (
+                {"role": m.role, "content": m.message, "token_length": len(self.encoder.encode(m.message))}
+                for m in Message.objects.filter(chat_id=self.chat_id).order_by("-created_at").all()
+            )
+            messages_list = [ChatMessage(message=message).message for message in messages_iter]
+            self.cached_messages = messages_list
 
-    def summarize(self, messages):
+    def summarize(self, messages: list[BaseMessage]) -> AIMessage:
         summary_prompt = """
         Please summarize this conversation encoding the most important information a future agent would need to continue
         working on the problem with me. Please insure you do not call any functions providing an exclusively
         text based summary of the conversation to this point with all relevant context for the next agent.
         """
-        message = {"role": "user", "content": summary_prompt}
-
-        response = openai.ChatCompletion.create(model=self.model_type, user=self.user, messages=[*messages, message])
+        message = UserMessage(content=summary_prompt)
+        summary_messages = ChatMessages(messages=[*messages, message])
+        logging.info(f"Summarizing conversation for chat: {self.chat_id}")
+        response = openai.ChatCompletion.create(
+            model=self.model_type, user=self.user, messages=summary_messages.to_gpt()
+        )
 
         # this is hacky for now
-        summary_message = {"role": "assistant", "content": response.choices[0].message.content}
+        summary_message = AIMessage(content=response.choices[0].message.content)
         return summary_message
 
     @property
@@ -136,22 +371,31 @@ def model(self) -> Callable:
         return model
 
     def request(self, user_input: str) -> str:
-        messages = [self.prompt_message, *self.cached_messages, {"role": "user", "content": user_input}]
+        logging.info(f"Responding to request for: {self.chat_id}")
+        messages = ChatMessages(messages=[self.prompt_message, *self.cached_messages, UserMessage(content=user_input)])
 
+        result = None
         stop = False
         while not stop:
             try:
-                response = self.model(messages=messages)
+                response = self.model(messages=messages.to_gpt())
             except openai.InvalidRequestError:
                 # If it hits the token limit try to summarize
                 summary = self.summarize(messages[:-1])
-                messages = [self.prompt_message, summary, messages[-1]]
-                response = self.model(messages=messages)
 
-            result = response.choices[0]
+                messages = ChatMessages(messages=[self.prompt_message, summary, messages[-1]])
+                response = self.model(messages=messages.to_gpt())
+
+            if result:
+                for choice in response.choices:
+                    if result != choice:
+                        result = choice
+                        break
+            else:
+                result = response.choices[0]
 
             if stop := result.finish_reason == "stop":
-                messages.append({"role": "assistant", "content": result.message.content})
+                messages.append(AIMessage(content=result.message.content))
             elif result.finish_reason == "function_call":
                 func_id = result.message.function_call.name
                 func_kwargs = json.loads(result.message.function_call.arguments)
@@ -159,29 +403,36 @@ def request(self, user_input: str) -> str:
                 response = api.response(**func_kwargs)
 
                 if isinstance(api, InvalidAPI):
-                    messages.append({"role": "system", "content": response})
+                    messages.append(SystemMessage(content=response))
                 else:
-                    messages.append({"role": "function", "name": func_id, "content": response})
+                    messages.append(FunctionMessage(name=func_id, content=response))
             elif result.finish_reason == "length":
                 summary = self.summarize(messages[:-1])
-                messages = [self.prompt_message, summary, messages[-1]]
+                messages = ChatMessages(messages=[self.prompt_message, summary, messages[-1]])
             else:
                 # valid values include length, content_filter, null
                 raise NotImplementedError(f"No stop reason for {result.finish_reason}")
 
-        self.cached_messages = messages
+        self.cached_messages = messages.messages
         return result.message.content
 
 
-def get_chat_conversation(chat_id: str | uuid.UUID, model_type: str = settings.OPENAI_PREFERRED_MODEL):
+def get_chat_conversation(
+    chat_id: str | uuid.UUID, workspace: str | uuid.UUID, model_type: str = settings.OPENAI_PREFERRED_MODEL
+):
     chat_prompt = """
     You are a helpful assistant with domain expertise about an organizations data and data infrastructure.
-    You know how to query for additional context and metadata about any data in the organization.
-    Unique pieces of data like a column in a database is identified by a (name, namespace) tuple or a unique uuid.
-    You can help users discover new data or identify and correct issues such as broken data pipelines,
-    and BI dashboards.
+
+    * You know how to query for additional context and metadata about any data in the organization.
+    * Unique pieces of data like a column in a database is identified by a (name, namespace) tuple or a unique uuid.
+    * You can help users discover new data or identify and correct issues such as broken data pipelines, and BI dashboards.
+    * Your responses must use Markdown syntax
     """
-    functions = []
+    functions = [
+        NodeLookupAPI(workspace=workspace),
+        # EdgeLookupAPI(workspace=workspace),
+        FuzzyMatchNodesAPI(workspace=workspace),
+    ]
 
     conversation = BaseConversation(
         prompt=chat_prompt, model_type=model_type, functions=functions, chat_id=str(chat_id)
diff --git a/grai-server/app/grAI/consumers.py b/grai-server/app/grAI/consumers.py
index 8c4eac563..7cebfebc0 100644
--- a/grai-server/app/grAI/consumers.py
+++ b/grai-server/app/grAI/consumers.py
@@ -89,7 +89,7 @@ def chat_message(self, event: dict):
             agent = MessageRoles.SYSTEM.value
         else:
             if payload.chat_id not in self.conversations:
-                self.conversations[payload.chat_id] = get_chat_conversation(payload.chat_id)
+                self.conversations[payload.chat_id] = get_chat_conversation(payload.chat_id, workspace=self.workspace)
 
             response = self.conversations[payload.chat_id].request(payload.message)
             agent = MessageRoles.AGENT.value
diff --git a/grai-server/app/lineage/managers.py b/grai-server/app/lineage/managers.py
index 3bf52d71f..3ef57e838 100644
--- a/grai-server/app/lineage/managers.py
+++ b/grai-server/app/lineage/managers.py
@@ -5,25 +5,38 @@
 from psqlextra.manager import PostgresManager
 
 from .graph_cache import GraphCache
+from lineage.tasks import update_node_vector_index, EmbeddingTaskStatus
+from django.core.cache import cache
+from typing import TYPE_CHECKING, Iterator
+import uuid
+
+if TYPE_CHECKING:
+    from lineage.models import Node
 
 
 class CacheManager(TenantManagerMixin, models.Manager):
+    @staticmethod
+    def update_cache(objs: list):
+        workspace = objs[0].workspace
+        cache = GraphCache(workspace)
+
+        for obj in objs:
+            obj.cache_model(cache)
+
+        cache.layout_graph()
+
     def bulk_create(
         self,
         objs: Iterable[Any],
         batch_size: int | None = None,
         **kwargs,
     ) -> List:
-        result = super().bulk_create(objs, **kwargs)
-
-        if len(list(objs)) > 0:
-            workspace = objs[0].workspace
-            cache = GraphCache(workspace)
+        objs = list(objs)
 
-            for obj in objs:
-                obj.cache_model(cache)
+        result = super().bulk_create(objs, **kwargs)
 
-            cache.layout_graph()
+        if len(objs) > 0:
+            self.update_cache(objs)
 
         return result
 
@@ -33,23 +46,49 @@ def bulk_update(
         fields: Sequence[str],
         **kwargs,
     ) -> int:
+        objs = list(objs)
+
         result = super().bulk_update(
             objs,
             fields,
             **kwargs,
         )
 
-        if len(list(objs)) > 0:
-            workspace = objs[0].workspace
-            cache = GraphCache(workspace)
+        if len(objs) > 0:
+            self.update_cache(objs)
 
-            for obj in objs:
-                obj.cache_model(cache)
+        return result
+
+
+class NodeEmbeddingManager(models.Manager):
+    @staticmethod
+    def obj_generator(objs: Iterable["Node"], task_id: uuid.UUID) -> Iterator["Node"]:
+        cache.set(f"lineage:NodeEmbeddingTasks:{task_id}", EmbeddingTaskStatus.WAIT, timeout=60 * 60 * 24)
+        for obj in objs:
+            yield obj
+            update_node_vector_index.delay(obj.id)
+
+    @staticmethod
+    def start_task(task_id: uuid.UUID):
+        cache_key = f"lineage:NodeEmbeddingTasks:{task_id}"
+        cache.delete(cache_key)
 
-            cache.layout_graph()
+    def bulk_create(self, objs: Iterable[Any], **kwargs) -> List:
+        task_id = uuid.uuid4()
+        result = super().bulk_create(self.obj_generator(objs, task_id), **kwargs)
+        self.start_task(task_id)
+        return result
 
+    def bulk_update(self, objs: Iterable[Any], fields: Sequence[str], **kwargs) -> int:
+        task_id = uuid.uuid4()
+        result = super().bulk_update(self.obj_generator(objs, task_id), fields, **kwargs)
+        self.start_task(task_id)
         return result
 
 
+class NodeManager(CacheManager):  # (NodeEmbeddingManager, CacheManager):
+    pass
+
+
 class SourceManager(TenantManagerMixin, PostgresManager):
     pass
diff --git a/grai-server/app/lineage/migrations/0018_vector_embeddings.bak b/grai-server/app/lineage/migrations/0018_vector_embeddings.bak
new file mode 100644
index 000000000..16dee4d63
--- /dev/null
+++ b/grai-server/app/lineage/migrations/0018_vector_embeddings.bak
@@ -0,0 +1,41 @@
+# Generated by Django 4.2.6 on 2023-10-31 17:27
+
+from django.db import migrations, models
+import django.db.models.deletion
+import pgvector.django
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("lineage", "0017_invalid_source_lineage_state"),
+    ]
+
+    operations = [
+        pgvector.django.VectorExtension(),
+        migrations.CreateModel(
+            name="NodeEmbeddings",
+            fields=[
+                ("embedding", pgvector.django.VectorField(dimensions=1536)),
+                (
+                    "node",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        primary_key=True,
+                        serialize=False,
+                        to="lineage.node",
+                    ),
+                ),
+            ],
+            options={
+                "indexes": [
+                    pgvector.django.HnswIndex(
+                        ef_construction=128,
+                        fields=["embedding"],
+                        m=64,
+                        name="node_embedding_index",
+                        opclasses=["vector_ip_ops"],
+                    )
+                ],
+            },
+        ),
+    ]
diff --git a/grai-server/app/lineage/models.py b/grai-server/app/lineage/models.py
index 52b5a27d2..93a842905 100755
--- a/grai-server/app/lineage/models.py
+++ b/grai-server/app/lineage/models.py
@@ -1,26 +1,21 @@
-import json
-import pathlib
 import uuid
-from datetime import date, datetime, timezone
-from enum import Enum
-from typing import Any
-from uuid import UUID
 
-from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 from django.db.models import F, Q
 from django_multitenant.models import TenantModel
 from grai_schemas.serializers import GraiEncoder
-from pydantic import BaseModel
+
 
 from .graph_cache import GraphCache
 from .graph_tasks import cache_edge, cache_node
-from .managers import CacheManager, SourceManager
+from .managers import CacheManager, SourceManager, NodeManager
+from pgvector.django import VectorField, HnswIndex
+from django.conf import settings
+from lineage.tasks import update_node_vector_index
 
 
-# Create your models here.
 class Node(TenantModel):
-    objects = CacheManager()
+    objects = NodeManager()
 
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     namespace = models.TextField(default="default")
@@ -60,6 +55,9 @@ def save(self, *args, **kwargs):
         super().save(*args, **kwargs)
         self.cache_model()
 
+        # if self.workspace.ai_enabled and settings.HAS_OPENAI:
+        #     update_node_vector_index.delay(self.id)
+
     def delete(self, *args, **kwargs):
         super().delete(*args, **kwargs)
         self.cache_model(delete=True)
@@ -98,6 +96,26 @@ class Meta:
         ]
 
 
+# class NodeEmbeddings(models.Model):
+#     embedding = VectorField(dimensions=1536)
+#     node = models.OneToOneField(
+#         Node,
+#         on_delete=models.CASCADE,
+#         primary_key=True,
+#     )
+#
+#     class Meta:
+#         indexes = [
+#             HnswIndex(
+#                 name="node_embedding_index",
+#                 fields=["embedding"],
+#                 m=64,
+#                 ef_construction=128,  # should be at least 2x m.
+#                 opclasses=["vector_ip_ops"],
+#             )
+#         ]
+
+
 class Edge(TenantModel):
     objects = CacheManager()
 
diff --git a/grai-server/app/lineage/tasks.py b/grai-server/app/lineage/tasks.py
new file mode 100644
index 000000000..758511df0
--- /dev/null
+++ b/grai-server/app/lineage/tasks.py
@@ -0,0 +1,68 @@
+from celery import shared_task
+from uuid import UUID
+import openai
+import logging
+from grai_schemas.serializers import GraiYamlSerializer
+from django_celery_beat.models import PeriodicTask, PeriodicTasks
+from datetime import datetime
+from typing import TYPE_CHECKING
+from django.apps import apps
+from django.core.cache import cache
+
+
+if TYPE_CHECKING:
+    from lineage.models import Node
+
+
+class EmbeddingTaskStatus:
+    WAIT = 0
+
+
+def create_node_vector_index(node: "Node"):
+    from connections.adapters.schemas import model_to_schema
+    from lineage.models import NodeEmbeddings
+
+    schema = model_to_schema(node, "NodeV1")
+    content = GraiYamlSerializer.dump(schema)
+    embedding_resp = openai.Embedding.create(input=content, model="text-embedding-ada-002")
+    NodeEmbeddings.objects.update_or_create(node=node, embedding=embedding_resp.data[0].embedding)
+
+
+def get_embedding_task_state(task_id: UUID | None) -> int | None:
+    if task_id is None:
+        return task_id
+
+    cache_key = f"lineage:NodeEmbeddingTasks:{task_id}"
+    return cache.get(cache_key, None)
+
+
+@shared_task(bind=True, max_retries=None)
+def update_node_vector_index(self, node_id: UUID, task_id: UUID | None = None):
+    from lineage.models import Node
+
+    logging.info(f"Creating embedding for node {node_id}")
+
+    task_status = get_embedding_task_state(task_id)
+    if task_status == EmbeddingTaskStatus.WAIT:
+        logging.info(f"Task {task_id} is waiting")
+        self.retry(countdown=10)
+        return
+
+    node = Node.objects.get(id=node_id)
+    try:
+        create_node_vector_index(node)
+    except openai.error.RateLimitError:
+        logging.info(f"Openai rate limit reach retrying in 10 seconds")
+        self.retry(countdown=10)
+        return
+
+
+@shared_task
+def bulk_update_embeddings():
+    from lineage.models import Node
+
+    task = PeriodicTask.objects.get(name="lineage:Node:bulk_update_embeddings")
+    last_run_at = task.last_run_at if task.last_run_at is not None else datetime.min
+
+    for node in Node.objects.filter(updated_at__gt=last_run_at).all():
+        update_node_vector_index.delay(node.id)
diff --git a/grai-server/app/poetry.lock b/grai-server/app/poetry.lock
index cd56da1bf..0f96d5df6 100644
--- a/grai-server/app/poetry.lock
+++ b/grai-server/app/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand.
 
 [[package]]
 name = "aiohttp"
@@ -555,29 +555,29 @@ ujson = ["ujson (>=5.4.0,<6.0.0)"]
 
 [[package]]
 name = "celery"
-version = "5.3.4"
+version = "5.3.5"
 description = "Distributed Task Queue."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "celery-5.3.4-py3-none-any.whl", hash = "sha256:1e6ed40af72695464ce98ca2c201ad0ef8fd192246f6c9eac8bba343b980ad34"},
-    {file = "celery-5.3.4.tar.gz", hash = "sha256:9023df6a8962da79eb30c0c84d5f4863d9793a466354cc931d7f72423996de28"},
+    {file = "celery-5.3.5-py3-none-any.whl", hash = "sha256:30b75ac60fb081c2d9f8881382c148ed7c9052031a75a1e8743ff4b4b071f184"},
+    {file = "celery-5.3.5.tar.gz", hash = "sha256:6b65d8dd5db499dd6190c45aa6398e171b99592f2af62c312f7391587feb5458"},
 ]
 
 [package.dependencies]
-billiard = ">=4.1.0,<5.0"
+billiard = ">=4.2.0,<5.0"
 click = ">=8.1.2,<9.0"
 click-didyoumean = ">=0.3.0"
 click-plugins = ">=1.1.1"
 click-repl = ">=0.2.0"
-kombu = ">=5.3.2,<6.0"
+kombu = ">=5.3.3,<6.0"
 python-dateutil = ">=2.8.2"
 tzdata = ">=2022.7"
-vine = ">=5.0.0,<6.0"
+vine = ">=5.1.0,<6.0"
 
 [package.extras]
 arangodb = ["pyArango (>=2.0.2)"]
-auth = ["cryptography (==41.0.3)"]
+auth = ["cryptography (==41.0.5)"]
 azureblockblob = ["azure-storage-blob (>=12.15.0)"]
 brotli = ["brotli (>=1.0.0)", "brotlipy (>=0.7.0)"]
 cassandra = ["cassandra-driver (>=3.25.0,<4)"]
@@ -587,26 +587,26 @@ couchbase = ["couchbase (>=3.0.0)"]
 couchdb = ["pycouchdb (==1.14.2)"]
 django = ["Django (>=2.2.28)"]
 dynamodb = ["boto3 (>=1.26.143)"]
-elasticsearch = ["elasticsearch (<8.0)"]
+elasticsearch = ["elastic-transport (<=8.10.0)", "elasticsearch (<=8.10.1)"]
 eventlet = ["eventlet (>=0.32.0)"]
 gevent = ["gevent (>=1.5.0)"]
 librabbitmq = ["librabbitmq (>=2.0.0)"]
 memcache = ["pylibmc (==1.6.3)"]
 mongodb = ["pymongo[srv] (>=4.0.2)"]
-msgpack = ["msgpack (==1.0.5)"]
+msgpack = ["msgpack (==1.0.7)"]
 pymemcache = ["python-memcached (==1.59)"]
 pyro = ["pyro4 (==4.82)"]
 pytest = ["pytest-celery (==0.0.0)"]
-redis = ["redis (>=4.5.2,!=4.5.5,<5.0.0)"]
+redis = ["redis (>=4.5.2,!=4.5.5,<6.0.0)"]
 s3 = ["boto3 (>=1.26.143)"]
 slmq = ["softlayer-messaging (>=1.0.3)"]
-solar = ["ephem (==4.1.4)"]
+solar = ["ephem (==4.1.5)"]
 sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"]
 sqs = ["boto3 (>=1.26.143)", "kombu[sqs] (>=5.3.0)", "pycurl (>=7.43.0.5)", "urllib3 (>=1.26.16)"]
 tblib = ["tblib (>=1.3.0)", "tblib (>=1.5.0)"]
 yaml = ["PyYAML (>=3.10)"]
 zookeeper = ["kazoo (>=1.3.1)"]
-zstd = ["zstandard (==0.21.0)"]
+zstd = ["zstandard (==0.22.0)"]
 
 [[package]]
 name = "certifi"
@@ -1096,7 +1096,7 @@ packaging = "*"
 prance = ">=0.18.2"
 pydantic = [
     {version = ">=1.10.0,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.4.0 || >2.4.0,<3.0", extras = ["email"], markers = "python_version >= \"3.12\" and python_version < \"4.0\""},
-    {version = ">=1.10.0,<2.4.0 || >2.4.0,<3.0", extras = ["email"], markers = "python_version >= \"3.11\" and python_version < \"4.0\""},
+    {version = ">=1.10.0,<2.4.0 || >2.4.0,<3.0", extras = ["email"], markers = "python_version >= \"3.11\" and python_version < \"3.12\""},
 ]
 PySnooper = ">=0.4.1,<2.0.0"
 toml = ">=0.10.0,<1.0.0"
@@ -1649,13 +1649,13 @@ idna = ">=2.0.0"
 
 [[package]]
 name = "faker"
-version = "19.13.0"
+version = "20.0.0"
 description = "Faker is a Python package that generates fake data for you."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "Faker-19.13.0-py3-none-any.whl", hash = "sha256:da880a76322db7a879c848a0771e129338e0a680a9f695fd9a3e7a6ac82b45e1"},
-    {file = "Faker-19.13.0.tar.gz", hash = "sha256:14ccb0aec342d33aa3889a864a56e5b3c2d56bce1b89f9189f4fbc128b9afc1e"},
+    {file = "Faker-20.0.0-py3-none-any.whl", hash = "sha256:171b27ba106cf69e30a91ac471407c2362bd6af27738e2461dc441aeff5eed91"},
+    {file = "Faker-20.0.0.tar.gz", hash = "sha256:df44b68b9d231e784f4bfe616d781576cfef9f0c5d9a17671bf84dc10d7b44d6"},
 ]
 
 [package.dependencies]
@@ -1834,14 +1834,8 @@ files = [
 [package.dependencies]
 google-auth = ">=2.14.1,<3.0.dev0"
 googleapis-common-protos = ">=1.56.2,<2.0.dev0"
-grpcio = [
-    {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""},
-    {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""},
-]
-grpcio-status = [
-    {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "extra == \"grpc\""},
-    {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""},
-]
+grpcio = {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}
+grpcio-status = {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}
 protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0"
 requests = ">=2.18.0,<3.0.0.dev0"
 
@@ -3296,38 +3290,38 @@ files = [
 
 [[package]]
 name = "mypy"
-version = "1.6.1"
+version = "1.7.0"
 description = "Optional static typing for Python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "mypy-1.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e5012e5cc2ac628177eaac0e83d622b2dd499e28253d4107a08ecc59ede3fc2c"},
-    {file = "mypy-1.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d8fbb68711905f8912e5af474ca8b78d077447d8f3918997fecbf26943ff3cbb"},
-    {file = "mypy-1.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21a1ad938fee7d2d96ca666c77b7c494c3c5bd88dff792220e1afbebb2925b5e"},
-    {file = "mypy-1.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b96ae2c1279d1065413965c607712006205a9ac541895004a1e0d4f281f2ff9f"},
-    {file = "mypy-1.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:40b1844d2e8b232ed92e50a4bd11c48d2daa351f9deee6c194b83bf03e418b0c"},
-    {file = "mypy-1.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:81af8adaa5e3099469e7623436881eff6b3b06db5ef75e6f5b6d4871263547e5"},
-    {file = "mypy-1.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8c223fa57cb154c7eab5156856c231c3f5eace1e0bed9b32a24696b7ba3c3245"},
-    {file = "mypy-1.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8032e00ce71c3ceb93eeba63963b864bf635a18f6c0c12da6c13c450eedb183"},
-    {file = "mypy-1.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4c46b51de523817a0045b150ed11b56f9fff55f12b9edd0f3ed35b15a2809de0"},
-    {file = "mypy-1.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:19f905bcfd9e167159b3d63ecd8cb5e696151c3e59a1742e79bc3bcb540c42c7"},
-    {file = "mypy-1.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:82e469518d3e9a321912955cc702d418773a2fd1e91c651280a1bda10622f02f"},
-    {file = "mypy-1.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d4473c22cc296425bbbce7e9429588e76e05bc7342da359d6520b6427bf76660"},
-    {file = "mypy-1.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59a0d7d24dfb26729e0a068639a6ce3500e31d6655df8557156c51c1cb874ce7"},
-    {file = "mypy-1.6.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cfd13d47b29ed3bbaafaff7d8b21e90d827631afda134836962011acb5904b71"},
-    {file = "mypy-1.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:eb4f18589d196a4cbe5290b435d135dee96567e07c2b2d43b5c4621b6501531a"},
-    {file = "mypy-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:41697773aa0bf53ff917aa077e2cde7aa50254f28750f9b88884acea38a16169"},
-    {file = "mypy-1.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7274b0c57737bd3476d2229c6389b2ec9eefeb090bbaf77777e9d6b1b5a9d143"},
-    {file = "mypy-1.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbaf4662e498c8c2e352da5f5bca5ab29d378895fa2d980630656178bd607c46"},
-    {file = "mypy-1.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bb8ccb4724f7d8601938571bf3f24da0da791fe2db7be3d9e79849cb64e0ae85"},
-    {file = "mypy-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:68351911e85145f582b5aa6cd9ad666c8958bcae897a1bfda8f4940472463c45"},
-    {file = "mypy-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:49ae115da099dcc0922a7a895c1eec82c1518109ea5c162ed50e3b3594c71208"},
-    {file = "mypy-1.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8b27958f8c76bed8edaa63da0739d76e4e9ad4ed325c814f9b3851425582a3cd"},
-    {file = "mypy-1.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:925cd6a3b7b55dfba252b7c4561892311c5358c6b5a601847015a1ad4eb7d332"},
-    {file = "mypy-1.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8f57e6b6927a49550da3d122f0cb983d400f843a8a82e65b3b380d3d7259468f"},
-    {file = "mypy-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:a43ef1c8ddfdb9575691720b6352761f3f53d85f1b57d7745701041053deff30"},
-    {file = "mypy-1.6.1-py3-none-any.whl", hash = "sha256:4cbe68ef919c28ea561165206a2dcb68591c50f3bcf777932323bc208d949cf1"},
-    {file = "mypy-1.6.1.tar.gz", hash = "sha256:4d01c00d09a0be62a4ca3f933e315455bde83f37f892ba4b08ce92f3cf44bcc1"},
+    {file = "mypy-1.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5da84d7bf257fd8f66b4f759a904fd2c5a765f70d8b52dde62b521972a0a2357"},
+    {file = "mypy-1.7.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a3637c03f4025f6405737570d6cbfa4f1400eb3c649317634d273687a09ffc2f"},
+    {file = "mypy-1.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b633f188fc5ae1b6edca39dae566974d7ef4e9aaaae00bc36efe1f855e5173ac"},
+    {file = "mypy-1.7.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d6ed9a3997b90c6f891138e3f83fb8f475c74db4ccaa942a1c7bf99e83a989a1"},
+    {file = "mypy-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:1fe46e96ae319df21359c8db77e1aecac8e5949da4773c0274c0ef3d8d1268a9"},
+    {file = "mypy-1.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:df67fbeb666ee8828f675fee724cc2cbd2e4828cc3df56703e02fe6a421b7401"},
+    {file = "mypy-1.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a79cdc12a02eb526d808a32a934c6fe6df07b05f3573d210e41808020aed8b5d"},
+    {file = "mypy-1.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f65f385a6f43211effe8c682e8ec3f55d79391f70a201575def73d08db68ead1"},
+    {file = "mypy-1.7.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e81ffd120ee24959b449b647c4b2fbfcf8acf3465e082b8d58fd6c4c2b27e46"},
+    {file = "mypy-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:f29386804c3577c83d76520abf18cfcd7d68264c7e431c5907d250ab502658ee"},
+    {file = "mypy-1.7.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:87c076c174e2c7ef8ab416c4e252d94c08cd4980a10967754f91571070bf5fbe"},
+    {file = "mypy-1.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6cb8d5f6d0fcd9e708bb190b224089e45902cacef6f6915481806b0c77f7786d"},
+    {file = "mypy-1.7.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d93e76c2256aa50d9c82a88e2f569232e9862c9982095f6d54e13509f01222fc"},
+    {file = "mypy-1.7.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cddee95dea7990e2215576fae95f6b78a8c12f4c089d7e4367564704e99118d3"},
+    {file = "mypy-1.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:d01921dbd691c4061a3e2ecdbfbfad029410c5c2b1ee88946bf45c62c6c91210"},
+    {file = "mypy-1.7.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:185cff9b9a7fec1f9f7d8352dff8a4c713b2e3eea9c6c4b5ff7f0edf46b91e41"},
+    {file = "mypy-1.7.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7a7b1e399c47b18feb6f8ad4a3eef3813e28c1e871ea7d4ea5d444b2ac03c418"},
+    {file = "mypy-1.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc9fe455ad58a20ec68599139ed1113b21f977b536a91b42bef3ffed5cce7391"},
+    {file = "mypy-1.7.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d0fa29919d2e720c8dbaf07d5578f93d7b313c3e9954c8ec05b6d83da592e5d9"},
+    {file = "mypy-1.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:2b53655a295c1ed1af9e96b462a736bf083adba7b314ae775563e3fb4e6795f5"},
+    {file = "mypy-1.7.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c1b06b4b109e342f7dccc9efda965fc3970a604db70f8560ddfdee7ef19afb05"},
+    {file = "mypy-1.7.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bf7a2f0a6907f231d5e41adba1a82d7d88cf1f61a70335889412dec99feeb0f8"},
+    {file = "mypy-1.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:551d4a0cdcbd1d2cccdcc7cb516bb4ae888794929f5b040bb51aae1846062901"},
+    {file = "mypy-1.7.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:55d28d7963bef00c330cb6461db80b0b72afe2f3c4e2963c99517cf06454e665"},
+    {file = "mypy-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:870bd1ffc8a5862e593185a4c169804f2744112b4a7c55b93eb50f48e7a77010"},
+    {file = "mypy-1.7.0-py3-none-any.whl", hash = "sha256:96650d9a4c651bc2a4991cf46f100973f656d69edc7faf91844e87fe627f7e96"},
+    {file = "mypy-1.7.0.tar.gz", hash = "sha256:1e280b5697202efa698372d2f39e9a6713a0395a756b1c6bd48995f8d72690dc"},
 ]
 
 [package.dependencies]
@@ -3337,6 +3331,7 @@ typing-extensions = ">=4.1.0"
 [package.extras]
 dmypy = ["psutil (>=4.0)"]
 install-types = ["pip"]
+mypyc = ["setuptools (>=50)"]
 reports = ["lxml"]
 
 [[package]]
@@ -3590,10 +3585,7 @@ files = [
 ]
 
 [package.dependencies]
-numpy = [
-    {version = ">=1.21.0", markers = "python_version >= \"3.10\""},
-    {version = ">=1.23.2", markers = "python_version >= \"3.11\""},
-]
+numpy = {version = ">=1.23.2", markers = "python_version >= \"3.11\""}
 python-dateutil = ">=2.8.1"
 pytz = ">=2020.1"
 
@@ -3622,6 +3614,19 @@ files = [
     {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"},
 ]
 
+[[package]]
+name = "pgvector"
+version = "0.2.3"
+description = "pgvector support for Python"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pgvector-0.2.3-py2.py3-none-any.whl", hash = "sha256:9d53dc01138ecc7c9aca64e4680cfa9edf4c38f9cb8ed7098317871fdd211824"},
+]
+
+[package.dependencies]
+numpy = "*"
+
 [[package]]
 name = "phonenumberslite"
 version = "8.13.24"
@@ -4392,6 +4397,103 @@ files = [
 attrs = ">=22.2.0"
 rpds-py = ">=0.7.0"
 
+[[package]]
+name = "regex"
+version = "2023.10.3"
+description = "Alternative regular expression module, to replace re."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "regex-2023.10.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4c34d4f73ea738223a094d8e0ffd6d2c1a1b4c175da34d6b0de3d8d69bee6bcc"},
+    {file = "regex-2023.10.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a8f4e49fc3ce020f65411432183e6775f24e02dff617281094ba6ab079ef0915"},
+    {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cd1bccf99d3ef1ab6ba835308ad85be040e6a11b0977ef7ea8c8005f01a3c29"},
+    {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:81dce2ddc9f6e8f543d94b05d56e70d03a0774d32f6cca53e978dc01e4fc75b8"},
+    {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c6b4d23c04831e3ab61717a707a5d763b300213db49ca680edf8bf13ab5d91b"},
+    {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c15ad0aee158a15e17e0495e1e18741573d04eb6da06d8b84af726cfc1ed02ee"},
+    {file = "regex-2023.10.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6239d4e2e0b52c8bd38c51b760cd870069f0bdf99700a62cd509d7a031749a55"},
+    {file = "regex-2023.10.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4a8bf76e3182797c6b1afa5b822d1d5802ff30284abe4599e1247be4fd6b03be"},
+    {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9c727bbcf0065cbb20f39d2b4f932f8fa1631c3e01fcedc979bd4f51fe051c5"},
+    {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3ccf2716add72f80714b9a63899b67fa711b654be3fcdd34fa391d2d274ce767"},
+    {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:107ac60d1bfdc3edb53be75e2a52aff7481b92817cfdddd9b4519ccf0e54a6ff"},
+    {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:00ba3c9818e33f1fa974693fb55d24cdc8ebafcb2e4207680669d8f8d7cca79a"},
+    {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f0a47efb1dbef13af9c9a54a94a0b814902e547b7f21acb29434504d18f36e3a"},
+    {file = "regex-2023.10.3-cp310-cp310-win32.whl", hash = "sha256:36362386b813fa6c9146da6149a001b7bd063dabc4d49522a1f7aa65b725c7ec"},
+    {file = "regex-2023.10.3-cp310-cp310-win_amd64.whl", hash = "sha256:c65a3b5330b54103e7d21cac3f6bf3900d46f6d50138d73343d9e5b2900b2353"},
+    {file = "regex-2023.10.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:90a79bce019c442604662d17bf69df99090e24cdc6ad95b18b6725c2988a490e"},
+    {file = "regex-2023.10.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c7964c2183c3e6cce3f497e3a9f49d182e969f2dc3aeeadfa18945ff7bdd7051"},
+    {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ef80829117a8061f974b2fda8ec799717242353bff55f8a29411794d635d964"},
+    {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5addc9d0209a9afca5fc070f93b726bf7003bd63a427f65ef797a931782e7edc"},
+    {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c148bec483cc4b421562b4bcedb8e28a3b84fcc8f0aa4418e10898f3c2c0eb9b"},
+    {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d1f21af4c1539051049796a0f50aa342f9a27cde57318f2fc41ed50b0dbc4ac"},
+    {file = "regex-2023.10.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b9ac09853b2a3e0d0082104036579809679e7715671cfbf89d83c1cb2a30f58"},
+    {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ebedc192abbc7fd13c5ee800e83a6df252bec691eb2c4bedc9f8b2e2903f5e2a"},
+    {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d8a993c0a0ffd5f2d3bda23d0cd75e7086736f8f8268de8a82fbc4bd0ac6791e"},
+    {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:be6b7b8d42d3090b6c80793524fa66c57ad7ee3fe9722b258aec6d0672543fd0"},
+    {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4023e2efc35a30e66e938de5aef42b520c20e7eda7bb5fb12c35e5d09a4c43f6"},
+    {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0d47840dc05e0ba04fe2e26f15126de7c755496d5a8aae4a08bda4dd8d646c54"},
+    {file = "regex-2023.10.3-cp311-cp311-win32.whl", hash = "sha256:9145f092b5d1977ec8c0ab46e7b3381b2fd069957b9862a43bd383e5c01d18c2"},
+    {file = "regex-2023.10.3-cp311-cp311-win_amd64.whl", hash = "sha256:b6104f9a46bd8743e4f738afef69b153c4b8b592d35ae46db07fc28ae3d5fb7c"},
+    {file = "regex-2023.10.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:bff507ae210371d4b1fe316d03433ac099f184d570a1a611e541923f78f05037"},
+    {file = "regex-2023.10.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:be5e22bbb67924dea15039c3282fa4cc6cdfbe0cbbd1c0515f9223186fc2ec5f"},
+    {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a992f702c9be9c72fa46f01ca6e18d131906a7180950958f766c2aa294d4b41"},
+    {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7434a61b158be563c1362d9071358f8ab91b8d928728cd2882af060481244c9e"},
+    {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2169b2dcabf4e608416f7f9468737583ce5f0a6e8677c4efbf795ce81109d7c"},
+    {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9e908ef5889cda4de038892b9accc36d33d72fb3e12c747e2799a0e806ec841"},
+    {file = "regex-2023.10.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12bd4bc2c632742c7ce20db48e0d99afdc05e03f0b4c1af90542e05b809a03d9"},
+    {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bc72c231f5449d86d6c7d9cc7cd819b6eb30134bb770b8cfdc0765e48ef9c420"},
+    {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bce8814b076f0ce5766dc87d5a056b0e9437b8e0cd351b9a6c4e1134a7dfbda9"},
+    {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:ba7cd6dc4d585ea544c1412019921570ebd8a597fabf475acc4528210d7c4a6f"},
+    {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b0c7d2f698e83f15228ba41c135501cfe7d5740181d5903e250e47f617eb4292"},
+    {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5a8f91c64f390ecee09ff793319f30a0f32492e99f5dc1c72bc361f23ccd0a9a"},
+    {file = "regex-2023.10.3-cp312-cp312-win32.whl", hash = "sha256:ad08a69728ff3c79866d729b095872afe1e0557251da4abb2c5faff15a91d19a"},
+    {file = "regex-2023.10.3-cp312-cp312-win_amd64.whl", hash = "sha256:39cdf8d141d6d44e8d5a12a8569d5a227f645c87df4f92179bd06e2e2705e76b"},
+    {file = "regex-2023.10.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4a3ee019a9befe84fa3e917a2dd378807e423d013377a884c1970a3c2792d293"},
+    {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76066d7ff61ba6bf3cb5efe2428fc82aac91802844c022d849a1f0f53820502d"},
+    {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe50b61bab1b1ec260fa7cd91106fa9fece57e6beba05630afe27c71259c59b"},
+    {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fd88f373cb71e6b59b7fa597e47e518282455c2734fd4306a05ca219a1991b0"},
+    {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3ab05a182c7937fb374f7e946f04fb23a0c0699c0450e9fb02ef567412d2fa3"},
+    {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dac37cf08fcf2094159922edc7a2784cfcc5c70f8354469f79ed085f0328ebdf"},
+    {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e54ddd0bb8fb626aa1f9ba7b36629564544954fff9669b15da3610c22b9a0991"},
+    {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3367007ad1951fde612bf65b0dffc8fd681a4ab98ac86957d16491400d661302"},
+    {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:16f8740eb6dbacc7113e3097b0a36065a02e37b47c936b551805d40340fb9971"},
+    {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:f4f2ca6df64cbdd27f27b34f35adb640b5d2d77264228554e68deda54456eb11"},
+    {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:39807cbcbe406efca2a233884e169d056c35aa7e9f343d4e78665246a332f597"},
+    {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:7eece6fbd3eae4a92d7c748ae825cbc1ee41a89bb1c3db05b5578ed3cfcfd7cb"},
+    {file = "regex-2023.10.3-cp37-cp37m-win32.whl", hash = "sha256:ce615c92d90df8373d9e13acddd154152645c0dc060871abf6bd43809673d20a"},
+    {file = "regex-2023.10.3-cp37-cp37m-win_amd64.whl", hash = "sha256:0f649fa32fe734c4abdfd4edbb8381c74abf5f34bc0b3271ce687b23729299ed"},
+    {file = "regex-2023.10.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9b98b7681a9437262947f41c7fac567c7e1f6eddd94b0483596d320092004533"},
+    {file = "regex-2023.10.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:91dc1d531f80c862441d7b66c4505cd6ea9d312f01fb2f4654f40c6fdf5cc37a"},
+    {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82fcc1f1cc3ff1ab8a57ba619b149b907072e750815c5ba63e7aa2e1163384a4"},
+    {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7979b834ec7a33aafae34a90aad9f914c41fd6eaa8474e66953f3f6f7cbd4368"},
+    {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef71561f82a89af6cfcbee47f0fabfdb6e63788a9258e913955d89fdd96902ab"},
+    {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd829712de97753367153ed84f2de752b86cd1f7a88b55a3a775eb52eafe8a94"},
+    {file = "regex-2023.10.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00e871d83a45eee2f8688d7e6849609c2ca2a04a6d48fba3dff4deef35d14f07"},
+    {file = "regex-2023.10.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:706e7b739fdd17cb89e1fbf712d9dc21311fc2333f6d435eac2d4ee81985098c"},
+    {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cc3f1c053b73f20c7ad88b0d1d23be7e7b3901229ce89f5000a8399746a6e039"},
+    {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6f85739e80d13644b981a88f529d79c5bdf646b460ba190bffcaf6d57b2a9863"},
+    {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:741ba2f511cc9626b7561a440f87d658aabb3d6b744a86a3c025f866b4d19e7f"},
+    {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e77c90ab5997e85901da85131fd36acd0ed2221368199b65f0d11bca44549711"},
+    {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:979c24cbefaf2420c4e377ecd1f165ea08cc3d1fbb44bdc51bccbbf7c66a2cb4"},
+    {file = "regex-2023.10.3-cp38-cp38-win32.whl", hash = "sha256:58837f9d221744d4c92d2cf7201c6acd19623b50c643b56992cbd2b745485d3d"},
+    {file = "regex-2023.10.3-cp38-cp38-win_amd64.whl", hash = "sha256:c55853684fe08d4897c37dfc5faeff70607a5f1806c8be148f1695be4a63414b"},
+    {file = "regex-2023.10.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2c54e23836650bdf2c18222c87f6f840d4943944146ca479858404fedeb9f9af"},
+    {file = "regex-2023.10.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:69c0771ca5653c7d4b65203cbfc5e66db9375f1078689459fe196fe08b7b4930"},
+    {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ac965a998e1388e6ff2e9781f499ad1eaa41e962a40d11c7823c9952c77123e"},
+    {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c0e8fae5b27caa34177bdfa5a960c46ff2f78ee2d45c6db15ae3f64ecadde14"},
+    {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6c56c3d47da04f921b73ff9415fbaa939f684d47293f071aa9cbb13c94afc17d"},
+    {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ef1e014eed78ab650bef9a6a9cbe50b052c0aebe553fb2881e0453717573f52"},
+    {file = "regex-2023.10.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d29338556a59423d9ff7b6eb0cb89ead2b0875e08fe522f3e068b955c3e7b59b"},
+    {file = "regex-2023.10.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9c6d0ced3c06d0f183b73d3c5920727268d2201aa0fe6d55c60d68c792ff3588"},
+    {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:994645a46c6a740ee8ce8df7911d4aee458d9b1bc5639bc968226763d07f00fa"},
+    {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:66e2fe786ef28da2b28e222c89502b2af984858091675044d93cb50e6f46d7af"},
+    {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:11175910f62b2b8c055f2b089e0fedd694fe2be3941b3e2633653bc51064c528"},
+    {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:06e9abc0e4c9ab4779c74ad99c3fc10d3967d03114449acc2c2762ad4472b8ca"},
+    {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fb02e4257376ae25c6dd95a5aec377f9b18c09be6ebdefa7ad209b9137b73d48"},
+    {file = "regex-2023.10.3-cp39-cp39-win32.whl", hash = "sha256:3b2c3502603fab52d7619b882c25a6850b766ebd1b18de3df23b2f939360e1bd"},
+    {file = "regex-2023.10.3-cp39-cp39-win_amd64.whl", hash = "sha256:adbccd17dcaff65704c856bd29951c58a1bd4b2b0f8ad6b826dbd543fe740988"},
+    {file = "regex-2023.10.3.tar.gz", hash = "sha256:3fef4f844d2290ee0ba57addcec17eec9e3df73f10a2748485dfd6a3a188cc0f"},
+]
+
 [[package]]
 name = "requests"
 version = "2.31.0"
@@ -5090,6 +5192,51 @@ strawberry-graphql = ">=0.199.0"
 debug-toolbar = ["django-debug-toolbar (>=3.4)"]
 enum = ["django-choices-field (>=2.2.2)"]
 
+[[package]]
+name = "tiktoken"
+version = "0.5.1"
+description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "tiktoken-0.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2b0bae3fd56de1c0a5874fb6577667a3c75bf231a6cef599338820210c16e40a"},
+    {file = "tiktoken-0.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e529578d017045e2f0ed12d2e00e7e99f780f477234da4aae799ec4afca89f37"},
+    {file = "tiktoken-0.5.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edd2ffbb789712d83fee19ab009949f998a35c51ad9f9beb39109357416344ff"},
+    {file = "tiktoken-0.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4c73d47bdc1a3f1f66ffa019af0386c48effdc6e8797e5e76875f6388ff72e9"},
+    {file = "tiktoken-0.5.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:46b8554b9f351561b1989157c6bb54462056f3d44e43aa4e671367c5d62535fc"},
+    {file = "tiktoken-0.5.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:92ed3bbf71a175a6a4e5fbfcdb2c422bdd72d9b20407e00f435cf22a68b4ea9b"},
+    {file = "tiktoken-0.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:714efb2f4a082635d9f5afe0bf7e62989b72b65ac52f004eb7ac939f506c03a4"},
+    {file = "tiktoken-0.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a10488d1d1a5f9c9d2b2052fdb4cf807bba545818cb1ef724a7f5d44d9f7c3d4"},
+    {file = "tiktoken-0.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8079ac065572fe0e7c696dbd63e1fdc12ce4cdca9933935d038689d4732451df"},
+    {file = "tiktoken-0.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ef730db4097f5b13df8d960f7fdda2744fe21d203ea2bb80c120bb58661b155"},
+    {file = "tiktoken-0.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:426e7def5f3f23645dada816be119fa61e587dfb4755de250e136b47a045c365"},
+    {file = "tiktoken-0.5.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:323cec0031358bc09aa965c2c5c1f9f59baf76e5b17e62dcc06d1bb9bc3a3c7c"},
+    {file = "tiktoken-0.5.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5abd9436f02e2c8eda5cce2ff8015ce91f33e782a7423de2a1859f772928f714"},
+    {file = "tiktoken-0.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:1fe99953b63aabc0c9536fbc91c3c9000d78e4755edc28cc2e10825372046a2d"},
+    {file = "tiktoken-0.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:dcdc630461927718b317e6f8be7707bd0fc768cee1fdc78ddaa1e93f4dc6b2b1"},
+    {file = "tiktoken-0.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1f2b3b253e22322b7f53a111e1f6d7ecfa199b4f08f3efdeb0480f4033b5cdc6"},
+    {file = "tiktoken-0.5.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43ce0199f315776dec3ea7bf86f35df86d24b6fcde1babd3e53c38f17352442f"},
+    {file = "tiktoken-0.5.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a84657c083d458593c0235926b5c993eec0b586a2508d6a2020556e5347c2f0d"},
+    {file = "tiktoken-0.5.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c008375c0f3d97c36e81725308699116cd5804fdac0f9b7afc732056329d2790"},
+    {file = "tiktoken-0.5.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:779c4dea5edd1d3178734d144d32231e0b814976bec1ec09636d1003ffe4725f"},
+    {file = "tiktoken-0.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:b5dcfcf9bfb798e86fbce76d40a1d5d9e3f92131aecfa3d1e5c9ea1a20f1ef1a"},
+    {file = "tiktoken-0.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b180a22db0bbcc447f691ffc3cf7a580e9e0587d87379e35e58b826ebf5bc7b"},
+    {file = "tiktoken-0.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2b756a65d98b7cf760617a6b68762a23ab8b6ef79922be5afdb00f5e8a9f4e76"},
+    {file = "tiktoken-0.5.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba9873c253ca1f670e662192a0afcb72b41e0ba3e730f16c665099e12f4dac2d"},
+    {file = "tiktoken-0.5.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74c90d2be0b4c1a2b3f7dde95cd976757817d4df080d6af0ee8d461568c2e2ad"},
+    {file = "tiktoken-0.5.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:709a5220891f2b56caad8327fab86281787704931ed484d9548f65598dea9ce4"},
+    {file = "tiktoken-0.5.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5d5a187ff9c786fae6aadd49f47f019ff19e99071dc5b0fe91bfecc94d37c686"},
+    {file = "tiktoken-0.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:e21840043dbe2e280e99ad41951c00eff8ee3b63daf57cd4c1508a3fd8583ea2"},
+    {file = "tiktoken-0.5.1.tar.gz", hash = "sha256:27e773564232004f4f810fd1f85236673ec3a56ed7f1206fc9ed8670ebedb97a"},
+]
+
+[package.dependencies]
+regex = ">=2022.1.18"
+requests = ">=2.26.0"
+
+[package.extras]
+blobfile = ["blobfile (>=2)"]
+
 [[package]]
 name = "toml"
 version = "0.10.2"
@@ -5807,4 +5954,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.11"
-content-hash = "3fa0415dfc91463fe63c3fd4feab789217fbf93b0c52035c2fc8da85f83ed9a9"
+content-hash = "4deb40390223246c067c00b597ff10f6c048131b5274664108eba5d88b801a19"
diff --git a/grai-server/app/pyproject.toml b/grai-server/app/pyproject.toml
index 3c697f01c..71d3257f1 100644
--- a/grai-server/app/pyproject.toml
+++ b/grai-server/app/pyproject.toml
@@ -72,6 +72,8 @@ asgi-cors-strawberry = "^0.2.0"
 overrides = "^7.4.0"
 wsproto = "^1.2.0"
 openai = "^0.28.1"
+tiktoken = "^0.5.1"
+pgvector = "^0.2.3"
 pyjwt = "^2.8.0"
 
 [tool.poetry.group.dev.dependencies]
diff --git a/grai-server/app/the_guide/celery.py b/grai-server/app/the_guide/celery.py
index cc8034c13..ddbc509d9 100644
--- a/grai-server/app/the_guide/celery.py
+++ b/grai-server/app/the_guide/celery.py
@@ -1,5 +1,5 @@
 import os
-
+from datetime import timedelta
 from celery import Celery
 
 # Set the default Django settings module for the 'celery' program.
@@ -13,5 +13,12 @@
 #   should have a `CELERY_` prefix.
 app.config_from_object("django.conf:settings", namespace="CELERY")
 
+app.conf.beat_schedule = {
+    # "lineage:Node:bulk_update_embeddings": {
+    #     "task": "lineage.tasks.bulk_update_embeddings",
+    #     "schedule": timedelta(minutes=5),
+    # },
+}
+
 # Load task modules from all registered Django apps.
 app.autodiscover_tasks()
diff --git a/grai-server/app/the_guide/settings/base.py b/grai-server/app/the_guide/settings/base.py
index 88a111a35..46b60a46a 100755
--- a/grai-server/app/the_guide/settings/base.py
+++ b/grai-server/app/the_guide/settings/base.py
@@ -2,7 +2,7 @@
 import subprocess
 import warnings
 from pathlib import Path
-
+import logging
 import openai
 from decouple import config
 
@@ -368,36 +368,55 @@ def inner(value: str | bool) -> bool:
 
 OPENAI_API_KEY = config("OPENAI_API_KEY", None)
 OPENAI_ORG_ID = config("OPENAI_ORG_ID", None)
-OPENAI_PREFERRED_MODEL = config("OPENAI_PREFERRED_MODEL", "gpt-3.5-turbo")
+OPENAI_PREFERRED_MODEL = config("OPENAI_PREFERRED_MODEL", "gpt-3.5-turbo-1106")
 
 openai.organization = OPENAI_ORG_ID
 openai.api_key = OPENAI_API_KEY
 
-try:
-    models = [item["id"] for item in openai.Model.list()["data"]]
-except openai.error.AuthenticationError as e:
-    HAS_OPENAI = False
-else:
-    if len(models) == 0:
-        message = f"Provided OpenAI API key does not have access to any models as a result we've disabled OpenAI."
-        warnings.warn(message)
-
+if OPENAI_API_KEY is not None and OPENAI_ORG_ID is not None:
+    try:
+        models = [item["id"] for item in openai.Model.list()["data"]]
+    except openai.error.AuthenticationError as e:
+        warnings.warn("Could not authenticate with OpenAI API key and organization id.")
         HAS_OPENAI = False
-        OPENAI_PREFERRED_MODEL = ""
-    elif OPENAI_PREFERRED_MODEL not in models:
-        default_model = models[0]
-        message = (
-            f"Provided OpenAI API key does not have access to the preferred model {OPENAI_PREFERRED_MODEL}. "
-            f"If you wish to use {OPENAI_PREFERRED_MODEL} please provide an API key with appropriate permissions. "
-            f"In the mean time we've defaulted to {default_model}."
-        )
-        warnings.warn(message)
-
-        HAS_OPENAI = True
-        OPENAI_PREFERRED_MODEL = default_model
     else:
-        HAS_OPENAI = True
+        if len(models) == 0:
+            message = f"Provided OpenAI API key does not have access to any models as a result we've disabled OpenAI."
+            warnings.warn(message)
+
+            HAS_OPENAI = False
+            OPENAI_PREFERRED_MODEL = ""
+        elif OPENAI_PREFERRED_MODEL not in models:
+            default_model = models[0]
+            message = (
+                f"Provided OpenAI API key does not have access to the preferred model {OPENAI_PREFERRED_MODEL}. "
+                f"If you wish to use {OPENAI_PREFERRED_MODEL} please provide an API key with appropriate permissions. "
+                f"In the mean time we've defaulted to {default_model}."
+            )
+            warnings.warn(message)
+
+            HAS_OPENAI = True
+            OPENAI_PREFERRED_MODEL = default_model
+        else:
+            HAS_OPENAI = True
+else:
+    HAS_OPENAI = False
+
 
+if HAS_OPENAI:
+    pass
+    # TODO: Need to bake the encodings into the docker image otherwise it gets fetched every time
+    TIKTOKEN_CACHE_DIR = "/TIKTOKEN_CACHE_DIR"
+    # if not os.path.exists(TIKTOKEN_CACHE_DIR):
+    #     os.makedirs(TIKTOKEN_CACHE_DIR)
+
+    # os.environ["TIKTOKEN_CACHE_DIR"] = TIKTOKEN_CACHE_DIR
+    # import tiktoken
+    # # download the OpenAI preferred model encoder
+    # try:
+    #     tiktoken.encoding_for_model(OPENAI_PREFERRED_MODEL)
+    # except:
+    #     logging.error("Could not download OpenAI preferred model encoder with tiktoken")
 
 SPECTACULAR_SETTINGS = {
     "TITLE": "Grai Server",
diff --git a/grai-server/docker-compose.yml b/grai-server/docker-compose.yml
index f954e4404..25dabb13c 100755
--- a/grai-server/docker-compose.yml
+++ b/grai-server/docker-compose.yml
@@ -38,6 +38,8 @@ services:
       - DB_HOST=db
       - REDIS_HOST=redis
       - DJANGO_SETTINGS_MODULE=the_guide.settings.dev
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - OPENAI_ORG_ID=${OPENAI_ORG_ID}
     depends_on:
       - redis
       - db
@@ -53,6 +55,8 @@ services:
       - DB_HOST=db
       - REDIS_HOST=redis
       - DJANGO_SETTINGS_MODULE=the_guide.settings.dev
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - OPENAI_ORG_ID=${OPENAI_ORG_ID}
     depends_on:
       - redis
       - db
@@ -67,7 +71,7 @@ services:
       - the_guide
 
   db:
-    image: postgres:14.3-alpine
+    image: ankane/pgvector:latest
     shm_size: 1g
     environment:
       - POSTGRES_USER=grai