Merge pull request #63 from mindvalley/feat/add-at-connector

Adding support for AT connector

backend/danswer/configs/constants.py

@@ -66,6 +66,7 @@
 
 class DocumentSource(str, Enum):
     # Special case, document passed in via Danswer APIs without specifying a source type
+    AIRTABLE = "airtable"
     INGESTION_API = "ingestion_api"
     SLACK = "slack"
     WEB = "web"

backend/danswer/connectors/airtable/connector.py

@@ -0,0 +1,70 @@
+import json
+from typing import Any
+
+from danswer.configs.app_configs import INDEX_BATCH_SIZE
+from danswer.configs.constants import DocumentSource
+from danswer.connectors.interfaces import GenerateDocumentsOutput
+from danswer.connectors.interfaces import LoadConnector
+from danswer.connectors.interfaces import PollConnector
+from danswer.connectors.interfaces import SecondsSinceUnixEpoch
+from danswer.connectors.models import Document
+from danswer.connectors.models import Section
+from pyairtable import Api as AirtableApi
+
+
+class AirtableClientNotSetUpError(PermissionError):
+    def __init__(self) -> None:
+        super().__init__("Airtable Client is not set up, was load_credentials called?")
+
+
+class AirtableConnector(LoadConnector, PollConnector):
+    def __init__(
+        self,
+        base_id: str,
+        table_name_or_id: str,
+        batch_size: int = INDEX_BATCH_SIZE,
+    ) -> None:
+        self.base_id = base_id
+        self.table_name_or_id = table_name_or_id
+        self.batch_size = batch_size
+        self.airtable_client: AirtableApi | None = None
+
+    def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
+        self.airtable_client = AirtableApi(credentials["airtable_access_token"])
+
+        return None
+
+    def poll_source(
+        self, start: SecondsSinceUnixEpoch | None, end: SecondsSinceUnixEpoch | None
+    ) -> GenerateDocumentsOutput:
+        if not self.airtable_client:
+            raise AirtableClientNotSetUpError()
+
+        table = self.airtable_client.table(self.base_id, self.table_name_or_id)
+        all_records = table.all()
+
+        record_documents = []
+        for record in all_records:
+            record_document = Document(
+                id=str(record.get("id")),
+                sections=[
+                    Section(
+                        link=f"https://airtable.com/{self.base_id}/{self.table_name_or_id}/",
+                        text=json.dumps(record.get("fields")),
+                    )
+                ],
+                source=DocumentSource.AIRTABLE,
+                semantic_identifier=f"Airtable Base ID: {self.base_id}. Table Name or ID: {self.table_name_or_id}",
+                metadata={
+                    "type": "airtable",
+                    "created_time": record.get("createdTime"),
+                },
+            )
+            record_documents.append(record_document)
+
+        yield record_documents
+
+    def load_from_state(self) -> GenerateDocumentsOutput:
+        if not self.airtable_client:
+            raise AirtableClientNotSetUpError()
+        return self.poll_source(None, None)

backend/danswer/connectors/factory.py

@@ -1,9 +1,8 @@
 from typing import Any
 from typing import Type
 
-from sqlalchemy.orm import Session
-
 from danswer.configs.constants import DocumentSource
+from danswer.connectors.airtable.connector import AirtableConnector
 from danswer.connectors.axero.connector import AxeroConnector
 from danswer.connectors.blob.connector import BlobStorageConnector
 from danswer.connectors.bookstack.connector import BookstackConnector
@@ -45,6 +44,7 @@
 from danswer.connectors.zulip.connector import ZulipConnector
 from danswer.db.credentials import backend_update_credential_json
 from danswer.db.models import Credential
+from sqlalchemy.orm import Session
 
 
 class ConnectorMissingException(Exception):
@@ -58,6 +58,7 @@ def identify_connector_class(
     connector_map = {
         DocumentSource.WEB: WebConnector,
         DocumentSource.FILE: LocalFileConnector,
+        DocumentSource.AIRTABLE: AirtableConnector,
         DocumentSource.SLACK: {
             InputType.LOAD_STATE: SlackLoadConnector,
             InputType.POLL: SlackPollConnector,

backend/requirements/default.txt

@@ -74,3 +74,4 @@ zenpy==2.0.41
 dropbox==11.36.2
 boto3-stubs[s3]==1.34.133
 ultimate_sitemap_parser==0.5
+pyairtable==3.0.0a3

web/src/components/icons/icons.tsx

@@ -88,6 +88,7 @@ import voyageIcon from "../../../public/Voyage.png";
 import googleIcon from "../../../public/Google.webp";
 
 import { FaRobot } from "react-icons/fa";
+import airtableIcon from "../../../public/Airtable.png";
 
 export interface IconProps {
   size?: number;
@@ -999,6 +1000,20 @@ export const LightSettingsIcon = ({
   );
 };
 
+export const AirtableIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => {
+  return (
+    <div
+      style={{ width: `${size}px`, height: `${size}px` }}
+      className={`w-[${size}px] h-[${size}px] ` + className}
+    >
+      <Image src={airtableIcon} alt="Logo" width="96" height="96" />
+    </div>
+  );
+};
+
 //
 //  COMPANY LOGOS
 //
@@ -2112,7 +2127,7 @@ export const CpuIcon = ({
         <path d="M16.5 7.5h-9v9h9v-9Z" />
         <path
           fillRule="evenodd"
-          d="M8.25 2.25A.75.75 0 0 1 9 3v.75h2.25V3a.75.75 0 0 1 1.5 0v.75H15V3a.75.75 0 0 1 1.5 0v.75h.75a3 3 0 0 1 3 3v.75H21A.75.75 0 0 1 21 9h-.75v2.25H21a.75.75 0 0 1 0 1.5h-.75V15H21a.75.75 0 0 1 0 1.5h-.75v.75a3 3 0 0 1-3 3h-.75V21a.75.75 0 0 1-1.5 0v-.75h-2.25V21a.75.75 0 0 1-1.5 0v-.75H9V21a.75.75 0 0 1-1.5 0v-.75h-.75a3 3 0 0 1-3-3v-.75H3A.75.75 0 0 1 3 15h.75v-2.25H3a.75.75 0 0 1 0-1.5h.75V9H3a.75.75 0 0 1 0-1.5h.75v-.75a3 3 0 0 1 3-3h.75V3a.75.75 0 0 1 .75-.75ZM6 6.75A.75.75 0 0 1 6.75 6h10.5a.75.75 0 0 1 .75.75v10.5a.75.75 0 0 1-.75.75H6.75a.75.75 0 0 1-.75-.75V6.75Z"
+          d="M8.25 2.25A.75.75 0 0 1 9 3v.75h2.25V3a.75.75 0 0 1 1.5 0v.75H15V3a.75.75 0 0 1 1.5 0v.75h.75a3 3 0 0 1 3 3v.75H21A.75.75 0 0 1 21 9h-.75v2.25H21a.75.75 0 0 1 0 1.5h-.75V15H21a.75.75 0 0 1 0 1.5h-.75v.75a3 3 0 0 1-3 3h-.75V21a.75.75 0 0 1-1.5 0v-.75h-2.25V21a.75.75 0 0 1-1.5 0v-.75H9V21a.75.75 0 0 1-1.5 0v-.75h-.75a3 3 0 0 1-3-3v-.75H3A.75.75 0 0 1 3 15h.75v-2.25H3a.75.75 0 0 1 0-1.5h.75v-.75a3 3 0 0 1 3-3h.75V3a.75.75 0 0 1 .75-.75ZM6 6.75A.75.75 0 0 1 6.75 6h10.5a.75.75 0 0 1 .75.75v10.5a.75.75 0 0 1-.75.75H6.75a.75.75 0 0 1-.75-.75V6.75Z"
           clipRule="evenodd"
         />
       </g>

web/src/lib/connectors/connectors.ts

@@ -809,6 +809,28 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
       },
     ],
   },
+  airtable: {
+    description: "Configure Airtable connector",
+    values: [
+      {
+        type: "text",
+        query: "Enter the Airtable Base ID:",
+        label: "Base ID",
+        name: "base_id",
+        optional: false,
+        description: "The ID of the Airtable base you want to connect to.",
+      },
+      {
+        type: "text",
+        query: "Enter the Airtable Table Name or ID:",
+        label: "Table Name or ID",
+        name: "table_name_or_id",
+        optional: false,
+        description:
+          "The name or ID of the specific table within the Airtable base.",
+      },
+    ],
+  },
 };
 export function createConnectorInitialValues(
   connector: ConfigurableSources
@@ -819,21 +841,18 @@ export function createConnectorInitialValues(
     name: "",
     groups: [],
     is_public: true,
-    ...configuration.values.reduce(
-      (acc, field) => {
-        if (field.type === "select") {
-          acc[field.name] = null;
-        } else if (field.type === "list") {
-          acc[field.name] = field.default || [];
-        } else if (field.type === "checkbox") {
-          acc[field.name] = field.default || false;
-        } else if (field.default !== undefined) {
-          acc[field.name] = field.default;
-        }
-        return acc;
-      },
-      {} as { [record: string]: any }
-    ),
+    ...configuration.values.reduce((acc, field) => {
+      if (field.type === "select") {
+        acc[field.name] = null;
+      } else if (field.type === "list") {
+        acc[field.name] = field.default || [];
+      } else if (field.type === "checkbox") {
+        acc[field.name] = field.default || false;
+      } else if (field.default !== undefined) {
+        acc[field.name] = field.default;
+      }
+      return acc;
+    }, {} as { [record: string]: any }),
   };
 }
 
@@ -844,28 +863,25 @@ export function createConnectorValidationSchema(
 
   return Yup.object().shape({
     name: Yup.string().required("Connector Name is required"),
-    ...configuration.values.reduce(
-      (acc, field) => {
-        let schema: any =
-          field.type === "select"
-            ? Yup.string()
-            : field.type === "list"
-              ? Yup.array().of(Yup.string())
-              : field.type === "checkbox"
-                ? Yup.boolean()
-                : field.type === "file"
-                  ? Yup.mixed()
-                  : Yup.string();
+    ...configuration.values.reduce((acc, field) => {
+      let schema: any =
+        field.type === "select"
+          ? Yup.string()
+          : field.type === "list"
+          ? Yup.array().of(Yup.string())
+          : field.type === "checkbox"
+          ? Yup.boolean()
+          : field.type === "file"
+          ? Yup.mixed()
+          : Yup.string();
 
-        if (!field.optional) {
-          schema = schema.required(`${field.label} is required`);
-        }
+      if (!field.optional) {
+        schema = schema.required(`${field.label} is required`);
+      }
 
-        acc[field.name] = schema;
-        return acc;
-      },
-      {} as Record<string, any>
-    ),
+      acc[field.name] = schema;
+      return acc;
+    }, {} as Record<string, any>),
     // These are advanced settings
     indexingStart: Yup.string().nullable(),
     pruneFreq: Yup.number().min(0, "Prune frequency must be non-negative"),
@@ -1060,3 +1076,8 @@ export interface MediaWikiConfig extends MediaWikiBaseConfig {
 }
 
 export interface WikipediaConfig extends MediaWikiBaseConfig {}
+
+export interface AirtableConfig {
+  base_id: string;
+  table_name_or_id: string;
+}

web/src/lib/connectors/credentials.ts

@@ -182,6 +182,10 @@ export interface AxeroCredentialJson {
   axero_api_token: string;
 }
 
+export interface AirtableCredentialJson {
+  airtable_access_token: string;
+}
+
 export interface MediaWikiCredentialJson {}
 export interface WikipediaCredentialJson extends MediaWikiCredentialJson {}
 
@@ -282,6 +286,7 @@ export const credentialTemplates: Record<ValidSources, any> = {
     access_key_id: "",
     secret_access_key: "",
   } as OCICredentialJson,
+  airtable: { airtable_access_token: "" } as AirtableCredentialJson,
   google_sites: null,
   file: null,
   wikipedia: null,
@@ -424,6 +429,9 @@ export const credentialDisplayNames: Record<string, string> = {
   // Axero
   base_url: "Axero Base URL",
   axero_api_token: "Axero API Token",
+
+  // Airtable
+  airtable_access_token: "Airtable Access Token",
 };
 export function getDisplayNameForCredentialKey(key: string): string {
   return credentialDisplayNames[key] || key;

web/src/lib/sources.ts

@@ -36,6 +36,7 @@ import {
   OCIStorageIcon,
   GoogleStorageIcon,
   ColorSlackIcon,
+  AirtableIcon,
 } from "@/components/icons/icons";
 import { ValidSources } from "./types";
 import {
@@ -277,6 +278,12 @@ const SOURCE_METADATA_MAP: SourceMap = {
     displayName: "Ingestion",
     category: SourceCategory.Other,
   },
+  airtable: {
+    icon: AirtableIcon,
+    displayName: "Airtable",
+    category: SourceCategory.Wiki,
+    docs: "https://docs.danswer.dev/connectors/airtable",
+  },
   // currently used for the Internet Search tool docs, which is why
   // a globe is used
   not_applicable: {

web/src/lib/types.ts

@@ -212,6 +212,7 @@ export interface UserGroup {
 }
 
 const validSources = [
+  "airtable",
   "web",
   "github",
   "gitlab",