-
- {indexAttempt.error_msg || "-"}
-
+ {indexAttempt.error_count > 0 && (
+
+
+
+ View Errors
+
+
+ )}
+
+ {indexAttempt.status === "success" && (
+
+ {"-"}
+
+ )}
+
+ {indexAttempt.status === "failed" &&
+ indexAttempt.error_msg && (
+
+ {indexAttempt.error_msg}
+
+ )}
+
{indexAttempt.full_exception_trace && (
{
diff --git a/web/src/app/admin/connector/[ccPairId]/ModifyStatusButtonCluster.tsx b/web/src/app/admin/connector/[ccPairId]/ModifyStatusButtonCluster.tsx
index 83d6363f621..10460459e32 100644
--- a/web/src/app/admin/connector/[ccPairId]/ModifyStatusButtonCluster.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/ModifyStatusButtonCluster.tsx
@@ -1,11 +1,11 @@
"use client";
import { Button } from "@tremor/react";
-import { CCPairFullInfo } from "./types";
+import { CCPairFullInfo, ConnectorCredentialPairStatus } from "./types";
import { usePopup } from "@/components/admin/connectors/Popup";
-import { disableConnector } from "@/lib/connector";
import { mutate } from "swr";
import { buildCCPairInfoUrl } from "./lib";
+import { setCCPairStatus } from "@/lib/ccPair";
export function ModifyStatusButtonCluster({
ccPair,
@@ -17,13 +17,16 @@ export function ModifyStatusButtonCluster({
return (
<>
{popup}
- {ccPair.connector.disabled ? (
+ {ccPair.status === ConnectorCredentialPairStatus.PAUSED ? (
);
diff --git a/web/src/app/admin/connector/[ccPairId]/lib.ts b/web/src/app/admin/connector/[ccPairId]/lib.ts
index e83f3d406d0..c2d02b23d75 100644
--- a/web/src/app/admin/connector/[ccPairId]/lib.ts
+++ b/web/src/app/admin/connector/[ccPairId]/lib.ts
@@ -1,3 +1,13 @@
+import { ValidSources } from "@/lib/types";
+
export function buildCCPairInfoUrl(ccPairId: string | number) {
return `/api/manage/admin/cc-pair/${ccPairId}`;
}
+
+export function buildSimilarCredentialInfoURL(
+ source_type: ValidSources,
+ get_editable: boolean = false
+) {
+ const base = `/api/manage/admin/similar-credentials/${source_type}`;
+ return get_editable ? `${base}?get_editable=True` : base;
+}
diff --git a/web/src/app/admin/connector/[ccPairId]/page.tsx b/web/src/app/admin/connector/[ccPairId]/page.tsx
index 7e613461bfc..f5da225a867 100644
--- a/web/src/app/admin/connector/[ccPairId]/page.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/page.tsx
@@ -1,23 +1,29 @@
"use client";
-import { CCPairFullInfo } from "./types";
+import { CCPairFullInfo, ConnectorCredentialPairStatus } from "./types";
import { HealthCheckBanner } from "@/components/health/healthcheck";
import { CCPairStatus } from "@/components/Status";
import { BackButton } from "@/components/BackButton";
-import { Divider, Title } from "@tremor/react";
+import { Button, Divider, Title } from "@tremor/react";
import { IndexingAttemptsTable } from "./IndexingAttemptsTable";
-import { Text } from "@tremor/react";
-import { ConfigDisplay } from "./ConfigDisplay";
+import { AdvancedConfigDisplay, ConfigDisplay } from "./ConfigDisplay";
import { ModifyStatusButtonCluster } from "./ModifyStatusButtonCluster";
import { DeletionButton } from "./DeletionButton";
import { ErrorCallout } from "@/components/ErrorCallout";
import { ReIndexButton } from "./ReIndexButton";
import { isCurrentlyDeleting } from "@/lib/documentDeletion";
import { ValidSources } from "@/lib/types";
-import useSWR from "swr";
+import useSWR, { mutate } from "swr";
import { errorHandlingFetcher } from "@/lib/fetcher";
import { ThreeDotsLoader } from "@/components/Loading";
+import CredentialSection from "@/components/credentials/CredentialSection";
import { buildCCPairInfoUrl } from "./lib";
+import { SourceIcon } from "@/components/SourceIcon";
+import { credentialTemplates } from "@/lib/connectors/credentials";
+import { useEffect, useRef, useState } from "react";
+import { CheckmarkIcon, EditIcon, XIcon } from "@/components/icons/icons";
+import { usePopup } from "@/components/admin/connectors/Popup";
+import { updateConnectorCredentialPairName } from "@/lib/connector";
// since the uploaded files are cleaned up after some period of time
// re-indexing will not work for the file connector. Also, it would not
@@ -35,6 +41,43 @@ function Main({ ccPairId }: { ccPairId: number }) {
{ refreshInterval: 5000 } // 5 seconds
);
+ const [editableName, setEditableName] = useState(ccPair?.name || "");
+ const [isEditing, setIsEditing] = useState(false);
+ const inputRef = useRef
(null);
+
+ const { popup, setPopup } = usePopup();
+ useEffect(() => {
+ if (isEditing && inputRef.current) {
+ inputRef.current.focus();
+ }
+ }, [isEditing]);
+ const handleNameChange = (e: React.ChangeEvent) => {
+ setEditableName(e.target.value);
+ };
+
+ const handleUpdateName = async () => {
+ try {
+ const response = await updateConnectorCredentialPairName(
+ ccPair?.id!,
+ editableName
+ );
+ if (!response.ok) {
+ throw new Error(await response.text());
+ }
+ mutate(buildCCPairInfoUrl(ccPairId));
+ setIsEditing(false);
+ setPopup({
+ message: "Connector name updated successfully",
+ type: "success",
+ });
+ } catch (error) {
+ setPopup({
+ message: `Failed to update connector name`,
+ type: "error",
+ });
+ }
+ };
+
if (isLoading) {
return ;
}
@@ -49,7 +92,7 @@ function Main({ ccPairId }: { ccPairId: number }) {
}
const lastIndexAttempt = ccPair.index_attempts[0];
- const isDeleting = isCurrentlyDeleting(ccPair.latest_deletion_attempt);
+ const isDeleting = ccPair.status === ConnectorCredentialPairStatus.DELETING;
// figure out if we need to artificially deflate the number of docs indexed.
// This is required since the total number of docs indexed by a CC Pair is
@@ -61,72 +104,143 @@ function Main({ ccPairId }: { ccPairId: number }) {
? lastIndexAttempt.total_docs_indexed
: ccPair.num_docs_indexed;
+ const refresh = () => {
+ mutate(buildCCPairInfoUrl(ccPairId));
+ };
+
+ const startEditing = () => {
+ setEditableName(ccPair.name);
+ setIsEditing(true);
+ };
+
+ const resetEditing = () => {
+ setIsEditing(false);
+ setEditableName(ccPair.name);
+ };
+
+ const {
+ prune_freq: pruneFreq,
+ refresh_freq: refreshFreq,
+ indexing_start: indexingStart,
+ } = ccPair.connector;
return (
<>
+ {popup}
-
{ccPair.name}
-
-
+ {ccPair.is_editable_for_current_user && isEditing ? (
+
+
+
+
+
+ ) : (
+
+ ccPair.is_editable_for_current_user && startEditing()
+ }
+ className={`group flex ${ccPair.is_editable_for_current_user ? "cursor-pointer" : ""} text-3xl text-emphasis gap-x-2 items-center font-bold`}
+ >
+ {ccPair.name}
+ {ccPair.is_editable_for_current_user && (
+
+ )}
+
+ )}
+
+ {ccPair.is_editable_for_current_user && (
+
+ {!CONNECTOR_TYPES_THAT_CANT_REINDEX.includes(
+ ccPair.connector.source
+ ) && (
+
+ )}
+ {!isDeleting && }
+
+ )}
+
Total Documents Indexed:{" "}
{totalDocsIndexed}
+ {!ccPair.is_editable_for_current_user && (
+
+ {ccPair.is_public
+ ? "Public connectors are not editable by curators."
+ : "This connector belongs to groups where you don't have curator permissions, so it's not editable."}
+
+ )}
+ {credentialTemplates[ccPair.connector.source] &&
+ ccPair.is_editable_for_current_user && (
+ <>
+
-
+ Credentials
+ refresh()}
+ />
+
+ )}
+
+
+ {(pruneFreq || indexingStart || refreshFreq) && (
+
+ )}
+
{/* NOTE: no divider / title here for `ConfigDisplay` since it is optional and we need
to render these conditionally.*/}
-
Indexing Attempts
-
- {!CONNECTOR_TYPES_THAT_CANT_REINDEX.includes(
- ccPair.connector.source
- ) && (
-
- )}
-
-
-
-
-
Delete Connector
-
- Deleting the connector will also delete all associated documents.
-
-
-
-
+
+
+ {ccPair.is_editable_for_current_user && (
-
+ )}
-
- {/* TODO: add document search*/}
);
}
@@ -136,10 +250,6 @@ export default function Page({ params }: { params: { ccPairId: string } }) {
return (
);
diff --git a/web/src/app/admin/connector/[ccPairId]/types.ts b/web/src/app/admin/connector/[ccPairId]/types.ts
index ab4921180cf..1cc43311e21 100644
--- a/web/src/app/admin/connector/[ccPairId]/types.ts
+++ b/web/src/app/admin/connector/[ccPairId]/types.ts
@@ -1,16 +1,22 @@
-import {
- Connector,
- Credential,
- DeletionAttemptSnapshot,
- IndexAttemptSnapshot,
-} from "@/lib/types";
+import { Connector } from "@/lib/connectors/connectors";
+import { Credential } from "@/lib/connectors/credentials";
+import { DeletionAttemptSnapshot, IndexAttemptSnapshot } from "@/lib/types";
+
+export enum ConnectorCredentialPairStatus {
+ ACTIVE = "ACTIVE",
+ PAUSED = "PAUSED",
+ DELETING = "DELETING",
+}
export interface CCPairFullInfo {
id: number;
name: string;
+ status: ConnectorCredentialPairStatus;
num_docs_indexed: number;
connector: Connector
;
credential: Credential;
index_attempts: IndexAttemptSnapshot[];
latest_deletion_attempt: DeletionAttemptSnapshot | null;
+ is_public: boolean;
+ is_editable_for_current_user: boolean;
}
diff --git a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
new file mode 100644
index 00000000000..dd8d19ca720
--- /dev/null
+++ b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
@@ -0,0 +1,617 @@
+"use client";
+
+import * as Yup from "yup";
+import { TrashIcon } from "@/components/icons/icons";
+import { errorHandlingFetcher } from "@/lib/fetcher";
+import useSWR, { mutate } from "swr";
+import { HealthCheckBanner } from "@/components/health/healthcheck";
+
+import { Card, Divider, Title } from "@tremor/react";
+import { AdminPageTitle } from "@/components/admin/Title";
+import { buildSimilarCredentialInfoURL } from "@/app/admin/connector/[ccPairId]/lib";
+import { usePopup } from "@/components/admin/connectors/Popup";
+import { useFormContext } from "@/components/context/FormContext";
+import { getSourceDisplayName } from "@/lib/sources";
+import { SourceIcon } from "@/components/SourceIcon";
+import { useRef, useState, useEffect } from "react";
+import { submitConnector } from "@/components/admin/connectors/ConnectorForm";
+import { deleteCredential, linkCredential } from "@/lib/credential";
+import { submitFiles } from "./pages/utils/files";
+import { submitGoogleSite } from "./pages/utils/google_site";
+import AdvancedFormPage from "./pages/Advanced";
+import DynamicConnectionForm from "./pages/DynamicConnectorCreationForm";
+import CreateCredential from "@/components/credentials/actions/CreateCredential";
+import ModifyCredential from "@/components/credentials/actions/ModifyCredential";
+import { ValidSources } from "@/lib/types";
+import { Credential, credentialTemplates } from "@/lib/connectors/credentials";
+import {
+ ConnectionConfiguration,
+ connectorConfigs,
+} from "@/lib/connectors/connectors";
+import { Modal } from "@/components/Modal";
+import { ArrowRight } from "@phosphor-icons/react";
+import { ArrowLeft } from "@phosphor-icons/react/dist/ssr";
+import { FiPlus } from "react-icons/fi";
+import GDriveMain from "./pages/gdrive/GoogleDrivePage";
+import { GmailMain } from "./pages/gmail/GmailPage";
+import {
+ useGmailCredentials,
+ useGoogleDriveCredentials,
+} from "./pages/utils/hooks";
+import { Formik, FormikProps } from "formik";
+import {
+ IsPublicGroupSelector,
+ IsPublicGroupSelectorFormType,
+} from "@/components/IsPublicGroupSelector";
+import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
+import { AdminBooleanFormField } from "@/components/credentials/CredentialFields";
+
+export type AdvancedConfigFinal = {
+ pruneFreq: number | null;
+ refreshFreq: number | null;
+ indexingStart: Date | null;
+};
+
+export default function AddConnector({
+ connector,
+}: {
+ connector: ValidSources;
+}) {
+ const [currentCredential, setCurrentCredential] =
+ useState | null>(null);
+
+ const { data: credentials } = useSWR[]>(
+ buildSimilarCredentialInfoURL(connector),
+ errorHandlingFetcher,
+ { refreshInterval: 5000 }
+ );
+
+ const { data: editableCredentials } = useSWR[]>(
+ buildSimilarCredentialInfoURL(connector, true),
+ errorHandlingFetcher,
+ { refreshInterval: 5000 }
+ );
+ const [selectedFiles, setSelectedFiles] = useState([]);
+
+ const credentialTemplate = credentialTemplates[connector];
+
+ const {
+ setFormStep,
+ setAllowAdvanced,
+ setAlowCreate,
+ formStep,
+ nextFormStep,
+ prevFormStep,
+ } = useFormContext();
+
+ const { popup, setPopup } = usePopup();
+
+ const configuration: ConnectionConfiguration = connectorConfigs[connector];
+ const [formValues, setFormValues] = useState<
+ Record & IsPublicGroupSelectorFormType
+ >({
+ name: "",
+ groups: [],
+ is_public: false,
+ ...configuration.values.reduce(
+ (acc, field) => {
+ if (field.type === "list") {
+ acc[field.name] = field.default || [];
+ } else if (field.type === "checkbox") {
+ acc[field.name] = field.default || false;
+ } else if (field.default !== undefined) {
+ acc[field.name] = field.default;
+ }
+ return acc;
+ },
+ {} as { [record: string]: any }
+ ),
+ });
+
+ const isPaidEnterpriseFeaturesEnabled = usePaidEnterpriseFeaturesEnabled();
+
+ // Default to 10 minutes unless otherwise specified
+ const defaultAdvancedSettings = {
+ refreshFreq: formValues.overrideDefaultFreq || 10,
+ pruneFreq: 30,
+ indexingStart: null as string | null,
+ };
+
+ const [advancedSettings, setAdvancedSettings] = useState(
+ defaultAdvancedSettings
+ );
+
+ const [createConnectorToggle, setCreateConnectorToggle] = useState(false);
+ const formRef = useRef>(null);
+
+ const [isFormValid, setIsFormValid] = useState(false);
+
+ const handleFormStatusChange = (isValid: boolean) => {
+ setIsFormValid(isValid || connector == "file");
+ };
+
+ const { liveGDriveCredential } = useGoogleDriveCredentials();
+
+ const { liveGmailCredential } = useGmailCredentials();
+
+ const credentialActivated =
+ (connector === "google_drive" && liveGDriveCredential) ||
+ (connector === "gmail" && liveGmailCredential) ||
+ currentCredential;
+
+ const noCredentials = credentialTemplate == null;
+
+ if (noCredentials && 1 != formStep) {
+ setFormStep(Math.max(1, formStep));
+ }
+
+ if (!noCredentials && !credentialActivated && formStep != 0) {
+ setFormStep(Math.min(formStep, 0));
+ }
+
+ const resetAdvancedConfigs = (formikProps: FormikProps) => {
+ formikProps.resetForm({ values: defaultAdvancedSettings });
+ setAdvancedSettings(defaultAdvancedSettings);
+ };
+
+ const convertStringToDateTime = (indexingStart: string | null) => {
+ return indexingStart ? new Date(indexingStart) : null;
+ };
+
+ const createConnector = async () => {
+ const {
+ name,
+ groups,
+ is_public: isPublic,
+ ...connector_specific_config
+ } = formValues;
+ const { pruneFreq, indexingStart, refreshFreq } = advancedSettings;
+
+ // Apply transforms from connectors.ts configuration
+ const transformedConnectorSpecificConfig = Object.entries(
+ connector_specific_config
+ ).reduce(
+ (acc, [key, value]) => {
+ const matchingConfigValue = configuration.values.find(
+ (configValue) => configValue.name === key
+ );
+ if (
+ matchingConfigValue &&
+ "transform" in matchingConfigValue &&
+ matchingConfigValue.transform
+ ) {
+ acc[key] = matchingConfigValue.transform(value as string[]);
+ } else {
+ acc[key] = value;
+ }
+ return acc;
+ },
+ {} as Record
+ );
+
+ const AdvancedConfig: AdvancedConfigFinal = {
+ pruneFreq: advancedSettings.pruneFreq * 60 * 60 * 24,
+ indexingStart: convertStringToDateTime(indexingStart),
+ refreshFreq: advancedSettings.refreshFreq * 60,
+ };
+
+ // google sites-specific handling
+ if (connector == "google_site") {
+ const response = await submitGoogleSite(
+ selectedFiles,
+ formValues?.base_url,
+ setPopup,
+ AdvancedConfig,
+ name
+ );
+ if (response) {
+ setTimeout(() => {
+ window.open("/admin/indexing/status", "_self");
+ }, 1000);
+ }
+ return;
+ }
+
+ // file-specific handling
+ if (connector == "file" && selectedFiles.length > 0) {
+ const response = await submitFiles(
+ selectedFiles,
+ setPopup,
+ setSelectedFiles,
+ name,
+ AdvancedConfig,
+ isPublic,
+ groups
+ );
+ if (response) {
+ setTimeout(() => {
+ window.open("/admin/indexing/status", "_self");
+ }, 1000);
+ }
+ return;
+ }
+
+ const { message, isSuccess, response } = await submitConnector(
+ {
+ connector_specific_config: transformedConnectorSpecificConfig,
+ input_type: connector == "web" ? "load_state" : "poll", // single case
+ name: name,
+ source: connector,
+ refresh_freq: refreshFreq * 60 || null,
+ prune_freq: pruneFreq * 60 * 60 * 24 || null,
+ indexing_start: convertStringToDateTime(indexingStart),
+ is_public: isPublic,
+ groups: groups,
+ },
+ undefined,
+ credentialActivated ? false : true,
+ isPublic
+ );
+ // If no credential
+ if (!credentialActivated) {
+ if (isSuccess) {
+ setPopup({
+ message: "Connector created! Redirecting to connector home page",
+ type: "success",
+ });
+ setTimeout(() => {
+ window.open("/admin/indexing/status", "_self");
+ }, 1000);
+ } else {
+ setPopup({ message: message, type: "error" });
+ }
+ }
+
+ // Without credential
+ if (credentialActivated && isSuccess && response) {
+ const credential =
+ currentCredential || liveGDriveCredential || liveGmailCredential;
+ const linkCredentialResponse = await linkCredential(
+ response.id,
+ credential?.id!,
+ name,
+ isPublic,
+ groups
+ );
+ if (linkCredentialResponse.ok) {
+ setPopup({
+ message: "Connector created! Redirecting to connector home page",
+ type: "success",
+ });
+ setTimeout(() => {
+ window.open("/admin/indexing/status", "_self");
+ }, 1000);
+ } else {
+ const errorData = await linkCredentialResponse.json();
+ setPopup({
+ message: errorData.message,
+ type: "error",
+ });
+ }
+ } else if (isSuccess) {
+ setPopup({
+ message:
+ "Credential created succsfully! Redirecting to connector home page",
+ type: "success",
+ });
+ } else {
+ setPopup({ message: message, type: "error" });
+ }
+ };
+
+ const displayName = getSourceDisplayName(connector) || connector;
+ if (!credentials || !editableCredentials) {
+ return <>;
+ }
+
+ const refresh = () => {
+ mutate(buildSimilarCredentialInfoURL(connector));
+ };
+ const onDeleteCredential = async (credential: Credential) => {
+ const response = await deleteCredential(credential.id, true);
+ if (response.ok) {
+ setPopup({
+ message: "Credential deleted successfully!",
+ type: "success",
+ });
+ } else {
+ const errorData = await response.json();
+ setPopup({
+ message: errorData.message,
+ type: "error",
+ });
+ }
+ };
+
+ const onSwap = async (selectedCredential: Credential) => {
+ setCurrentCredential(selectedCredential);
+ setAlowCreate(true);
+ setPopup({
+ message: "Swapped credential successfully!",
+ type: "success",
+ });
+ refresh();
+ };
+
+ const validationSchema = Yup.object().shape({
+ name: Yup.string().required("Connector Name is required"),
+ ...configuration.values.reduce(
+ (acc, field) => {
+ let schema: any =
+ field.type === "list"
+ ? Yup.array().of(Yup.string())
+ : field.type === "checkbox"
+ ? Yup.boolean()
+ : Yup.string();
+
+ if (!field.optional) {
+ schema = schema.required(`${field.label} is required`);
+ }
+ acc[field.name] = schema;
+ return acc;
+ },
+ {} as Record
+ ),
+ });
+
+ const advancedValidationSchema = Yup.object().shape({
+ indexingStart: Yup.string().nullable(),
+ pruneFreq: Yup.number().min(0, "Prune frequency must be non-negative"),
+ refreshFreq: Yup.number().min(0, "Refresh frequency must be non-negative"),
+ });
+
+ const isFormSubmittable = (values: any) => {
+ return (
+ values.name.trim() !== "" &&
+ Object.keys(values).every((key) => {
+ const field = configuration.values.find((f) => f.name === key);
+ return field?.optional || values[key] !== "";
+ })
+ );
+ };
+
+ return (
+
+ {popup}
+
+
+
+
+
}
+ title={displayName}
+ />
+
+ {formStep == 0 &&
+ (connector == "google_drive" ? (
+ <>
+
+ Select a credential
+
+
+
+
+
+
+ ) : connector == "gmail" ? (
+ <>
+
+ Select a credential
+
+
+
+
+
+
+ ) : (
+ <>
+
+ Select a credential
+
+ {!createConnectorToggle && (
+
+ )}
+
+ {!(connector == "google_drive") && createConnectorToggle && (
+ setCreateConnectorToggle(false)}
+ >
+ <>
+
+ Create a {getSourceDisplayName(connector)} credential
+
+ setCreateConnectorToggle(false)}
+ />
+
+
+ )}
+
+
+
+
+
+ ))}
+
+ {formStep == 1 && (
+ <>
+
+ {
+ // Can be utilized for logging purposes
+ }}
+ >
+ {(formikProps) => {
+ setFormValues(formikProps.values);
+ handleFormStatusChange(
+ formikProps.isValid && isFormSubmittable(formikProps.values)
+ );
+ setAllowAdvanced(
+ formikProps.isValid && isFormSubmittable(formikProps.values)
+ );
+
+ return (
+
+
+ {isPaidEnterpriseFeaturesEnabled && (
+ <>
+
+
+ )}
+
+ );
+ }}
+
+
+
+ {!noCredentials ? (
+
+ ) : (
+
+ )}
+
+
+ {!(connector == "file") && (
+
+
+
+ )}
+
+
+ )}
+
+ {formStep === 2 && (
+ <>
+
+ {}}
+ >
+ {(formikProps) => {
+ setAdvancedSettings(formikProps.values);
+
+ return (
+ <>
+
+
+
+
+
+
+
+
+
+ )}
+
+
+
+ {!isValidSource(connector) ? (
+
+
+ ‘{connector}‘ is not a valid Connector Type!
+
+
+
+ ) : (
+
+ )}
+
+
+
+
+
+
+
+
+
+
+ {enterpriseSettings && enterpriseSettings.application_name ? (
+ {enterpriseSettings.application_name}
+ ) : (
+
+ EveAI
+
+ )}
+
+
+
+
+
+
+
+
+ {connector != "file" && (
+
+ )}
+ {settingSteps.map((step, index) => {
+ const allowed =
+ (step == "Connector" && allowCreate) ||
+ (step == "Advanced (optional)" && allowAdvanced) ||
+ index <= formStep;
+
+ return (
+
{
+ if (allowed) {
+ setFormStep(index - (noCredential ? 1 : 0));
+ }
+ }}
+ >
+
+
+ {formStep === index && (
+
+ )}
+
+
+
+ {step}
+
+
+ );
+ })}
+
+
+
+
+
+
+
+ Advanced Configuration
+
+
+
+
+
+ {field.description && (
+
{field.description}
+ )}
+
+ {fieldProps.value.map((value: string, index: number) => (
+
+
+
+
+ ))}
+
+
+
+
+ {description && {description}}
+
+
+
+ );
+}
diff --git a/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/SelectInput.tsx b/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/SelectInput.tsx
new file mode 100644
index 00000000000..e01a02dc323
--- /dev/null
+++ b/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/SelectInput.tsx
@@ -0,0 +1,45 @@
+import CredentialSubText from "@/components/credentials/CredentialFields";
+import { ListOption, SelectOption } from "@/lib/connectors/connectors";
+import { Field } from "formik";
+
+export default function SelectInput({
+ field,
+ value,
+ onChange,
+}: {
+ field: SelectOption;
+ value: any;
+ onChange?: (e: Event) => void;
+}) {
+ return (
+ <>
+
+ {field.description && (
+ {field.description}
+ )}
+
+
+
+ {field.options?.map((option: any) => (
+
+ ))}
+
+
+ );
+}
diff --git a/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx b/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
new file mode 100644
index 00000000000..507b976f9a8
--- /dev/null
+++ b/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
@@ -0,0 +1,115 @@
+import React, {
+ ChangeEvent,
+ Dispatch,
+ FC,
+ SetStateAction,
+ useEffect,
+ useState,
+} from "react";
+import { Formik, Form, Field, FieldArray, FormikProps } from "formik";
+import * as Yup from "yup";
+import { FaPlus } from "react-icons/fa";
+import { useUserGroups } from "@/lib/hooks";
+import { UserGroup, User, UserRole } from "@/lib/types";
+import { Divider } from "@tremor/react";
+import CredentialSubText, {
+ AdminBooleanFormField,
+} from "@/components/credentials/CredentialFields";
+import { TrashIcon } from "@/components/icons/icons";
+import { FileUpload } from "@/components/admin/connectors/FileUpload";
+import { ConnectionConfiguration } from "@/lib/connectors/connectors";
+import { useFormContext } from "@/components/context/FormContext";
+import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
+import { Text } from "@tremor/react";
+import { getCurrentUser } from "@/lib/user";
+import { FiUsers } from "react-icons/fi";
+import SelectInput from "./ConnectorInput/SelectInput";
+import NumberInput from "./ConnectorInput/NumberInput";
+import { TextFormField } from "@/components/admin/connectors/Field";
+import ListInput from "./ConnectorInput/ListInput";
+import FileInput from "./ConnectorInput/FileInput";
+
+export interface DynamicConnectionFormProps {
+ config: ConnectionConfiguration;
+ selectedFiles: File[];
+ setSelectedFiles: Dispatch>;
+ values: any;
+}
+
+const DynamicConnectionForm: FC = ({
+ config,
+ selectedFiles,
+ setSelectedFiles,
+ values,
+}) => {
+ return (
+ <>
+ {config.description}
+
+ {config.subtext && (
+ {config.subtext}
+ )}
+
+
+
+ {config.values.map((field) => {
+ if (!field.hidden) {
+ return (
+
+ {field.type == "file" ? (
+
+ ) : field.type == "zip" ? (
+
+ ) : field.type === "list" ? (
+
+ ) : field.type === "select" ? (
+
+ ) : field.type === "number" ? (
+
+ ) : field.type === "checkbox" ? (
+
+ ) : (
+
+ )}
+
+
+ {description && {description}}
+
+
+
+ );
+}
diff --git a/web/src/app/admin/connectors/google-drive/Credential.tsx b/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx
similarity index 86%
rename from web/src/app/admin/connectors/google-drive/Credential.tsx
rename to web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx
index 849057e71e3..a320d466b40 100644
--- a/web/src/app/admin/connectors/google-drive/Credential.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx
@@ -4,11 +4,6 @@ import { useState } from "react";
import { useSWRConfig } from "swr";
import * as Yup from "yup";
import { useRouter } from "next/navigation";
-import {
- Credential,
- GoogleDriveCredentialJson,
- GoogleDriveServiceAccountCredentialJson,
-} from "@/lib/types";
import { adminDeleteCredential } from "@/lib/credential";
import { setupGoogleDriveOAuth } from "@/lib/googleDrive";
import { GOOGLE_DRIVE_AUTH_IS_ADMIN_COOKIE_NAME } from "@/lib/constants";
@@ -16,10 +11,15 @@ import Cookies from "js-cookie";
import { TextFormField } from "@/components/admin/connectors/Field";
import { Form, Formik } from "formik";
import { Card } from "@tremor/react";
+import {
+ Credential,
+ GoogleDriveCredentialJson,
+ GoogleDriveServiceAccountCredentialJson,
+} from "@/lib/connectors/credentials";
type GoogleDriveCredentialJsonTypes = "authorized_user" | "service_account";
-const DriveJsonUpload = ({
+export const DriveJsonUpload = ({
setPopup,
}: {
setPopup: (popupSpec: PopupSpec | null) => void;
@@ -33,8 +33,8 @@ const DriveJsonUpload = ({
<>
void;
appCredentialData?: { client_id: string };
serviceAccountCredentialData?: { service_account_email: string };
+ isAdmin: boolean;
}
export const DriveJsonUploadSection = ({
setPopup,
appCredentialData,
serviceAccountCredentialData,
+ isAdmin,
}: DriveJsonUploadSectionProps) => {
const { mutate } = useSWRConfig();
@@ -165,38 +167,48 @@ export const DriveJsonUploadSection = ({
{serviceAccountCredentialData.service_account_email}
-
- If you want to update these credentials, delete the existing
- credentials through the button below, and then upload a new
- credentials JSON.
-
-
+ {isAdmin ? (
+ <>
+
+ If you want to update these credentials, delete the existing
+ credentials through the button below, and then upload a new
+ credentials JSON.
+
+
+
+ ) : (
+ <>
+
+ To change these credentials, please contact an administrator.
+
+
+ )}
+
+ Curators are unable to set up the google drive credentials. To add a
+ Google Drive connector, please contact an administrator.
+
+
diff --git a/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx b/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx
new file mode 100644
index 00000000000..4494e4b22ee
--- /dev/null
+++ b/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx
@@ -0,0 +1,155 @@
+"use client";
+
+import React from "react";
+import { useState, useEffect } from "react";
+import useSWR from "swr";
+import { FetchError, errorHandlingFetcher } from "@/lib/fetcher";
+import { ErrorCallout } from "@/components/ErrorCallout";
+import { LoadingAnimation } from "@/components/Loading";
+import { usePopup } from "@/components/admin/connectors/Popup";
+import { ConnectorIndexingStatus } from "@/lib/types";
+import { getCurrentUser } from "@/lib/user";
+import { User, UserRole } from "@/lib/types";
+import { usePublicCredentials } from "@/lib/hooks";
+import { Title } from "@tremor/react";
+import { DriveJsonUploadSection, DriveOAuthSection } from "./Credential";
+import {
+ Credential,
+ GoogleDriveCredentialJson,
+ GoogleDriveServiceAccountCredentialJson,
+} from "@/lib/connectors/credentials";
+import { GoogleDriveConfig } from "@/lib/connectors/connectors";
+import { useUser } from "@/components/user/UserProvider";
+import { useConnectorCredentialIndexingStatus } from "@/lib/hooks";
+
+const GDriveMain = ({}: {}) => {
+ const { isLoadingUser, isAdmin } = useUser();
+
+ const {
+ data: appCredentialData,
+ isLoading: isAppCredentialLoading,
+ error: isAppCredentialError,
+ } = useSWR<{ client_id: string }, FetchError>(
+ "/api/manage/admin/connector/google-drive/app-credential",
+ errorHandlingFetcher
+ );
+
+ const {
+ data: serviceAccountKeyData,
+ isLoading: isServiceAccountKeyLoading,
+ error: isServiceAccountKeyError,
+ } = useSWR<{ service_account_email: string }, FetchError>(
+ "/api/manage/admin/connector/google-drive/service-account-key",
+ errorHandlingFetcher
+ );
+
+ const {
+ data: connectorIndexingStatuses,
+ isLoading: isConnectorIndexingStatusesLoading,
+ error: connectorIndexingStatusesError,
+ } = useConnectorCredentialIndexingStatus();
+ const {
+ data: credentialsData,
+ isLoading: isCredentialsLoading,
+ error: credentialsError,
+ refreshCredentials,
+ } = usePublicCredentials();
+
+ const { popup, setPopup } = usePopup();
+
+ const appCredentialSuccessfullyFetched =
+ appCredentialData ||
+ (isAppCredentialError && isAppCredentialError.status === 404);
+ const serviceAccountKeySuccessfullyFetched =
+ serviceAccountKeyData ||
+ (isServiceAccountKeyError && isServiceAccountKeyError.status === 404);
+
+ if (isLoadingUser) {
+ return <>;
+ }
+
+ if (
+ (!appCredentialSuccessfullyFetched && isAppCredentialLoading) ||
+ (!serviceAccountKeySuccessfullyFetched && isServiceAccountKeyLoading) ||
+ (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) ||
+ (!credentialsData && isCredentialsLoading)
+ ) {
+ return (
+
+
+
+ );
+ }
+
+ if (credentialsError || !credentialsData) {
+ return
;
+ }
+
+ if (connectorIndexingStatusesError || !connectorIndexingStatuses) {
+ return
;
+ }
+
+ if (
+ !appCredentialSuccessfullyFetched ||
+ !serviceAccountKeySuccessfullyFetched
+ ) {
+ return (
+
+ );
+ }
+
+ const googleDrivePublicCredential:
+ | Credential
+ | undefined = credentialsData.find(
+ (credential) =>
+ credential.credential_json?.google_drive_tokens && credential.admin_public
+ );
+ const googleDriveServiceAccountCredential:
+ | Credential
+ | undefined = credentialsData.find(
+ (credential) => credential.credential_json?.google_drive_service_account_key
+ );
+ const googleDriveConnectorIndexingStatuses: ConnectorIndexingStatus<
+ GoogleDriveConfig,
+ GoogleDriveCredentialJson
+ >[] = connectorIndexingStatuses.filter(
+ (connectorIndexingStatus) =>
+ connectorIndexingStatus.connector.source === "google_drive"
+ );
+
+ return (
+ <>
+ {popup}
+
+ Step 1: Provide your Credentials
+
+
+
+ {isAdmin && (
+ <>
+
+ Step 2: Authenticate with Danswer
+
+ 0}
+ />
+
+ )}
+
+ );
+};
+
+export default GDriveMain;
diff --git a/web/src/app/admin/connectors/gmail/Credential.tsx b/web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx
similarity index 86%
rename from web/src/app/admin/connectors/gmail/Credential.tsx
rename to web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx
index 68f5bba2d20..8b456884f1a 100644
--- a/web/src/app/admin/connectors/gmail/Credential.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx
@@ -4,11 +4,6 @@ import { useState } from "react";
import { useSWRConfig } from "swr";
import * as Yup from "yup";
import { useRouter } from "next/navigation";
-import {
- Credential,
- GmailCredentialJson,
- GmailServiceAccountCredentialJson,
-} from "@/lib/types";
import { adminDeleteCredential } from "@/lib/credential";
import { setupGmailOAuth } from "@/lib/gmail";
import { GMAIL_AUTH_IS_ADMIN_COOKIE_NAME } from "@/lib/constants";
@@ -16,6 +11,11 @@ import Cookies from "js-cookie";
import { TextFormField } from "@/components/admin/connectors/Field";
import { Form, Formik } from "formik";
import { Card } from "@tremor/react";
+import {
+ Credential,
+ GmailCredentialJson,
+ GmailServiceAccountCredentialJson,
+} from "@/lib/connectors/credentials";
type GmailCredentialJsonTypes = "authorized_user" | "service_account";
@@ -33,8 +33,8 @@ const DriveJsonUpload = ({
<>
void;
appCredentialData?: { client_id: string };
serviceAccountCredentialData?: { service_account_email: string };
+ isAdmin: boolean;
}
export const GmailJsonUploadSection = ({
setPopup,
appCredentialData,
serviceAccountCredentialData,
+ isAdmin,
}: DriveJsonUploadSectionProps) => {
const { mutate } = useSWRConfig();
@@ -163,36 +165,48 @@ export const GmailJsonUploadSection = ({
{serviceAccountCredentialData.service_account_email}
-
- If you want to update these credentials, delete the existing
- credentials through the button below, and then upload a new
- credentials JSON.
-
-
+ {isAdmin ? (
+ <>
+
+ If you want to update these credentials, delete the existing
+ credentials through the button below, and then upload a new
+ credentials JSON.
+
+
+
+ ) : (
+ <>
+
+ To change these credentials, please contact an administrator.
+
+
+ )}
);
}
@@ -238,6 +252,17 @@ export const GmailJsonUploadSection = ({
);
}
+ if (!isAdmin) {
+ return (
+
+
+ Curators are unable to set up the Gmail credentials. To add a Gmail
+ connector, please contact an administrator.
+
+
+ );
+ }
+
return (
diff --git a/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx b/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx
new file mode 100644
index 00000000000..5f52eb31013
--- /dev/null
+++ b/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx
@@ -0,0 +1,159 @@
+"use client";
+
+import useSWR from "swr";
+import { errorHandlingFetcher } from "@/lib/fetcher";
+import { LoadingAnimation } from "@/components/Loading";
+import { usePopup } from "@/components/admin/connectors/Popup";
+import { ConnectorIndexingStatus } from "@/lib/types";
+import { getCurrentUser } from "@/lib/user";
+import { User, UserRole } from "@/lib/types";
+import {
+ Credential,
+ GmailCredentialJson,
+ GmailServiceAccountCredentialJson,
+} from "@/lib/connectors/credentials";
+import { GmailOAuthSection, GmailJsonUploadSection } from "./Credential";
+import { usePublicCredentials } from "@/lib/hooks";
+import { Title } from "@tremor/react";
+import { GmailConfig } from "@/lib/connectors/connectors";
+import { useState, useEffect } from "react";
+import { useUser } from "@/components/user/UserProvider";
+import { useConnectorCredentialIndexingStatus } from "@/lib/hooks";
+
+export const GmailMain = () => {
+ const { isLoadingUser, isAdmin } = useUser();
+
+ const {
+ data: appCredentialData,
+ isLoading: isAppCredentialLoading,
+ error: isAppCredentialError,
+ } = useSWR<{ client_id: string }>(
+ "/api/manage/admin/connector/gmail/app-credential",
+ errorHandlingFetcher
+ );
+ const {
+ data: serviceAccountKeyData,
+ isLoading: isServiceAccountKeyLoading,
+ error: isServiceAccountKeyError,
+ } = useSWR<{ service_account_email: string }>(
+ "/api/manage/admin/connector/gmail/service-account-key",
+ errorHandlingFetcher
+ );
+ const {
+ data: connectorIndexingStatuses,
+ isLoading: isConnectorIndexingStatusesLoading,
+ error: connectorIndexingStatusesError,
+ } = useConnectorCredentialIndexingStatus();
+
+ const {
+ data: credentialsData,
+ isLoading: isCredentialsLoading,
+ error: credentialsError,
+ refreshCredentials,
+ } = usePublicCredentials();
+
+ const { popup, setPopup } = usePopup();
+
+ const appCredentialSuccessfullyFetched =
+ appCredentialData ||
+ (isAppCredentialError && isAppCredentialError.status === 404);
+ const serviceAccountKeySuccessfullyFetched =
+ serviceAccountKeyData ||
+ (isServiceAccountKeyError && isServiceAccountKeyError.status === 404);
+
+ if (isLoadingUser) {
+ return <>;
+ }
+
+ if (
+ (!appCredentialSuccessfullyFetched && isAppCredentialLoading) ||
+ (!serviceAccountKeySuccessfullyFetched && isServiceAccountKeyLoading) ||
+ (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) ||
+ (!credentialsData && isCredentialsLoading)
+ ) {
+ return (
+
+
+
+ );
+ }
+
+ if (credentialsError || !credentialsData) {
+ return (
+
+
Failed to load credentials.
+
+ );
+ }
+
+ if (connectorIndexingStatusesError || !connectorIndexingStatuses) {
+ return (
+
+
Failed to load connectors.
+
+ );
+ }
+
+ if (
+ !appCredentialSuccessfullyFetched ||
+ !serviceAccountKeySuccessfullyFetched
+ ) {
+ return (
+
+
+ Error loading Gmail app credentials. Contact an administrator.
+
+
+ );
+ }
+
+ const gmailPublicCredential: Credential
| undefined =
+ credentialsData.find(
+ (credential) =>
+ credential.credential_json?.gmail_tokens && credential.admin_public
+ );
+ const gmailServiceAccountCredential:
+ | Credential
+ | undefined = credentialsData.find(
+ (credential) => credential.credential_json?.gmail_service_account_key
+ );
+ const gmailConnectorIndexingStatuses: ConnectorIndexingStatus<
+ GmailConfig,
+ GmailCredentialJson
+ >[] = connectorIndexingStatuses.filter(
+ (connectorIndexingStatus) =>
+ connectorIndexingStatus.connector.source === "gmail"
+ );
+
+ return (
+ <>
+ {popup}
+
+ Step 1: Provide your Credentials
+
+
+
+ {isAdmin && (
+ <>
+
+ Step 2: Authenticate with Danswer
+
+ 0}
+ />
+
+ )}
+
+ );
+};
diff --git a/web/src/app/admin/connectors/[connector]/pages/utils/files.ts b/web/src/app/admin/connectors/[connector]/pages/utils/files.ts
new file mode 100644
index 00000000000..d847efe89d1
--- /dev/null
+++ b/web/src/app/admin/connectors/[connector]/pages/utils/files.ts
@@ -0,0 +1,112 @@
+import { PopupSpec } from "@/components/admin/connectors/Popup";
+import { createConnector, runConnector } from "@/lib/connector";
+import { createCredential, linkCredential } from "@/lib/credential";
+import { FileConfig } from "@/lib/connectors/connectors";
+import { AdvancedConfigFinal } from "../../AddConnectorPage";
+
+export const submitFiles = async (
+ selectedFiles: File[],
+ setPopup: (popup: PopupSpec) => void,
+ setSelectedFiles: (files: File[]) => void,
+ name: string,
+ advancedConfig: AdvancedConfigFinal,
+ isPublic: boolean,
+ groups?: number[]
+) => {
+ const formData = new FormData();
+
+ selectedFiles.forEach((file) => {
+ formData.append("files", file);
+ });
+
+ const response = await fetch("/api/manage/admin/connector/file/upload", {
+ method: "POST",
+ body: formData,
+ });
+ const responseJson = await response.json();
+ if (!response.ok) {
+ setPopup({
+ message: `Unable to upload files - ${responseJson.detail}`,
+ type: "error",
+ });
+ return;
+ }
+
+ const filePaths = responseJson.file_paths as string[];
+
+ const [connectorErrorMsg, connector] = await createConnector({
+ name: "FileConnector-" + Date.now(),
+ source: "file",
+ input_type: "load_state",
+ connector_specific_config: {
+ file_locations: filePaths,
+ },
+ refresh_freq: null,
+ prune_freq: null,
+ indexing_start: null,
+ is_public: isPublic,
+ groups: groups,
+ });
+ if (connectorErrorMsg || !connector) {
+ setPopup({
+ message: `Unable to create connector - ${connectorErrorMsg}`,
+ type: "error",
+ });
+ return;
+ }
+
+ // Since there is no "real" credential associated with a file connector
+ // we create a dummy one here so that we can associate the CC Pair with a
+ // user. This is needed since the user for a CC Pair is found via the credential
+ // associated with it.
+ const createCredentialResponse = await createCredential({
+ credential_json: {},
+ admin_public: true,
+ source: "file",
+ curator_public: isPublic,
+ groups: groups,
+ name,
+ });
+ if (!createCredentialResponse.ok) {
+ const errorMsg = await createCredentialResponse.text();
+ setPopup({
+ message: `Error creating credential for CC Pair - ${errorMsg}`,
+ type: "error",
+ });
+ return;
+ false;
+ }
+ const credentialId = (await createCredentialResponse.json()).id;
+
+ const credentialResponse = await linkCredential(
+ connector.id,
+ credentialId,
+ name,
+ isPublic,
+ groups
+ );
+ if (!credentialResponse.ok) {
+ const credentialResponseJson = await credentialResponse.json();
+ setPopup({
+ message: `Unable to link connector to credential - ${credentialResponseJson.detail}`,
+ type: "error",
+ });
+ return false;
+ }
+
+ const runConnectorErrorMsg = await runConnector(connector.id, [0]);
+ if (runConnectorErrorMsg) {
+ setPopup({
+ message: `Unable to run connector - ${runConnectorErrorMsg}`,
+ type: "error",
+ });
+ return false;
+ }
+
+ setSelectedFiles([]);
+ setPopup({
+ type: "success",
+ message: "Successfully uploaded files!",
+ });
+ return true;
+};
diff --git a/web/src/app/admin/connectors/[connector]/pages/utils/google_site.ts b/web/src/app/admin/connectors/[connector]/pages/utils/google_site.ts
new file mode 100644
index 00000000000..f1689e8fcdf
--- /dev/null
+++ b/web/src/app/admin/connectors/[connector]/pages/utils/google_site.ts
@@ -0,0 +1,87 @@
+import { PopupSpec } from "@/components/admin/connectors/Popup";
+import { createConnector, runConnector } from "@/lib/connector";
+import { linkCredential } from "@/lib/credential";
+import { GoogleSitesConfig } from "@/lib/connectors/connectors";
+import { AdvancedConfigFinal } from "../../AddConnectorPage";
+
+export const submitGoogleSite = async (
+ selectedFiles: File[],
+ base_url: any,
+ setPopup: (popup: PopupSpec) => void,
+ advancedConfig: AdvancedConfigFinal,
+ name?: string
+) => {
+ const uploadCreateAndTriggerConnector = async () => {
+ const formData = new FormData();
+
+ selectedFiles.forEach((file) => {
+ formData.append("files", file);
+ });
+
+ const response = await fetch("/api/manage/admin/connector/file/upload", {
+ method: "POST",
+ body: formData,
+ });
+ const responseJson = await response.json();
+ if (!response.ok) {
+ setPopup({
+ message: `Unable to upload files - ${responseJson.detail}`,
+ type: "error",
+ });
+ return false;
+ }
+
+ const filePaths = responseJson.file_paths as string[];
+ const [connectorErrorMsg, connector] =
+ await createConnector({
+ name: name ? name : `GoogleSitesConnector-${base_url}`,
+ source: "google_sites",
+ input_type: "load_state",
+ connector_specific_config: {
+ base_url: base_url,
+ zip_path: filePaths[0],
+ },
+ refresh_freq: advancedConfig.refreshFreq,
+ prune_freq: advancedConfig.pruneFreq,
+ indexing_start: advancedConfig.indexingStart,
+ });
+ if (connectorErrorMsg || !connector) {
+ setPopup({
+ message: `Unable to create connector - ${connectorErrorMsg}`,
+ type: "error",
+ });
+ return false;
+ }
+
+ const credentialResponse = await linkCredential(connector.id, 0, base_url);
+ if (!credentialResponse.ok) {
+ const credentialResponseJson = await credentialResponse.json();
+ setPopup({
+ message: `Unable to link connector to credential - ${credentialResponseJson.detail}`,
+ type: "error",
+ });
+ return false;
+ }
+
+ const runConnectorErrorMsg = await runConnector(connector.id, [0]);
+ if (runConnectorErrorMsg) {
+ setPopup({
+ message: `Unable to run connector - ${runConnectorErrorMsg}`,
+ type: "error",
+ });
+ return false;
+ }
+ setPopup({
+ type: "success",
+ message: "Successfully created Google Site connector!",
+ });
+ return true;
+ };
+
+ try {
+ const response = await uploadCreateAndTriggerConnector();
+ return response;
+ } catch (e) {
+ return false;
+ }
+};
diff --git a/web/src/app/admin/connectors/[connector]/pages/utils/hooks.ts b/web/src/app/admin/connectors/[connector]/pages/utils/hooks.ts
new file mode 100644
index 00000000000..d3a48e3a26b
--- /dev/null
+++ b/web/src/app/admin/connectors/[connector]/pages/utils/hooks.ts
@@ -0,0 +1,65 @@
+import { GmailConfig } from "@/lib/connectors/connectors";
+
+export const gmailConnectorNameBuilder = (values: GmailConfig) =>
+ "GmailConnector";
+
+import { usePublicCredentials } from "@/lib/hooks";
+import {
+ Credential,
+ GmailCredentialJson,
+ GmailServiceAccountCredentialJson,
+ GoogleDriveCredentialJson,
+ GoogleDriveServiceAccountCredentialJson,
+} from "@/lib/connectors/credentials";
+
+export const useGmailCredentials = () => {
+ const {
+ data: credentialsData,
+ isLoading: isCredentialsLoading,
+ error: credentialsError,
+ refreshCredentials,
+ } = usePublicCredentials();
+
+ const gmailPublicCredential: Credential | undefined =
+ credentialsData?.find(
+ (credential) =>
+ credential.credential_json?.gmail_tokens && credential.admin_public
+ );
+
+ const gmailServiceAccountCredential:
+ | Credential
+ | undefined = credentialsData?.find(
+ (credential) => credential.credential_json?.gmail_service_account_key
+ );
+
+ const liveGmailCredential =
+ gmailPublicCredential || gmailServiceAccountCredential;
+
+ return {
+ liveGmailCredential,
+ };
+};
+
+export const useGoogleDriveCredentials = () => {
+ const { data: credentialsData } = usePublicCredentials();
+
+ const googleDrivePublicCredential:
+ | Credential
+ | undefined = credentialsData?.find(
+ (credential) =>
+ credential.credential_json?.google_drive_tokens && credential.admin_public
+ );
+
+ const googleDriveServiceAccountCredential:
+ | Credential
+ | undefined = credentialsData?.find(
+ (credential) => credential.credential_json?.google_drive_service_account_key
+ );
+
+ const liveGDriveCredential =
+ googleDrivePublicCredential || googleDriveServiceAccountCredential;
+
+ return {
+ liveGDriveCredential,
+ };
+};
diff --git a/web/src/app/admin/connectors/axero/page.tsx b/web/src/app/admin/connectors/axero/page.tsx
deleted file mode 100644
index 6d4a5af8bcd..00000000000
--- a/web/src/app/admin/connectors/axero/page.tsx
+++ /dev/null
@@ -1,260 +0,0 @@
-"use client";
-
-import * as Yup from "yup";
-import { AxeroIcon, TrashIcon } from "@/components/icons/icons";
-import { errorHandlingFetcher } from "@/lib/fetcher";
-import { ErrorCallout } from "@/components/ErrorCallout";
-import useSWR, { useSWRConfig } from "swr";
-import { LoadingAnimation } from "@/components/Loading";
-import { HealthCheckBanner } from "@/components/health/healthcheck";
-import {
- AxeroConfig,
- AxeroCredentialJson,
- ConnectorIndexingStatus,
- Credential,
-} from "@/lib/types";
-import { adminDeleteCredential, linkCredential } from "@/lib/credential";
-import { CredentialForm } from "@/components/admin/connectors/CredentialForm";
-import {
- TextFormField,
- TextArrayFieldBuilder,
-} from "@/components/admin/connectors/Field";
-import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable";
-import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
-import { usePublicCredentials } from "@/lib/hooks";
-import { Button, Card, Divider, Text, Title } from "@tremor/react";
-import { AdminPageTitle } from "@/components/admin/Title";
-
-const MainSection = () => {
- const { mutate } = useSWRConfig();
- const {
- data: connectorIndexingStatuses,
- isLoading: isConnectorIndexingStatusesLoading,
- error: connectorIndexingStatusesError,
- } = useSWR[]>(
- "/api/manage/admin/connector/indexing-status",
- errorHandlingFetcher
- );
-
- const {
- data: credentialsData,
- isLoading: isCredentialsLoading,
- error: credentialsError,
- refreshCredentials,
- } = usePublicCredentials();
-
- if (
- (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) ||
- (!credentialsData && isCredentialsLoading)
- ) {
- return ;
- }
-
- if (connectorIndexingStatusesError || !connectorIndexingStatuses) {
- return (
-
- );
- }
-
- if (credentialsError || !credentialsData) {
- return (
-
- );
- }
-
- const axeroConnectorIndexingStatuses: ConnectorIndexingStatus<
- AxeroConfig,
- AxeroCredentialJson
- >[] = connectorIndexingStatuses.filter(
- (connectorIndexingStatus) =>
- connectorIndexingStatus.connector.source === "axero"
- );
- const axeroCredential: Credential | undefined =
- credentialsData.find(
- (credential) => credential.credential_json?.axero_api_token
- );
-
- return (
- <>
-
- Step 1: Provide Axero API Key
-
- {axeroCredential ? (
- <>
-
- Existing Axero API Key:
-
- {axeroCredential.credential_json.axero_api_token}
-
-
-
-
- ) : (
- <>
-
- To use the Axero connector, first follow the guide{" "}
-
- here
- {" "}
- to generate an API Key.
-
-
-
- formBody={
- <>
-
-
-
- }
- validationSchema={Yup.object().shape({
- base_url: Yup.string().required(
- "Please enter the base URL of your Axero instance"
- ),
- axero_api_token: Yup.string().required(
- "Please enter your Axero API Token"
- ),
- })}
- initialValues={{
- base_url: "",
- axero_api_token: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Which spaces do you want to connect?
-
-
- {axeroConnectorIndexingStatuses.length > 0 && (
- <>
-
- We pull the latest Articles, Blogs, Wikis and{" "}
- Forums once per day.
-
-
-
- connectorIndexingStatuses={axeroConnectorIndexingStatuses}
- liveCredential={axeroCredential}
- getCredential={(credential) =>
- credential.credential_json.axero_api_token
- }
- specialColumns={[
- {
- header: "Space",
- key: "spaces",
- getValue: (ccPairStatus) => {
- const connectorConfig =
- ccPairStatus.connector.connector_specific_config;
- return connectorConfig.spaces &&
- connectorConfig.spaces.length > 0
- ? connectorConfig.spaces.join(", ")
- : "";
- },
- },
- ]}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- onCredentialLink={async (connectorId) => {
- if (axeroCredential) {
- await linkCredential(connectorId, axeroCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- />
- Configure an Axero Connector
-
- nameBuilder={(values) =>
- values.spaces
- ? `AxeroConnector-${values.spaces.join("_")}`
- : `AxeroConnector`
- }
- source="axero"
- inputType="poll"
- formBodyBuilder={(values) => {
- return (
- <>
-
- {TextArrayFieldBuilder({
- name: "spaces",
- label: "Space IDs:",
- subtext: `
- Specify zero or more Spaces to index (by the Space IDs). If no Space IDs
- are specified, all Spaces will be indexed.`,
- })(values)}
-
- );
- }}
- validationSchema={Yup.object().shape({
- spaces: Yup.array()
- .of(Yup.string().required("Space Ids cannot be empty"))
- .required(),
- })}
- initialValues={{
- spaces: [],
- }}
- refreshFreq={60 * 60 * 24} // 1 day
- credentialId={axeroCredential.id}
- />
-
- ) : (
-
- Please provide your Axero API Token in Step 1 first! Once done with
- that, you can then specify which spaces you want to connect.
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Axero" />
-
-
-
- Existing API Token:
-
- {bookstackCredential.credential_json?.bookstack_api_token_id}
-
-
-
-
- ) : (
- <>
-
- To get started you'll need API token details for your BookStack
- instance. You can get these by editing your (or another) user
- account in BookStack and creating a token via the 'API
- Tokens' section at the bottom. Your user account will require
- to be assigned a BookStack role which has the 'Access system
- API' system permission assigned.
-
-
-
- formBody={
- <>
-
-
-
-
- }
- validationSchema={Yup.object().shape({
- bookstack_base_url: Yup.string().required(
- "Please enter the base URL for your BookStack instance"
- ),
- bookstack_api_token_id: Yup.string().required(
- "Please enter your BookStack API token ID"
- ),
- bookstack_api_token_secret: Yup.string().required(
- "Please enter your BookStack API token secret"
- ),
- })}
- initialValues={{
- bookstack_base_url: "",
- bookstack_api_token_id: "",
- bookstack_api_token_secret: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- />
-
-
- )}
-
- {bookstackConnectorIndexingStatuses.length > 0 && (
- <>
-
- BookStack indexing status
-
-
- The latest page, chapter, book and shelf changes are fetched every
- 10 minutes.
-
-
-
- connectorIndexingStatuses={bookstackConnectorIndexingStatuses}
- liveCredential={bookstackCredential}
- getCredential={(credential) => {
- return (
-
-
{credential.credential_json.bookstack_api_token_id}
-
Create Connection
-
- Press connect below to start the connection to your BookStack
- instance.
-
-
- nameBuilder={(values) => `BookStackConnector`}
- ccPairNameBuilder={(values) => `BookStackConnector`}
- source="bookstack"
- inputType="poll"
- formBody={<>}
- validationSchema={Yup.object().shape({})}
- initialValues={{}}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={bookstackCredential.id}
- />
-
-
- )}
-
- {!bookstackCredential && (
- <>
-
- Please provide your API details in Step 1 first! Once done with
- that, you'll be able to start the connection then see indexing
- status.
-
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Bookstack" />
-
-
-
Failed to load connectors
;
- }
-
- if (isCredentialsError || !credentialsData) {
- return Failed to load credentials
;
- }
-
- const clickupConnectorIndexingStatuses: ConnectorIndexingStatus<
- ClickupConfig,
- ClickupCredentialJson
- >[] = connectorIndexingStatuses.filter(
- (connectorIndexingStatus) =>
- connectorIndexingStatus.connector.source === "clickup"
- );
-
- const clickupCredential: Credential | undefined =
- credentialsData.find(
- (credential) => credential.credential_json?.clickup_api_token
- );
-
- return (
- <>
- {popup}
-
- Step 1: Provide Credentials
-
-
- {clickupCredential ? (
- <>
-
- Existing Clickup API Token:
-
- {clickupCredential.credential_json.clickup_api_token}
-
-
-
-
- ) : (
- <>
-
- To use the Clickup connector, you must first provide the API token
- and Team ID corresponding to your Clickup setup. See setup guide{" "}
-
- here
- {" "}
- for more detail.
-
-
-
- formBody={
- <>
-
-
-
- }
- validationSchema={Yup.object().shape({
- clickup_api_token: Yup.string().required(
- "Please enter your Clickup API token"
- ),
- clickup_team_id: Yup.string().required(
- "Please enter your Team ID"
- ),
- })}
- initialValues={{
- clickup_api_token: "",
- clickup_team_id: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Do you want to search particular space(s), folder(s), list(s),
- or entire workspace?
-
-
- {clickupConnectorIndexingStatuses.length > 0 && (
- <>
-
- We index the latest articles from either the entire workspace, or
- specified space(s), folder(s), list(s) listed below regularly.
-
-
-
- connectorIndexingStatuses={clickupConnectorIndexingStatuses}
- liveCredential={clickupCredential}
- getCredential={(credential) =>
- credential.credential_json.clickup_api_token
- }
- specialColumns={[
- {
- header: "Connector Type",
- key: "connector_type",
- getValue: (ccPairStatus) =>
- ccPairStatus.connector.connector_specific_config
- .connector_type,
- },
- {
- header: "ID(s)",
- key: "connector_ids",
- getValue: (ccPairStatus) =>
- ccPairStatus.connector.connector_specific_config
- .connector_ids &&
- ccPairStatus.connector.connector_specific_config
- .connector_ids.length > 0
- ? ccPairStatus.connector.connector_specific_config.connector_ids.join(
- ", "
- )
- : "",
- },
- {
- header: "Retrieve Task Comments?",
- key: "retrieve_task_comments",
- getValue: (ccPairStatus) =>
- ccPairStatus.connector.connector_specific_config
- .retrieve_task_comments
- ? "Yes"
- : "No",
- },
- ]}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- onCredentialLink={async (connectorId) => {
- if (clickupCredential) {
- await linkCredential(connectorId, clickupCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- />
-
-
-
- )}
-
- {clickupCredential ? (
-
- Connect to a New Workspace
-
- nameBuilder={(values) =>
- values.connector_ids
- ? `ClickupConnector-${
- values.connector_type
- }-${values.connector_ids.join("_")}`
- : `ClickupConnector-${values.connector_type}`
- }
- source="clickup"
- inputType="poll"
- formBody={
- <>
-
-
- }
- formBodyBuilder={(values) => {
- return (
- <>
-
- {TextArrayFieldBuilder({
- name: "connector_ids",
- label: "ID(s):",
- subtext: "Specify 0 or more id(s) to index from.",
- })(values)}
-
-
- );
- }}
- validationSchema={Yup.object().shape({
- connector_type: Yup.string()
- .oneOf(["workspace", "space", "folder", "list"])
- .required("Please select the connector_type to index"),
- connector_ids: Yup.array()
- .of(Yup.string().required("ID(s) must be strings"))
- .test(
- "idsRequired",
- "At least 1 ID is required if space, folder or list is selected",
- function (value) {
- if (this.parent.connector_type === "workspace") return true;
- else if (value !== undefined && value.length > 0)
- return true;
- setPopup({
- type: "error",
- message: `Add at least one ${this.parent.connector_type} ID`,
- });
- return false;
- }
- ),
- retrieve_task_comments: Yup.boolean().required(),
- })}
- initialValues={{
- connector_type: "workspace",
- connector_ids: [],
- retrieve_task_comments: true,
- }}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={clickupCredential.id}
- />
-
- ) : (
-
- Please provide your Clickup API token and Team ID in Step 1 first!
- Once done with that, you can then specify whether you want to make the
- entire workspace, or specified space(s), folder(s), list(s)
- searchable.
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Clickup" />
-
-
-
- {/*
-
Existing Username:
-
- {confluenceCredential.credential_json?.confluence_username}
-
{" "}
-
*/}
-
Existing Access Token:
-
- {confluenceCredential.credential_json?.confluence_access_token}
-
-
-
- Step 2: Which spaces do you want to make searchable?
-
- {confluenceCredential ? (
- <>
-
- Specify any link to a Confluence page below and click
- "Index" to Index. Based on the provided link, we will
- index either the entire page and its subpages OR the entire space.
- For example, entering{" "}
-
- https://danswer.atlassian.net/wiki/spaces/Engineering/overview
- {" "}
- and clicking the Index button will index the whole{" "}
- Engineering Confluence space, but entering
- https://danswer.atlassian.net/wiki/spaces/Engineering/pages/164331/example+page
- will index that page's children (and optionally, itself). Use
- the checkbox below to determine whether or not to index the parent
- page in addition to its children.
-
-
- {confluenceConnectorIndexingStatuses.length > 0 && (
- <>
-
- We pull the latest pages and comments from each space listed
- below every 10 minutes.
-
-
-
- connectorIndexingStatuses={
- confluenceConnectorIndexingStatuses
- }
- liveCredential={confluenceCredential}
- getCredential={(credential) => {
- return (
-
-
- {credential.credential_json.confluence_access_token}
-
-
Add a New Space or Page
-
- nameBuilder={(values) =>
- `ConfluenceConnector-${values.wiki_page_url}`
- }
- ccPairNameBuilder={(values) =>
- extractSpaceFromUrl(values.wiki_page_url)
- }
- source="confluence"
- inputType="poll"
- formBody={
- <>
-
-
-
- }
- validationSchema={Yup.object().shape({
- wiki_page_url: Yup.string().required(
- "Please enter any link to a Confluence space or Page e.g. https://danswer.atlassian.net/wiki/spaces/Engineering/overview"
- ),
- index_origin: Yup.boolean(),
- })}
- initialValues={{
- wiki_page_url: "",
- index_origin: true,
- }}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={confluenceCredential.id}
- />
-
-
- ) : (
-
- Please provide your access token in Step 1 first! Once done with that,
- you can then specify which Confluence spaces you want to make
- searchable.
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Confluence" />
-
-
-
-
Existing API Key:
-
- {discourseCredential.credential_json?.discourse_api_key}
-
-
-
-
- connectorIndexingStatuses={discourseConnectorIndexingStatuses}
- liveCredential={discourseCredential}
- getCredential={(credential) =>
- credential.credential_json.discourse_api_username
- }
- specialColumns={[
- {
- header: "Categories",
- key: "categories",
- getValue: (ccPairStatus) =>
- ccPairStatus.connector.connector_specific_config
- .categories &&
- ccPairStatus.connector.connector_specific_config.categories
- .length > 0
- ? ccPairStatus.connector.connector_specific_config.categories.join(
- ", "
- )
- : "",
- },
- ]}
- includeName={true}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- onCredentialLink={async (connectorId) => {
- if (discourseCredential) {
- await linkCredential(connectorId, discourseCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- />
-
-
-
- )}
-
- {discourseCredential ? (
- <>
-
- Create a new Discourse Connector
-
- nameBuilder={(values) =>
- values.categories
- ? `${values.base_url}-${values.categories.join("_")}`
- : `${values.base_url}-All`
- }
- source="discourse"
- inputType="poll"
- formBody={
- <>
-
-
- }
- formBodyBuilder={TextArrayFieldBuilder({
- name: "categories",
- label: "Categories:",
- subtext:
- "Specify 0 or more Categories to index. If no Categories are specified, Topics from " +
- "all categories will be indexed.",
- })}
- validationSchema={Yup.object().shape({
- base_url: Yup.string().required(
- "Please the base URL of your Discourse site."
- ),
- categories: Yup.array().of(
- Yup.string().required("Category names must be strings")
- ),
- })}
- initialValues={{
- categories: [],
- base_url: "",
- }}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={discourseCredential.id}
- />
-
-
- ) : (
-
- Please provide your API Key Info in Step 1 first! Once done with that,
- you can then start indexing all your Discourse Topics.
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Discourse" />
-
-
-
- Existing Document360 API Token:
-
- {document360Credential.credential_json.document360_api_token}
-
-
-
-
- ) : (
- <>
-
- To use the Document360 connector, you must first provide the API
- token and portal ID corresponding to your Document360 setup. See
- setup guide{" "}
-
- here
- {" "}
- for more detail.
-
-
-
- formBody={
- <>
-
-
-
- }
- validationSchema={Yup.object().shape({
- document360_api_token: Yup.string().required(
- "Please enter your Document360 API token"
- ),
- portal_id: Yup.string().required("Please enter your portal ID"),
- })}
- initialValues={{
- document360_api_token: "",
- portal_id: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Which categories do you want to make searchable?
-
-
- {document360ConnectorIndexingStatuses.length > 0 && (
- <>
-
- We index the latest articles from each workspace listed below
- regularly.
-
-
-
- connectorIndexingStatuses={document360ConnectorIndexingStatuses}
- liveCredential={document360Credential}
- getCredential={(credential) =>
- credential.credential_json.document360_api_token
- }
- specialColumns={[
- {
- header: "Workspace",
- key: "workspace",
- getValue: (ccPairStatus) =>
- ccPairStatus.connector.connector_specific_config.workspace,
- },
- {
- header: "Categories",
- key: "categories",
- getValue: (ccPairStatus) =>
- ccPairStatus.connector.connector_specific_config
- .categories &&
- ccPairStatus.connector.connector_specific_config.categories
- .length > 0
- ? ccPairStatus.connector.connector_specific_config.categories.join(
- ", "
- )
- : "",
- },
- ]}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- onCredentialLink={async (connectorId) => {
- if (document360Credential) {
- await linkCredential(connectorId, document360Credential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- />
-
-
-
- )}
-
- {document360Credential ? (
-
- Connect to a New Workspace
-
- nameBuilder={(values) =>
- values.categories
- ? `Document360Connector-${
- values.workspace
- }-${values.categories.join("_")}`
- : `Document360Connector-${values.workspace}`
- }
- source="document360"
- inputType="poll"
- formBody={
- <>
-
-
- }
- formBodyBuilder={TextArrayFieldBuilder({
- name: "categories",
- label: "Categories:",
- subtext:
- "Specify 0 or more categories to index. For instance, specifying the category " +
- "'Help' will cause us to only index all content " +
- "within the 'Help' category. " +
- "If no categories are specified, all categories in your workspace will be indexed.",
- })}
- validationSchema={Yup.object().shape({
- workspace: Yup.string().required(
- "Please enter the workspace to index"
- ),
- categories: Yup.array()
- .of(Yup.string().required("Category names must be strings"))
- .required(),
- })}
- initialValues={{
- workspace: "",
- categories: [],
- }}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={document360Credential.id}
- />
-
- ) : (
-
- Please provide your Document360 API token and portal ID in Step 1
- first! Once done with that, you can then specify which Document360
- categories you want to make searchable.
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
}
- title="Document360"
- />
-
-
-
-
Existing API Token:
-
- {dropboxCredential.credential_json?.dropbox_access_token}
-
-
-
-
- connectorIndexingStatuses={dropboxConnectorIndexingStatuses}
- liveCredential={dropboxCredential}
- onCredentialLink={async (connectorId) => {
- if (dropboxCredential) {
- await linkCredential(connectorId, dropboxCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- />
-
-
- )}
-
- {dropboxCredential && dropboxConnectorIndexingStatuses.length === 0 && (
- <>
-
- Create Connection
-
- Press connect below to start the connection to your Dropbox
- instance.
-
-
- nameBuilder={(values) => `Dropbox`}
- ccPairNameBuilder={(values) => `Dropbox`}
- source="dropbox"
- inputType="poll"
- formBody={<>}
- validationSchema={Yup.object().shape({})}
- initialValues={{}}
- // refreshFreq={10 * 60} // disabled re-indexing
- credentialId={dropboxCredential.id}
- />
-
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
} title="Dropbox" />
-
-
- {popup}
- {filesAreUploading &&
}
-
- Specify files below, click the Upload button, and the contents of
- these files will be searchable via Danswer! Currently supported file
- types include .txt, .pdf, .docx, .pptx,{" "}
- .xlsx, .csv, .md, .mdx, .conf,{" "}
- .log, .json, .tsv, .xml, .yml,{" "}
- .yaml, .eml, .epub, and finally .zip files
- (containing supported file types).
-
-
- NOTE: if the original document is accessible via a link, you can
- add a line at the very beginning of the file that looks like:
-
-
- #DANSWER_METADATA={"{"}"link": "{""}"{"}"}
-
-
-
-
-
- {
- const uploadCreateAndTriggerConnector = async () => {
- const formData = new FormData();
-
- selectedFiles.forEach((file) => {
- formData.append("files", file);
- });
-
- const response = await fetch(
- "/api/manage/admin/connector/file/upload",
- { method: "POST", body: formData }
- );
- const responseJson = await response.json();
- if (!response.ok) {
- setPopup({
- message: `Unable to upload files - ${responseJson.detail}`,
- type: "error",
- });
- return;
- }
-
- const filePaths = responseJson.file_paths as string[];
- const [connectorErrorMsg, connector] =
- await createConnector({
- name: "FileConnector-" + Date.now(),
- source: "file",
- input_type: "load_state",
- connector_specific_config: {
- file_locations: filePaths,
- },
- refresh_freq: null,
- prune_freq: 0,
- disabled: false,
- });
- if (connectorErrorMsg || !connector) {
- setPopup({
- message: `Unable to create connector - ${connectorErrorMsg}`,
- type: "error",
- });
- return;
- }
-
- // Since there is no "real" credential associated with a file connector
- // we create a dummy one here so that we can associate the CC Pair with a
- // user. This is needed since the user for a CC Pair is found via the credential
- // associated with it.
- const createCredentialResponse = await createCredential({
- credential_json: {},
- admin_public: true,
- });
- if (!createCredentialResponse.ok) {
- const errorMsg = await createCredentialResponse.text();
- setPopup({
- message: `Error creating credential for CC Pair - ${errorMsg}`,
- type: "error",
- });
- formikHelpers.setSubmitting(false);
- return;
- }
- const credentialId = (await createCredentialResponse.json())
- .id;
-
- const credentialResponse = await linkCredential(
- connector.id,
- credentialId,
- values.name,
- values.is_public
- );
- if (!credentialResponse.ok) {
- const credentialResponseJson =
- await credentialResponse.json();
- setPopup({
- message: `Unable to link connector to credential - ${credentialResponseJson.detail}`,
- type: "error",
- });
- return;
- }
-
- const runConnectorErrorMsg = await runConnector(
- connector.id,
- [0]
- );
- if (runConnectorErrorMsg) {
- setPopup({
- message: `Unable to run connector - ${runConnectorErrorMsg}`,
- type: "error",
- });
- return;
- }
-
- mutate("/api/manage/admin/connector/indexing-status");
- setSelectedFiles([]);
- formikHelpers.resetForm();
- setPopup({
- type: "success",
- message: "Successfully uploaded files!",
- });
- };
-
- setFilesAreUploading(true);
- try {
- await uploadCreateAndTriggerConnector();
- } catch (e) {
- console.log("Failed to index filels: ", e);
- }
- setFilesAreUploading(false);
- }}
- >
- {({ values, isSubmitting }) => (
-
- )}
-
-
-
-
-
- {fileIndexingStatuses.length > 0 && (
-
-
-
Indexed Files
-
- connectorIndexingStatuses={fileIndexingStatuses}
- specialColumns={[
- {
- header: "File Names",
- key: "file_names",
- getValue: (ccPairStatus) =>
- ccPairStatus.connector.connector_specific_config.file_locations
- .map(getNameFromPath)
- .join(", "),
- },
- ]}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- />
-
- )}
-
-
-
-
-
-
} title="File" />
-
-
-
-
Existing Access Token:
-
- {githubCredential.credential_json.github_access_token}
-
{" "}
-
-
-
- connectorIndexingStatuses={githubConnectorIndexingStatuses}
- liveCredential={githubCredential}
- getCredential={(credential) =>
- credential.credential_json.github_access_token
- }
- onCredentialLink={async (connectorId) => {
- if (githubCredential) {
- await linkCredential(connectorId, githubCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- specialColumns={[
- {
- header: "Repository",
- key: "repository",
- getValue: (ccPairStatus) => {
- const connectorConfig =
- ccPairStatus.connector.connector_specific_config;
- return `${connectorConfig.repo_owner}/${connectorConfig.repo_name}`;
- },
- },
- ]}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- />
-
-
-
- )}
-
- {githubCredential ? (
-
- Connect to a New Repository
-
-
- The Github connector can index Pull Requests and Issues.
-
-
-
- nameBuilder={(values) =>
- `GithubConnector-${values.repo_owner}/${values.repo_name}`
- }
- ccPairNameBuilder={(values) =>
- `${values.repo_owner}/${values.repo_name}`
- }
- source="github"
- inputType="poll"
- formBody={
- <>
-
-
-
-
-
- }
- validationSchema={Yup.object().shape({
- repo_owner: Yup.string().required(
- "Please enter the owner of the repository to index e.g. danswer-ai"
- ),
- repo_name: Yup.string().required(
- "Please enter the name of the repository to index e.g. danswer "
- ),
- include_prs: Yup.boolean().required(),
- include_issues: Yup.boolean().required(),
- })}
- validate={(values) => {
- if (values.include_prs || values.include_issues) {
- return {} as Record;
- }
- return {
- include_issues:
- "Please select at least one of Pull Requests or Issues",
- };
- }}
- initialValues={{
- repo_owner: "",
- repo_name: "",
- include_prs: true,
- include_issues: true,
- }}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={githubCredential.id}
- />
-
- ) : (
-
- Please provide your access token in Step 1 first! Once done with that,
- you can then specify which Github repositories you want to make
- searchable.
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
}
- title="Github PRs + Issues"
- />
-
-
-
-
Existing Access Token:
-
- {gitlabCredential.credential_json.gitlab_access_token}
-
{" "}
-
-
-
- connectorIndexingStatuses={gitlabConnectorIndexingStatuses}
- liveCredential={gitlabCredential}
- getCredential={(credential) =>
- credential.credential_json.gitlab_access_token
- }
- onCredentialLink={async (connectorId) => {
- if (gitlabCredential) {
- await linkCredential(connectorId, gitlabCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- specialColumns={[
- {
- header: "Project",
- key: "project",
- getValue: (ccPairStatus) => {
- const connectorConfig =
- ccPairStatus.connector.connector_specific_config;
- return `${connectorConfig.project_owner}/${connectorConfig.project_name}`;
- },
- },
- ]}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- />
-
-
-
- )}
-
- {gitlabCredential ? (
-
- Connect to a New Project
-
- nameBuilder={(values) =>
- `GitlabConnector-${values.project_owner}/${values.project_name}`
- }
- ccPairNameBuilder={(values) =>
- `${values.project_owner}/${values.project_name}`
- }
- source="gitlab"
- inputType="poll"
- formBody={
- <>
-
-
-
- }
- validationSchema={Yup.object().shape({
- project_owner: Yup.string().required(
- "Please enter the owner of the project to index e.g. danswer-ai"
- ),
- project_name: Yup.string().required(
- "Please enter the name of the project to index e.g. danswer "
- ),
- include_mrs: Yup.boolean().required(),
- include_issues: Yup.boolean().required(),
- })}
- initialValues={{
- project_owner: "",
- project_name: "",
- include_mrs: true,
- include_issues: true,
- }}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={gitlabCredential.id}
- />
-
- ) : (
-
- Please provide your access token in Step 1 first! Once done with that,
- you can then specify which Gitlab repositories you want to make
- searchable.
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
}
- title="Gitlab MRs + Issues"
- />
-
-
-
-
-
-
- Status
- Delete
-
-
-
- {sortedGmailConnectorIndexingStatuses.map(
- (connectorIndexingStatus) => {
- return (
-
-
- {
- mutate("/api/manage/admin/connector/indexing-status");
- }}
- />
-
-
-
- mutate("/api/manage/admin/connector/indexing-status")
- }
- />
-
-
- );
- }
- )}
-
-
-
-
-
- {gmailConnectorIndexingStatuses.length > 0 ? (
- <>
- Checkout the{" "}
-
- status page
- {" "}
- for the latest indexing status. We fetch the latest mails from
- Gmail every
10 minutes.
-
- ) : (
-
- Fill out the form below to create a connector. We will refresh the
- latest documents from Gmail every 10 minutes.
-
- )}
-
- {gmailConnectorIndexingStatuses.length > 0 && (
- <>
-
Existing Connectors:
-
-
-
- )}
-
- {gmailConnectorIndexingStatuses.length > 0 && (
-
Add New Connector:
- )}
-
-
- nameBuilder={gmailConnectorNameBuilder}
- source="gmail"
- inputType="poll"
- formBody={null}
- validationSchema={Yup.object().shape({})}
- initialValues={{}}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={liveCredential.id}
- />
-
-
-
-
- );
- }
-
- if (credentialsError || !credentialsData) {
- return (
-
-
Failed to load credentials.
-
-
Failed to load connectors.
-
-
- Error loading Gmail app credentials. Contact an administrator.
-
-
-
-
-
-
-
} title="Gmail" />
-
-
-
-
Existing Access Key Secret:
-
- {gongCredential.credential_json?.gong_access_key_secret}
-
-
-
-
- connectorIndexingStatuses={gongConnectorIndexingStatuses}
- liveCredential={gongCredential}
- getCredential={(credential) =>
- credential.credential_json.gong_access_key
- }
- specialColumns={[
- {
- header: "Workspaces",
- key: "workspaces",
- getValue: (ccPairStatus) =>
- ccPairStatus.connector.connector_specific_config
- .workspaces &&
- ccPairStatus.connector.connector_specific_config.workspaces
- .length > 0
- ? ccPairStatus.connector.connector_specific_config.workspaces.join(
- ", "
- )
- : "",
- },
- ]}
- includeName={true}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- onCredentialLink={async (connectorId) => {
- if (gongCredential) {
- await linkCredential(connectorId, gongCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- />
-
-
-
- )}
-
- {gongCredential ? (
- <>
-
- Create a new Gong Connector
-
- nameBuilder={(values) =>
- values.workspaces
- ? `GongConnector-${values.workspaces.join("_")}`
- : `GongConnector-All`
- }
- source="gong"
- inputType="poll"
- formBodyBuilder={TextArrayFieldBuilder({
- name: "workspaces",
- label: "Workspaces:",
- subtext:
- "Specify 0 or more workspaces to index. Provide the workspace ID or the EXACT workspace " +
- "name from Gong. If no workspaces are specified, transcripts from all workspaces will " +
- "be indexed.",
- })}
- validationSchema={Yup.object().shape({
- workspaces: Yup.array().of(
- Yup.string().required("Workspace names must be strings")
- ),
- })}
- initialValues={{
- workspaces: [],
- }}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={gongCredential.id}
- />
-
-
- ) : (
-
- Please provide your API Access Info in Step 1 first! Once done with
- that, you can then start indexing all your Gong transcripts.
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Gong" />
-
-
-
-
- Update Google Drive Connector
-
-
-
-
-
-
- Modify the selected Google Drive connector by adjusting the values
- below!
-
-
-
-
-
- nameBuilder={googleDriveConnectorNameBuilder}
- existingConnector={existingConnector}
- formBodyBuilder={(values) => (
-
- {TextArrayFieldBuilder({
- name: "folder_paths",
- label: "Folder Paths",
- })(values)}
-
-
-
-
- )}
- validationSchema={Yup.object().shape({
- folder_paths: Yup.array()
- .of(
- Yup.string().required(
- "Please specify a folder path for your google drive e.g. 'Engineering/Materials'"
- )
- )
- .required(),
- include_shared: Yup.boolean().required(),
- follow_shortcuts: Yup.boolean().required(),
- only_org_public: Yup.boolean().required(),
- })}
- onSubmit={onSubmit}
- />
-
-
{
- setIsEditing(true);
- }}
- className="cursor-pointer"
- >
-
-
-
-
-
-
-
-
- Edit
- Folder Paths
- Include Shared
- Follow Shortcuts
- Only Org Public
- Status
- Delete
-
-
-
- {sortedGoogleDriveConnectorIndexingStatuses.map(
- (connectorIndexingStatus) => {
- return (
-
-
-
-
-
- {(
- connectorIndexingStatus.connector
- .connector_specific_config.folder_paths || []
- ).length > 0 ? (
-
- {(
- connectorIndexingStatus.connector
- .connector_specific_config.folder_paths || []
- ).map((path) => (
-
- - {path}
-
- ))}
-
- {connectorIndexingStatus.connector
- .connector_specific_config.include_shared ? (
- Yes
- ) : (
- No
- )}
-
-
-
-
- {connectorIndexingStatus.connector
- .connector_specific_config.follow_shortcuts ? (
- Yes
- ) : (
- No
- )}
-
-
-
-
- {connectorIndexingStatus.connector
- .connector_specific_config.only_org_public ? (
- Yes
- ) : (
- No
- )}
-
-
-
- {
- mutate("/api/manage/admin/connector/indexing-status");
- }}
- />
-
-
-
- mutate("/api/manage/admin/connector/indexing-status")
- }
- />
-
-
- );
- }
- )}
-
-
-
- {(
- connectorIndexingStatus.connector.connector_specific_config
- .folder_paths || []
- ).map((path) => (
-
- - {path}
-
- ))}
-
- {connectorIndexingStatus.connector.connector_specific_config
- .include_shared ? (
- Yes
- ) : (
- No
- )}
-
- ),
- follow_shortcuts: (
-
- {connectorIndexingStatus.connector.connector_specific_config
- .follow_shortcuts ? (
- Yes
- ) : (
- No
- )}
-
- ),
- only_org_public: (
-
- {connectorIndexingStatus.connector.connector_specific_config
- .only_org_public ? (
- Yes
- ) : (
- No
- )}
-
- ),
- status: (
- {
- mutate("/api/manage/admin/connector/indexing-status");
- }}
- />
- ),
- delete: (
-
- mutate("/api/manage/admin/connector/indexing-status")
- }
- />
- ),
- })
- )}
- />
- );
-};
diff --git a/web/src/app/admin/connectors/google-drive/page.tsx b/web/src/app/admin/connectors/google-drive/page.tsx
deleted file mode 100644
index 121d8af9d0a..00000000000
--- a/web/src/app/admin/connectors/google-drive/page.tsx
+++ /dev/null
@@ -1,427 +0,0 @@
-"use client";
-
-import * as Yup from "yup";
-import { GoogleDriveIcon } from "@/components/icons/icons";
-import useSWR, { useSWRConfig } from "swr";
-import { FetchError, errorHandlingFetcher } from "@/lib/fetcher";
-import { ErrorCallout } from "@/components/ErrorCallout";
-import { LoadingAnimation } from "@/components/Loading";
-import { PopupSpec, usePopup } from "@/components/admin/connectors/Popup";
-import { HealthCheckBanner } from "@/components/health/healthcheck";
-import {
- ConnectorIndexingStatus,
- Credential,
- GoogleDriveConfig,
- GoogleDriveCredentialJson,
- GoogleDriveServiceAccountCredentialJson,
-} from "@/lib/types";
-import { linkCredential } from "@/lib/credential";
-import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
-import {
- BooleanFormField,
- TextArrayFieldBuilder,
-} from "@/components/admin/connectors/Field";
-import { GoogleDriveConnectorsTable } from "./GoogleDriveConnectorsTable";
-import { googleDriveConnectorNameBuilder } from "./utils";
-import { DriveOAuthSection, DriveJsonUploadSection } from "./Credential";
-import { usePublicCredentials } from "@/lib/hooks";
-import { AdminPageTitle } from "@/components/admin/Title";
-import { Card, Divider, Text, Title } from "@tremor/react";
-
-interface GoogleDriveConnectorManagementProps {
- googleDrivePublicCredential?: Credential;
- googleDriveServiceAccountCredential?: Credential;
- googleDriveConnectorIndexingStatus: ConnectorIndexingStatus<
- GoogleDriveConfig,
- GoogleDriveCredentialJson
- > | null;
- googleDriveConnectorIndexingStatuses: ConnectorIndexingStatus<
- GoogleDriveConfig,
- GoogleDriveCredentialJson
- >[];
- credentialIsLinked: boolean;
- setPopup: (popupSpec: PopupSpec | null) => void;
-}
-
-const GoogleDriveConnectorManagement = ({
- googleDrivePublicCredential,
- googleDriveServiceAccountCredential,
- googleDriveConnectorIndexingStatus,
- googleDriveConnectorIndexingStatuses,
- credentialIsLinked,
- setPopup,
-}: GoogleDriveConnectorManagementProps) => {
- const { mutate } = useSWRConfig();
-
- const liveCredential =
- googleDrivePublicCredential || googleDriveServiceAccountCredential;
- if (!liveCredential) {
- return (
-
- Please authenticate with Google Drive as described in Step 2! Once done
- with that, you can then move on to enable this connector.
-
- );
- }
-
- // NOTE: if the connector has no credential linked, then it will not be
- // returned by the indexing-status API
- // if (!googleDriveConnectorIndexingStatus) {
- // return (
- // <>
- //
- // Fill out the form below to create a connector. We will refresh the
- // latest documents from Google Drive every 10 minutes.
- //
- //
- //
Add Connector
- //
- // nameBuilder={googleDriveConnectorNameBuilder}
- // source="google_drive"
- // inputType="poll"
- // formBodyBuilder={(values) => (
- //
- // {TextArrayFieldBuilder({
- // name: "folder_paths",
- // label: "Folder Paths",
- // subtext:
- // "Specify 0 or more folder paths to index! For example, specifying the path " +
- // "'Engineering/Materials' will cause us to only index all files contained " +
- // "within the 'Materials' folder within the 'Engineering' folder. " +
- // "If no folder paths are specified, we will index all documents in your drive.",
- // })(values)}
- //
- //
- // )}
- // validationSchema={Yup.object().shape({
- // folder_paths: Yup.array()
- // .of(
- // Yup.string().required(
- // "Please specify a folder path for your google drive e.g. 'Engineering/Materials'"
- // )
- // )
- // .required(),
- // include_shared: Yup.boolean().required(),
- // })}
- // initialValues={{
- // folder_paths: [],
- // }}
- // refreshFreq={10 * 60} // 10 minutes
- // onSubmit={async (isSuccess, responseJson) => {
- // if (isSuccess && responseJson) {
- // await linkCredential(
- // responseJson.id,
- // googleDrivePublicCredential.id
- // );
- // mutate("/api/manage/admin/connector/indexing-status");
- // }
- // }}
- // />
- //
- // Click the button below to link your credentials! Once this is done, all
- // public documents in your Google Drive will be searchable. We will
- // refresh the latest documents every 10 minutes.
- //
- //
- // ;
- // }
-
- return (
-
-
-
- {googleDriveConnectorIndexingStatuses.length > 0 ? (
- <>
- Checkout the{" "}
-
- status page
- {" "}
- for the latest indexing status. We fetch the latest documents from
- Google Drive every
10 minutes.
-
- ) : (
-
- Fill out the form below to create a connector. We will refresh the
- latest documents from Google Drive every 10 minutes.
-
- )}
-
- {googleDriveConnectorIndexingStatuses.length > 0 && (
- <>
-
Existing Connectors:
-
-
-
- )}
-
- {googleDriveConnectorIndexingStatuses.length > 0 && (
-
Add New Connector:
- )}
-
-
- nameBuilder={googleDriveConnectorNameBuilder}
- source="google_drive"
- inputType="poll"
- formBodyBuilder={(values) => (
- <>
- {TextArrayFieldBuilder({
- name: "folder_paths",
- label: "Folder Paths",
- subtext:
- "Specify 0 or more folder paths to index! For example, specifying the path " +
- "'Engineering/Materials' will cause us to only index all files contained " +
- "within the 'Materials' folder within the 'Engineering' folder. " +
- "If no folder paths are specified, we will index all documents in your drive.",
- })(values)}
-
-
-
-
- )}
- validationSchema={Yup.object().shape({
- folder_paths: Yup.array()
- .of(
- Yup.string().required(
- "Please specify a folder path for your google drive e.g. 'Engineering/Materials'"
- )
- )
- .required(),
- include_shared: Yup.boolean().required(),
- follow_shortcuts: Yup.boolean().required(),
- only_org_public: Yup.boolean().required(),
- })}
- initialValues={{
- folder_paths: [],
- include_shared: false,
- follow_shortcuts: false,
- only_org_public: false,
- }}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={liveCredential.id}
- />
-
-
-
-
- );
- }
-
- if (credentialsError || !credentialsData) {
- return ;
- }
-
- if (connectorIndexingStatusesError || !connectorIndexingStatuses) {
- return ;
- }
-
- if (
- !appCredentialSuccessfullyFetched ||
- !serviceAccountKeySuccessfullyFetched
- ) {
- return (
-
- );
- }
-
- const googleDrivePublicCredential:
- | Credential
- | undefined = credentialsData.find(
- (credential) =>
- credential.credential_json?.google_drive_tokens && credential.admin_public
- );
- const googleDriveServiceAccountCredential:
- | Credential
- | undefined = credentialsData.find(
- (credential) => credential.credential_json?.google_drive_service_account_key
- );
- const googleDriveConnectorIndexingStatuses: ConnectorIndexingStatus<
- GoogleDriveConfig,
- GoogleDriveCredentialJson
- >[] = connectorIndexingStatuses.filter(
- (connectorIndexingStatus) =>
- connectorIndexingStatus.connector.source === "google_drive"
- );
- const googleDriveConnectorIndexingStatus =
- googleDriveConnectorIndexingStatuses[0];
-
- const credentialIsLinked =
- (googleDriveConnectorIndexingStatus !== undefined &&
- googleDrivePublicCredential !== undefined &&
- googleDriveConnectorIndexingStatus.connector.credential_ids.includes(
- googleDrivePublicCredential.id
- )) ||
- (googleDriveConnectorIndexingStatus !== undefined &&
- googleDriveServiceAccountCredential !== undefined &&
- googleDriveConnectorIndexingStatus.connector.credential_ids.includes(
- googleDriveServiceAccountCredential.id
- ));
-
- return (
- <>
- {popup}
-
- Step 1: Provide your Credentials
-
-
-
-
- Step 2: Authenticate with Danswer
-
- 0}
- />
-
-
- Step 3: Start Indexing!
-
-
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
}
- title="Google Drive"
- />
-
-
-
-
-
-
-
-
}
- title="Google Sites"
- />
-
-
- For an in-depth guide on how to setup this connector, check out{" "}
-
- the documentation
-
- .
-
-
-
-
Upload Files
-
-
-
{
- const uploadCreateAndTriggerConnector = async () => {
- const formData = new FormData();
-
- selectedFiles.forEach((file) => {
- formData.append("files", file);
- });
-
- const response = await fetch(
- "/api/manage/admin/connector/file/upload",
- { method: "POST", body: formData }
- );
- const responseJson = await response.json();
- if (!response.ok) {
- setPopup({
- message: `Unable to upload files - ${responseJson.detail}`,
- type: "error",
- });
- return;
- }
-
- const filePaths = responseJson.file_paths as string[];
- const [connectorErrorMsg, connector] =
- await createConnector({
- name: `GoogleSitesConnector-${values.base_url}`,
- source: "google_sites",
- input_type: "load_state",
- connector_specific_config: {
- base_url: values.base_url,
- zip_path: filePaths[0],
- },
- refresh_freq: null,
- prune_freq: 0,
- disabled: false,
- });
- if (connectorErrorMsg || !connector) {
- setPopup({
- message: `Unable to create connector - ${connectorErrorMsg}`,
- type: "error",
- });
- return;
- }
-
- const credentialResponse = await linkCredential(
- connector.id,
- 0,
- values.base_url
- );
- if (!credentialResponse.ok) {
- const credentialResponseJson =
- await credentialResponse.json();
- setPopup({
- message: `Unable to link connector to credential - ${credentialResponseJson.detail}`,
- type: "error",
- });
- return;
- }
-
- const runConnectorErrorMsg = await runConnector(
- connector.id,
- [0]
- );
- if (runConnectorErrorMsg) {
- setPopup({
- message: `Unable to run connector - ${runConnectorErrorMsg}`,
- type: "error",
- });
- return;
- }
-
- mutate("/api/manage/admin/connector/indexing-status");
- setSelectedFiles([]);
- formikHelpers.resetForm();
- setPopup({
- type: "success",
- message: "Successfully uploaded files!",
- });
- };
-
- setFilesAreUploading(true);
- try {
- await uploadCreateAndTriggerConnector();
- } catch (e) {
- console.log("Failed to index filels: ", e);
- }
- setFilesAreUploading(false);
- }}
- >
- {({ values, isSubmitting }) => (
-
- )}
-
-
-
-
-
- Existing Google Site Connectors
-
- {isConnectorIndexingStatusesLoading ? (
-
- ) : connectorIndexingStatusesError || !connectorIndexingStatuses ? (
-
Error loading indexing history
- ) : googleSitesIndexingStatuses.length > 0 ? (
-
- connectorIndexingStatuses={googleSitesIndexingStatuses}
- specialColumns={[
- {
- header: "Base URL",
- key: "base_url",
- getValue: (ccPairStatus) => {
- const connectorConfig =
- ccPairStatus.connector.connector_specific_config;
- return (
-
- {connectorConfig.base_url}
-
- );
- },
- },
- ]}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- />
- ) : (
- No indexed Google Sites found
- )}
-
-
Existing GCS Access Key ID:
-
- {gcsCredential.credential_json.access_key_id}
-
- {", "}
-
Secret Access Key:
-
- {gcsCredential.credential_json.secret_access_key}
-
{" "}
-
-
- -
- Provide your GCS Project ID, Client Email, and Private Key for
- authentication.
-
- -
- These credentials will be used to access your GCS buckets.
-
-
-
-
-
- formBody={
- <>
-
-
-
-
- }
- validationSchema={Yup.object().shape({
- secret_access_key: Yup.string().required(
- "Client Email is required"
- ),
- access_key_id: Yup.string().required("Private Key is required"),
- })}
- initialValues={{
- secret_access_key: "",
- access_key_id: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Which GCS bucket do you want to make searchable?
-
-
- {gcsConnectorIndexingStatuses.length > 0 && (
- <>
-
- GCS indexing status
-
-
- The latest changes are fetched every 10 minutes.
-
-
-
- includeName={true}
- connectorIndexingStatuses={gcsConnectorIndexingStatuses}
- liveCredential={gcsCredential}
- getCredential={(credential) => {
- return ;
- }}
- onCredentialLink={async (connectorId) => {
- if (gcsCredential) {
- await linkCredential(connectorId, gcsCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- />
- Create Connection
-
- Press connect below to start the connection to your GCS bucket.
-
-
- nameBuilder={(values) => `GCSConnector-${values.bucket_name}`}
- ccPairNameBuilder={(values) =>
- `GCSConnector-${values.bucket_name}`
- }
- source="google_cloud_storage"
- inputType="poll"
- formBodyBuilder={(values) => (
-
-
-
-
- )}
- validationSchema={Yup.object().shape({
- bucket_type: Yup.string()
- .oneOf(["google_cloud_storage"])
- .required("Bucket type must be google_cloud_storage"),
- bucket_name: Yup.string().required(
- "Please enter the name of the GCS bucket to index, e.g. my-gcs-bucket"
- ),
- prefix: Yup.string().default(""),
- })}
- initialValues={{
- bucket_type: "google_cloud_storage",
- bucket_name: "",
- prefix: "",
- }}
- refreshFreq={60 * 60 * 24} // 1 day
- credentialId={gcsCredential.id}
- />
-
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
}
- title="Google Cloud Storage"
- />
-
-
- Existing Access Token:
-
- {guruCredential.credential_json?.guru_user_token}
-
-
-
-
- ) : (
- <>
-
- To use the Guru connector, first follow the guide{" "}
-
- here
- {" "}
- to generate a User Token.
-
-
-
- formBody={
- <>
-
-
-
- }
- validationSchema={Yup.object().shape({
- guru_user: Yup.string().required(
- "Please enter your Guru username"
- ),
- guru_user_token: Yup.string().required(
- "Please enter your Guru access token"
- ),
- })}
- initialValues={{
- guru_user: "",
- guru_user_token: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Start indexing!
-
- {guruCredential ? (
- !guruConnectorIndexingStatuses.length ? (
- <>
-
- Click the button below to start indexing! We will pull the latest
- features, components, and products from Guru every 10{" "}
- minutes.
-
-
-
- nameBuilder={() => "GuruConnector"}
- ccPairNameBuilder={() => "Guru"}
- source="guru"
- inputType="poll"
- formBody={null}
- validationSchema={Yup.object().shape({})}
- initialValues={{}}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={guruCredential.id}
- />
-
-
- ) : (
- <>
-
- Guru connector is setup! We are pulling the latest cards from Guru
- every 10 minutes.
-
-
- connectorIndexingStatuses={guruConnectorIndexingStatuses}
- liveCredential={guruCredential}
- getCredential={(credential) => {
- return (
-
-
{credential.credential_json.guru_user}
-
-
-
-
-
-
} title="Guru" />
-
-
-
- Existing Access Token:
-
- {hubSpotCredential.credential_json?.hubspot_access_token}
-
-
-
-
- ) : (
- <>
-
- To use the HubSpot connector, provide the HubSpot Access Token.
-
-
-
- formBody={
- <>
-
-
- }
- validationSchema={Yup.object().shape({
- hubspot_access_token: Yup.string().required(
- "Please enter your HubSpot Access Token"
- ),
- })}
- initialValues={{
- hubspot_access_token: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Start indexing!
-
- {hubSpotCredential ? (
- !hubSpotConnectorIndexingStatuses.length ? (
- <>
-
- Click the button below to start indexing! We will pull the latest
- tickets from HubSpot every 10 minutes.
-
-
-
- nameBuilder={() => "HubSpotConnector"}
- ccPairNameBuilder={() => "HubSpotConnector"}
- source="hubspot"
- inputType="poll"
- formBody={null}
- validationSchema={Yup.object().shape({})}
- initialValues={{}}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={hubSpotCredential.id}
- />
-
-
- ) : (
- <>
-
- HubSpot connector is setup! We are pulling the latest tickets from
- HubSpot every 10 minutes.
-
-
- connectorIndexingStatuses={hubSpotConnectorIndexingStatuses}
- liveCredential={hubSpotCredential}
- getCredential={(credential) => {
- return (
-
-
{credential.credential_json.hubspot_access_token}
-
-
-
-
-
-
} title="HubSpot" />
-
-
-
-
Existing Access Token:
-
- {jiraCredential.credential_json?.jira_api_token}
-
-
-
-
- connectorIndexingStatuses={jiraConnectorIndexingStatuses}
- liveCredential={jiraCredential}
- getCredential={(credential) => {
- return (
-
-
{credential.credential_json.jira_api_token}
-
Add a New Project
-
- nameBuilder={(values) =>
- `JiraConnector-${values.jira_project_url}`
- }
- ccPairNameBuilder={(values) =>
- extractJiraProject(values.jira_project_url)
- }
- credentialId={jiraCredential.id}
- source="jira"
- inputType="poll"
- formBody={
- <>
-
-
- }
- formBodyBuilder={(values) => {
- return (
- <>
-
- {TextArrayFieldBuilder({
- name: "comment_email_blacklist",
- label: "Disable comments from users:",
- subtext: `
- This is generally useful to ignore certain bots. Add user emails which comments should NOT be indexed.`,
- })(values)}
-
- );
- }}
- validationSchema={Yup.object().shape({
- jira_project_url: Yup.string().required(
- "Please enter any link to your jira project e.g. https://danswer.atlassian.net/jira/software/projects/DAN/boards/1"
- ),
- comment_email_blacklist: Yup.array()
- .of(Yup.string().required("Emails names must be strings"))
- .required(),
- })}
- initialValues={{
- jira_project_url: "",
- comment_email_blacklist: [],
- }}
- refreshFreq={10 * 60} // 10 minutes
- />
-
-
- ) : (
- <>
-
- Please provide your access token in Step 1 first! Once done with
- that, you can then specify which Jira projects you want to make
- searchable.
-
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Jira" />
-
-
-
- Existing API Key:
-
- {linearCredential.credential_json?.linear_api_key}
-
-
-
-
- ) : (
- <>
-
- To use the Linear connector, first follow the guide{" "}
-
- here
- {" "}
- to generate an API Key.
-
-
-
- formBody={
- <>
-
-
- }
- validationSchema={Yup.object().shape({
- linear_api_key: Yup.string().required(
- "Please enter your Linear API Key!"
- ),
- })}
- initialValues={{
- linear_api_key: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Start indexing
-
- {linearCredential ? (
- <>
- {linearConnectorIndexingStatuses.length > 0 ? (
- <>
-
- We pull the latest issues and comments every{" "}
- 10 minutes.
-
-
-
- connectorIndexingStatuses={linearConnectorIndexingStatuses}
- liveCredential={linearCredential}
- getCredential={(credential) => {
- return (
-
-
{credential.credential_json.linear_api_key}
-
Create Connector
-
- Press connect below to start the connection Linear. We pull the
- latest issues and comments every 10{" "}
- minutes.
-
-
- nameBuilder={() => "LinearConnector"}
- ccPairNameBuilder={() => "Linear"}
- source="linear"
- inputType="poll"
- formBody={<>}
- validationSchema={Yup.object().shape({})}
- initialValues={{}}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={linearCredential.id}
- />
-
- )}
-
- ) : (
- <>
-
- Please provide your access token in Step 1 first! Once done with
- that, you can then start indexing Linear.
-
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Linear" />
-
-
-
- This connector allows you to sync your Loopio Library Entries into
- Danswer
-
-
-
- Step 1: Provide your API Access info
-
-
- {loopioCredential ? (
- <>
-
-
Existing App Key Secret:
-
- {loopioCredential.credential_json?.loopio_client_token}
-
-
-
-
- formBody={
- <>
-
-
-
-
- }
- validationSchema={Yup.object().shape({
- loopio_subdomain: Yup.string().required(
- "Please enter your Loopio Account subdomain"
- ),
- loopio_client_id: Yup.string().required(
- "Please enter your Loopio App Key ID"
- ),
- loopio_client_token: Yup.string().required(
- "Please enter your Loopio App Key Secret"
- ),
- })}
- initialValues={{
- loopio_subdomain: "",
- loopio_client_id: "",
- loopio_client_token: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Which Stack do you want to make searchable?
-
-
- Leave this blank if you want to index all stacks.
-
-
- {loopioConnectorIndexingStatuses.length > 0 && (
- <>
-
- We pull the latest library entries every 24 hours.
-
-
-
- connectorIndexingStatuses={loopioConnectorIndexingStatuses}
- liveCredential={loopioCredential}
- getCredential={(credential) =>
- credential.credential_json.loopio_client_id
- }
- specialColumns={[
- {
- header: "Stack",
- key: "loopio_stack_name",
- getValue: (ccPairStatus) =>
- ccPairStatus.connector.connector_specific_config
- .loopio_stack_name || "All stacks",
- },
- ]}
- includeName={true}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- onCredentialLink={async (connectorId) => {
- if (loopioCredential) {
- await linkCredential(connectorId, loopioCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- />
-
-
- )}
-
- {loopioCredential ? (
- <>
-
-
Create a new Loopio Connector
-
- nameBuilder={(values) =>
- values?.loopio_stack_name
- ? `LoopioConnector-${values.loopio_stack_name}-Stack`
- : `LoopioConnector-AllStacks`
- }
- source="loopio"
- inputType="poll"
- formBody={
- <>
-
-
- }
- validationSchema={Yup.object().shape({
- loopio_stack_name: Yup.string(),
- })}
- initialValues={{
- loopio_stack_name: "",
- }}
- refreshFreq={60 * 60 * 24} // 24 hours
- credentialId={loopioCredential.id}
- />
-
-
- ) : (
-
- Please provide your API Access Info in Step 1 first! Once done with
- that, you can start indexing your Loopio library.
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
- );
-}
diff --git a/web/src/app/admin/connectors/mediawiki/page.tsx b/web/src/app/admin/connectors/mediawiki/page.tsx
deleted file mode 100644
index e0c17a6e72d..00000000000
--- a/web/src/app/admin/connectors/mediawiki/page.tsx
+++ /dev/null
@@ -1,219 +0,0 @@
-"use client";
-
-import * as Yup from "yup";
-import { MediaWikiIcon, TrashIcon } from "@/components/icons/icons";
-import {
- TextArrayField,
- TextArrayFieldBuilder,
- TextFormField,
-} from "@/components/admin/connectors/Field";
-import { HealthCheckBanner } from "@/components/health/healthcheck";
-import { CredentialForm } from "@/components/admin/connectors/CredentialForm";
-import {
- MediaWikiCredentialJson,
- MediaWikiConfig,
- ConnectorIndexingStatus,
- Credential,
-} from "@/lib/types";
-import useSWR, { useSWRConfig } from "swr";
-import { errorHandlingFetcher } from "@/lib/fetcher";
-import { ErrorCallout } from "@/components/ErrorCallout";
-import { LoadingAnimation } from "@/components/Loading";
-import { adminDeleteCredential, linkCredential } from "@/lib/credential";
-import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
-import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable";
-import { usePopup } from "@/components/admin/connectors/Popup";
-import { usePublicCredentials } from "@/lib/hooks";
-import { AdminPageTitle } from "@/components/admin/Title";
-import { Card, Text, Title } from "@tremor/react";
-
-const Main = () => {
- const { popup, setPopup } = usePopup();
-
- const { mutate } = useSWRConfig();
- const {
- data: connectorIndexingStatuses,
- isLoading: isConnectorIndexingStatusesLoading,
- error: connectorIndexingStatusesError,
- } = useSWR[]>(
- "/api/manage/admin/connector/indexing-status",
- errorHandlingFetcher
- );
- const {
- data: credentialsData,
- isLoading: isCredentialsLoading,
- error: credentialsError,
- refreshCredentials,
- } = usePublicCredentials();
-
- if (
- (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) ||
- (!credentialsData && isCredentialsLoading)
- ) {
- return ;
- }
-
- if (connectorIndexingStatusesError || !connectorIndexingStatuses) {
- return (
-
- );
- }
-
- if (credentialsError || !credentialsData) {
- return (
-
- );
- }
-
- const mediawikiConnectorIndexingStatuses: ConnectorIndexingStatus<
- MediaWikiConfig,
- MediaWikiCredentialJson
- >[] = connectorIndexingStatuses.filter(
- (connectorIndexingStatus) =>
- connectorIndexingStatus.connector.source === "mediawiki"
- );
- const mediawikiCredential: Credential | undefined =
- credentialsData.find((credential) => true);
-
- return (
- <>
- {popup}
- {mediawikiConnectorIndexingStatuses.length > 0 && (
- <>
-
- MediaWiki indexing status
-
-
- The latest page, chapter, book and shelf changes are fetched every
- 10 minutes.
-
-
-
- connectorIndexingStatuses={mediawikiConnectorIndexingStatuses}
- liveCredential={mediawikiCredential}
- getCredential={(credential) => {
- return ;
- }}
- onCredentialLink={async (connectorId) => {
- if (mediawikiCredential) {
- await linkCredential(connectorId, mediawikiCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- />
- Create Connection
-
- Press connect below to start the connection to your MediaWiki
- instance.
-
-
- nameBuilder={(values) =>
- `MediaWikiConnector-${values.connector_name}`
- }
- ccPairNameBuilder={(values) =>
- `MediaWikiConnector-${values.connector_name}`
- }
- source="mediawiki"
- inputType="poll"
- formBodyBuilder={(values) => (
-
-
-
-
- {TextArrayFieldBuilder({
- name: "pages",
- label: "Pages to index:",
- subtext:
- "Specify 0 or more names of pages to index. Only specify the name of the page, not its url.",
- })(values)}
- {TextArrayFieldBuilder({
- name: "categories",
- label: "Categories to index:",
- subtext:
- "Specify 0 or more names of categories to index. For most MediaWiki sites, these are pages" +
- " with a name of the form 'Category: XYZ', that are lists of other pages/categories. Only" +
- " specify the name of the category, not its url.",
- })(values)}
-
-
- )}
- validationSchema={Yup.object().shape({
- connector_name: Yup.string().required(
- "Please enter a name for your MediaWiki connector."
- ),
- hostname: Yup.string().required(
- "Please enter the base URL for your MediaWiki site"
- ),
- language_code: Yup.string().default("en"),
- categories: Yup.array().of(
- Yup.string().required(
- "Please enter categories to index from your MediaWiki site"
- )
- ),
- pages: Yup.array().of(
- Yup.string().required(
- "Please enter pages to index from your MediaWiki site"
- )
- ),
- recurse_depth: Yup.number().required(
- "Please enter the recursion depth for your MediaWiki site."
- ),
- })}
- initialValues={{
- connector_name: "",
- hostname: "",
- language_code: "en",
- categories: [],
- pages: [],
- recurse_depth: 0,
- }}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={mediawikiCredential.id}
- />
-
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="MediaWiki" />
-
-
-
-
Existing Integration Token:
-
- {notionCredential.credential_json?.notion_integration_token}
-
-
-
-
- connectorIndexingStatuses={notionConnectorIndexingStatuses}
- specialColumns={[
- {
- header: "Root Page ID",
- key: "root_page_id",
- getValue: (ccPairStatus) =>
- ccPairStatus.connector.connector_specific_config
- .root_page_id || "-",
- },
- ]}
- liveCredential={notionCredential}
- getCredential={(credential) => {
- return (
-
-
{credential.credential_json.notion_integration_token}
-
Create New Connection
-
- Press connect below to start the connection to Notion.
-
-
- nameBuilder={(values) =>
- values.root_page_id
- ? `NotionConnector-${values.root_page_id}`
- : "NotionConnector"
- }
- ccPairNameBuilder={(values) =>
- values.root_page_id ? `Notion-${values.root_page_id}` : "Notion"
- }
- source="notion"
- inputType="poll"
- formBody={
- <>
-
-
- }
- validationSchema={Yup.object().shape({
- root_page_id: Yup.string(),
- })}
- initialValues={{
- root_page_id: "",
- }}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={notionCredential.id}
- />
-
-
- )}
-
- {!notionCredential && (
- <>
-
- Please provide your integration details in Step 1 first! Once done
- with that, you'll be able to start the connection then see
- indexing status.
-
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Notion" />
-
-
-
-
Existing OCI Access Key ID:
-
- {ociCredential.credential_json.access_key_id}
-
- {", "}
-
Namespace:
-
- {ociCredential.credential_json.namespace}
-
{" "}
-
-
- -
- Provide your OCI Access Key ID, Secret Access Key, Namespace,
- and Region for authentication.
-
- -
- These credentials will be used to access your OCI buckets.
-
-
-
-
-
- formBody={
- <>
-
-
-
-
-
- }
- validationSchema={Yup.object().shape({
- access_key_id: Yup.string().required(
- "OCI Access Key ID is required"
- ),
- secret_access_key: Yup.string().required(
- "OCI Secret Access Key is required"
- ),
- namespace: Yup.string().required("Namespace is required"),
- region: Yup.string().required("Region is required"),
- })}
- initialValues={{
- access_key_id: "",
- secret_access_key: "",
- namespace: "",
- region: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Which OCI bucket do you want to make searchable?
-
-
- {ociConnectorIndexingStatuses.length > 0 && (
- <>
-
- OCI indexing status
-
-
- The latest changes are fetched every 10 minutes.
-
-
-
- includeName={true}
- connectorIndexingStatuses={ociConnectorIndexingStatuses}
- liveCredential={ociCredential}
- getCredential={(credential) => {
- return ;
- }}
- onCredentialLink={async (connectorId) => {
- if (ociCredential) {
- await linkCredential(connectorId, ociCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- />
- Create Connection
-
- Press connect below to start the connection to your OCI bucket.
-
-
- nameBuilder={(values) => `OCIConnector-${values.bucket_name}`}
- ccPairNameBuilder={(values) =>
- `OCIConnector-${values.bucket_name}`
- }
- source="oci_storage"
- inputType="poll"
- formBodyBuilder={(values) => (
-
-
-
-
- )}
- validationSchema={Yup.object().shape({
- bucket_type: Yup.string()
- .oneOf(["oci_storage"])
- .required("Bucket type must be oci_storage"),
- bucket_name: Yup.string().required(
- "Please enter the name of the OCI bucket to index, e.g. my-test-bucket"
- ),
- prefix: Yup.string().default(""),
- })}
- initialValues={{
- bucket_type: "oci_storage",
- bucket_name: "",
- prefix: "",
- }}
- refreshFreq={60 * 60 * 24} // 1 day
- credentialId={ociCredential.id}
- />
-
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
}
- title="Oracle Cloud Infrastructure"
- />
-
-
- Existing Access Token:
-
- {
- productboardCredential.credential_json
- ?.productboard_access_token
- }
-
-
-
-
- ) : (
- <>
-
- To use the Productboard connector, first follow the guide{" "}
-
- here
- {" "}
- to generate an Access Token.
-
-
-
- formBody={
- <>
-
-
- }
- validationSchema={Yup.object().shape({
- productboard_access_token: Yup.string().required(
- "Please enter your Productboard access token"
- ),
- })}
- initialValues={{
- productboard_access_token: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Start indexing!
-
- {productboardCredential ? (
- !productboardConnectorIndexingStatuses.length ? (
- <>
-
- Click the button below to start indexing! We will pull the latest
- features, components, and products from Productboard every{" "}
- 10 minutes.
-
-
-
- nameBuilder={() => "ProductboardConnector"}
- ccPairNameBuilder={() => "Productboard"}
- source="productboard"
- inputType="poll"
- formBody={null}
- validationSchema={Yup.object().shape({})}
- initialValues={{}}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={productboardCredential.id}
- />
-
-
- {credential.credential_json.productboard_access_token}
-
-
-
-
-
-
-
}
- title="Productboard"
- />
-
-
-
-
Existing R2 Access Key ID:
-
- {r2Credential.credential_json.r2_access_key_id}
-
- {", "}
-
Account ID:
-
- {r2Credential.credential_json.account_id}
-
{" "}
-
-
- -
- Provide your R2 Access Key ID, Secret Access Key, and Account ID
- for authentication.
-
- - These credentials will be used to access your R2 buckets.
-
-
-
-
- formBody={
- <>
-
-
-
-
- }
- validationSchema={Yup.object().shape({
- r2_access_key_id: Yup.string().required(
- "R2 Access Key ID is required"
- ),
- r2_secret_access_key: Yup.string().required(
- "R2 Secret Access Key is required"
- ),
- account_id: Yup.string().required("Account ID is required"),
- })}
- initialValues={{
- r2_access_key_id: "",
- r2_secret_access_key: "",
- account_id: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Which R2 bucket do you want to make searchable?
-
-
- {r2ConnectorIndexingStatuses.length > 0 && (
- <>
-
- R2 indexing status
-
-
- The latest changes are fetched every 10 minutes.
-
-
-
- includeName={true}
- connectorIndexingStatuses={r2ConnectorIndexingStatuses}
- liveCredential={r2Credential}
- getCredential={(credential) => {
- return ;
- }}
- onCredentialLink={async (connectorId) => {
- if (r2Credential) {
- await linkCredential(connectorId, r2Credential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- />
- Create Connection
-
- Press connect below to start the connection to your R2 bucket.
-
-
- nameBuilder={(values) => `R2Connector-${values.bucket_name}`}
- ccPairNameBuilder={(values) =>
- `R2Connector-${values.bucket_name}`
- }
- source="r2"
- inputType="poll"
- formBodyBuilder={(values) => (
-
-
-
-
- )}
- validationSchema={Yup.object().shape({
- bucket_type: Yup.string()
- .oneOf(["r2"])
- .required("Bucket type must be r2"),
- bucket_name: Yup.string().required(
- "Please enter the name of the r2 bucket to index, e.g. my-test-bucket"
- ),
- prefix: Yup.string().default(""),
- })}
- initialValues={{
- bucket_type: "r2",
- bucket_name: "",
- prefix: "",
- }}
- refreshFreq={60 * 60 * 24} // 1 day
- credentialId={r2Credential.id}
- />
-
-
- )}
-
- );
-};
-
-export default function Page() {
- const [selectedStorage, setSelectedStorage] = useState("s3");
-
- return (
-
-
-
-
-
} title="R2 Storage" />
-
-
- Existing Request Tracker username:
-
- {requestTrackerCredential.credential_json.requesttracker_username}
-
-
-
-
- ) : (
- <>
-
- To use the Request Tracker connector, provide a Request Tracker
- username, password, and base url.
-
-
- This connector currently supports{" "}
-
- Request Tracker REST API 1.0
-
- ,{" "}
- not the latest REST API 2.0 introduced in Request Tracker 5.0
- .
-
-
-
- formBody={
- <>
-
-
-
-
- }
- validationSchema={Yup.object().shape({
- requesttracker_username: Yup.string().required(
- "Please enter your Request Tracker username"
- ),
- requesttracker_password: Yup.string().required(
- "Please enter your Request Tracker password"
- ),
- requesttracker_base_url: Yup.string()
- .url()
- .required(
- "Please enter the base url of your RT installation"
- ),
- })}
- initialValues={{
- requesttracker_username: "",
- requesttracker_password: "",
- requesttracker_base_url: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Manage Request Tracker Connector
-
-
- {requestTrackerConnectorIndexingStatuses.length > 0 && (
- <>
-
- We index the most recently updated tickets from each Request Tracker
- instance listed below regularly.
-
-
- The initial poll at this time retrieves tickets updated in the past
- hour. All subsequent polls execute every ten minutes. This should be
- configurable in the future.
-
-
-
- connectorIndexingStatuses={
- requestTrackerConnectorIndexingStatuses
- }
- liveCredential={requestTrackerCredential}
- getCredential={(credential) =>
- credential.credential_json.requesttracker_base_url
- }
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- onCredentialLink={async (connectorId) => {
- if (requestTrackerCredential) {
- await linkCredential(
- connectorId,
- requestTrackerCredential.id
- );
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- />
-
-
- )}
-
- {requestTrackerCredential &&
- requestTrackerConnectorIndexingStatuses.length === 0 ? (
-
-
- nameBuilder={(values) =>
- `RequestTracker-${requestTrackerCredential.credential_json.requesttracker_base_url}`
- }
- ccPairNameBuilder={(values) =>
- `Request Tracker ${requestTrackerCredential.credential_json.requesttracker_base_url}`
- }
- source="requesttracker"
- inputType="poll"
- validationSchema={Yup.object().shape({})}
- formBody={<>}
- initialValues={{}}
- credentialId={requestTrackerCredential.id}
- refreshFreq={10 * 60} // 10 minutes
- />
-
- ) : (
- <>
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
}
- title="Request Tracker"
- />
-
-
-
-
Existing AWS Access Key ID:
-
- {s3Credential.credential_json.aws_access_key_id}
-
-
-
- -
- If AWS Access Key ID and AWS Secret Access Key are provided,
- they will be used for authenticating the connector.
-
- - Otherwise, the Profile Name will be used (if provided).
- -
- If no credentials are provided, then the connector will try to
- authenticate with any default AWS credentials available.
-
-
-
-
-
- formBody={
- <>
-
-
-
- }
- validationSchema={Yup.object().shape({
- aws_access_key_id: Yup.string().default(""),
- aws_secret_access_key: Yup.string().default(""),
- })}
- initialValues={{
- aws_access_key_id: "",
- aws_secret_access_key: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Which S3 bucket do you want to make searchable?
-
-
- {s3ConnectorIndexingStatuses.length > 0 && (
- <>
-
- S3 indexing status
-
-
- The latest changes are fetched every 10 minutes.
-
-
-
- includeName={true}
- connectorIndexingStatuses={s3ConnectorIndexingStatuses}
- liveCredential={s3Credential}
- getCredential={(credential) => {
- return ;
- }}
- onCredentialLink={async (connectorId) => {
- if (s3Credential) {
- await linkCredential(connectorId, s3Credential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- />
- Create Connection
-
- Press connect below to start the connection to your S3 bucket.
-
-
- nameBuilder={(values) => `S3Connector-${values.bucket_name}`}
- ccPairNameBuilder={(values) =>
- `S3Connector-${values.bucket_name}`
- }
- source="s3"
- inputType="poll"
- formBodyBuilder={(values) => (
-
-
-
-
- )}
- validationSchema={Yup.object().shape({
- bucket_type: Yup.string()
- .oneOf(["s3"])
- .required("Bucket type must be s3"),
- bucket_name: Yup.string().required(
- "Please enter the name of the s3 bucket to index, e.g. my-test-bucket"
- ),
- prefix: Yup.string().default(""),
- })}
- initialValues={{
- bucket_type: "s3",
- bucket_name: "",
- prefix: "",
- }}
- refreshFreq={60 * 60 * 24} // 1 day
- credentialId={s3Credential.id}
- />
-
-
- )}
-
- );
-};
-
-export default function Page() {
- const [selectedStorage, setSelectedStorage] = useState("s3");
-
- return (
-
-
-
-
-
} title="S3 Storage" />
-
-
-
Failed to load connectors
;
- }
-
- if (isCredentialsError || !credentialsData) {
- return Failed to load credentials
;
- }
-
- const SalesforceConnectorIndexingStatuses: ConnectorIndexingStatus<
- SalesforceConfig,
- SalesforceCredentialJson
- >[] = connectorIndexingStatuses.filter(
- (connectorIndexingStatus) =>
- connectorIndexingStatus.connector.source === "salesforce"
- );
-
- const SalesforceCredential: Credential | undefined =
- credentialsData.find(
- (credential) => credential.credential_json?.sf_username
- );
-
- return (
- <>
-
- The Salesforce connector allows you to index and search through your
- Salesforce data. Once setup, all indicated Salesforce data will be will
- be queryable within Danswer.
-
-
-
- Step 1: Provide Salesforce credentials
-
- {SalesforceCredential ? (
- <>
-
- Existing SalesForce Username:
-
- {SalesforceCredential.credential_json.sf_username}
-
-
-
-
- ) : (
- <>
-
- As a first step, please provide the Salesforce admin account's
- username, password, and Salesforce security token. You can follow
- the guide{" "}
-
- here
- {" "}
- to create get your Salesforce Security Token.
-
-
-
- formBody={
- <>
-
-
-
-
- }
- validationSchema={Yup.object().shape({
- sf_username: Yup.string().required(
- "Please enter your Salesforce username"
- ),
- sf_password: Yup.string().required(
- "Please enter your Salesforce password"
- ),
- sf_security_token: Yup.string().required(
- "Please enter your Salesforce security token"
- ),
- })}
- initialValues={{
- sf_username: "",
- sf_password: "",
- sf_security_token: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Manage Salesforce Connector
-
-
- {SalesforceConnectorIndexingStatuses.length > 0 && (
- <>
-
- The latest state of your Salesforce objects are fetched every 10
- minutes.
-
-
-
- connectorIndexingStatuses={SalesforceConnectorIndexingStatuses}
- liveCredential={SalesforceCredential}
- getCredential={(credential) =>
- credential.credential_json.sf_security_token
- }
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- onCredentialLink={async (connectorId) => {
- if (SalesforceCredential) {
- await linkCredential(connectorId, SalesforceCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- specialColumns={[
- {
- header: "Connectors",
- key: "connectors",
- getValue: (ccPairStatus) => {
- const connectorConfig =
- ccPairStatus.connector.connector_specific_config;
- return `${connectorConfig.requested_objects}`;
- },
- },
- ]}
- includeName
- />
-
-
- )}
-
- {SalesforceCredential ? (
-
-
- nameBuilder={(values) =>
- values.requested_objects && values.requested_objects.length > 0
- ? `Salesforce-${values.requested_objects.join("-")}`
- : "Salesforce"
- }
- ccPairNameBuilder={(values) =>
- values.requested_objects && values.requested_objects.length > 0
- ? `Salesforce-${values.requested_objects.join("-")}`
- : "Salesforce"
- }
- source="salesforce"
- inputType="poll"
- // formBody={<>}
- formBodyBuilder={TextArrayFieldBuilder({
- name: "requested_objects",
- label: "Specify Salesforce objects to organize by:",
- subtext: (
- <>
-
- Specify the Salesforce object types you want us to index.{" "}
-
-
- Click
-
- {" "}
- here{" "}
-
- for an example of how Danswer uses the objects.
-
- If unsure, don't specify any objects and Danswer will
- default to indexing by 'Account'.
-
-
- Hint: Use the singular form of the object name (e.g.,
- 'Opportunity' instead of 'Opportunities').
-
- ),
- })}
- validationSchema={Yup.object().shape({
- requested_objects: Yup.array()
- .of(
- Yup.string().required(
- "Salesforce object names must be strings"
- )
- )
- .required(),
- })}
- initialValues={{
- requested_objects: [],
- }}
- credentialId={SalesforceCredential.id}
- refreshFreq={10 * 60} // 10 minutes
- />
-
- ) : (
-
- Please provide all Salesforce info in Step 1 first! Once you're
- done with that, you can then specify which Salesforce objects you want
- to make searchable.
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Salesforce" />
-
-
-
- Existing Azure AD Client ID:
-
- {sharepointCredential.credential_json.sp_client_id}
-
-
-
-
- ) : (
- <>
-
- As a first step, please provide Application (client) ID, Directory
- (tenant) ID, and Client Secret. You can follow the guide{" "}
-
- here
- {" "}
- to create an Azure AD application and obtain these values.
-
-
-
- formBody={
- <>
-
-
-
-
- }
- validationSchema={Yup.object().shape({
- sp_client_id: Yup.string().required(
- "Please enter your Application (client) ID"
- ),
- sp_directory_id: Yup.string().required(
- "Please enter your Directory (tenant) ID"
- ),
- sp_client_secret: Yup.string().required(
- "Please enter your Client Secret"
- ),
- })}
- initialValues={{
- sp_client_id: "",
- sp_directory_id: "",
- sp_client_secret: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Manage Sharepoint Connector
-
-
- {sharepointConnectorIndexingStatuses.length > 0 && (
- <>
-
- The latest state of your Word documents, Excel files, PowerPoint
- presentations, OneNote notebooks, PDFs, and uploaded files are
- fetched every 10 minutes.
-
-
-
- connectorIndexingStatuses={sharepointConnectorIndexingStatuses}
- liveCredential={sharepointCredential}
- getCredential={(credential) =>
- credential.credential_json.sp_directory_id
- }
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- onCredentialLink={async (connectorId) => {
- if (sharepointCredential) {
- await linkCredential(connectorId, sharepointCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- specialColumns={[
- {
- header: "Connectors",
- key: "connectors",
- getValue: (ccPairStatus) => {
- const connectorConfig =
- ccPairStatus.connector.connector_specific_config;
- return `${connectorConfig.sites}`;
- },
- },
- ]}
- includeName
- />
-
-
- )}
-
- {sharepointCredential ? (
-
-
- nameBuilder={(values) =>
- values.sites && values.sites.length > 0
- ? `Sharepoint-${values.sites.join("-")}`
- : "Sharepoint"
- }
- ccPairNameBuilder={(values) =>
- values.sites && values.sites.length > 0
- ? `Sharepoint-${values.sites.join("-")}`
- : "Sharepoint"
- }
- source="sharepoint"
- inputType="poll"
- // formBody={<>}
- formBodyBuilder={TextArrayFieldBuilder({
- name: "sites",
- label: "Sites:",
- subtext: (
- <>
-
-
- -
- • If no sites are specified, all sites in your
- organization will be indexed (Sites.Read.All permission
- required).
-
- -
- • Specifying
- 'https://danswerai.sharepoint.com/sites/support'
- for example will only index documents within this site.
-
- -
- • Specifying
- 'https://danswerai.sharepoint.com/sites/support/subfolder'
- for example will only index documents within this folder.
-
-
-
- ),
- })}
- validationSchema={Yup.object().shape({
- sites: Yup.array()
- .of(Yup.string().required("Site names must be strings"))
- .required(),
- })}
- initialValues={{
- sites: [],
- }}
- credentialId={sharepointCredential.id}
- refreshFreq={10 * 60} // 10 minutes
- />
-
- ) : (
-
- Please provide all Azure info in Step 1 first! Once you're done
- with that, you can then specify which Sharepoint sites you want to
- make searchable.
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Sharepoint" />
-
-
-
- Existing Slab Bot Token:
-
- {slabCredential.credential_json?.slab_bot_token}
-
-
-
-
- ) : (
- <>
-
- To use the Slab connector, first follow the guide{" "}
-
- here
- {" "}
- to generate a Slab Bot Token.
-
-
-
- formBody={
- <>
-
-
- }
- validationSchema={Yup.object().shape({
- slab_bot_token: Yup.string().required(
- "Please enter your Slab Bot Token"
- ),
- })}
- initialValues={{
- slab_bot_token: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: What's the base URL for your Slab team?
-
- {slabCredential ? (
- <>
- {slabConnectorIndexingStatuses.length > 0 ? (
- <>
-
- We are pulling the latest documents from{" "}
-
- {
- slabConnectorIndexingStatuses[0].connector
- .connector_specific_config.base_url
- }
- {" "}
- every 10 minutes.
-
-
- connectorIndexingStatuses={slabConnectorIndexingStatuses}
- liveCredential={slabCredential}
- getCredential={(credential) => {
- return (
-
-
{credential.credential_json.slab_bot_token}
-
Add a New Space
-
- nameBuilder={(values) => `SlabConnector-${values.base_url}`}
- ccPairNameBuilder={(values) => values.base_url}
- source="slab"
- inputType="poll"
- formBody={
- <>
-
-
- }
- validationSchema={Yup.object().shape({
- base_url: Yup.string().required(
- "Please enter the base URL for your team e.g. https://danswer.slab.com/"
- ),
- })}
- initialValues={{
- base_url: "",
- }}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={slabCredential.id}
- />
-
-
- )}
-
- ) : (
- <>
-
- Please provide your access token in Step 1 first! Once done with
- that, you can then specify the URL for your Slab team and get
- started with indexing.
-
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Slab" />
-
-
-
- Existing Slack Bot Token:
-
- {slackCredential.credential_json.slack_bot_token}
-
-
-
-
- ) : (
- <>
-
- To use the Slack connector, you must first provide a Slack bot token
- corresponding to the Slack App set up in your workspace. For more
- details on setting up the Danswer Slack App, see the{" "}
-
- docs
-
- .
-
-
-
- formBody={
- <>
-
-
- }
- validationSchema={Yup.object().shape({
- slack_bot_token: Yup.string().required(
- "Please enter your Slack bot token"
- ),
- })}
- initialValues={{
- slack_bot_token: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Which channels do you want to make searchable?
-
-
- {slackConnectorIndexingStatuses.length > 0 && (
- <>
-
- We pull the latest messages from each workspace listed below every{" "}
- 10 minutes.
-
-
-
- connectorIndexingStatuses={slackConnectorIndexingStatuses}
- liveCredential={slackCredential}
- getCredential={(credential) =>
- credential.credential_json.slack_bot_token
- }
- specialColumns={[
- {
- header: "Workspace",
- key: "workspace",
- getValue: (ccPairStatus) =>
- ccPairStatus.connector.connector_specific_config.workspace,
- },
- {
- header: "Channels",
- key: "channels",
- getValue: (ccPairStatus) => {
- const connectorConfig =
- ccPairStatus.connector.connector_specific_config;
- return connectorConfig.channels &&
- connectorConfig.channels.length > 0
- ? connectorConfig.channels.join(", ")
- : "";
- },
- },
- ]}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- onCredentialLink={async (connectorId) => {
- if (slackCredential) {
- await linkCredential(connectorId, slackCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- />
-
-
-
- )}
-
- {slackCredential ? (
-
- Connect to a New Workspace
-
- nameBuilder={(values) =>
- values.channels
- ? `SlackConnector-${values.workspace}-${values.channels.join(
- "_"
- )}`
- : `SlackConnector-${values.workspace}`
- }
- source="slack"
- inputType="poll"
- formBody={
- <>
-
-
- }
- formBodyBuilder={(values) => {
- return (
- <>
-
- {TextArrayFieldBuilder({
- name: "channels",
- label: "Channels:",
- subtext: `
- Specify 0 or more channels to index. For example, specifying the channel
- "support" will cause us to only index all content within the "#support" channel.
- If no channels are specified, all channels in your workspace will be indexed.`,
- })(values)}
-
- If enabled, we will treat the "channels"
- specified above as regular expressions. A channel's
- messages will be pulled in by the connector if the name
- of the channel fully matches any of the specified
- regular expressions.
-
- For example, specifying .*-support.* as a
- "channel" will cause the connector to include
- any channels with "-support" in the name.
-
- }
- />
-
- );
- }}
- validationSchema={Yup.object().shape({
- workspace: Yup.string().required(
- "Please enter the workspace to index"
- ),
- channels: Yup.array()
- .of(Yup.string().required("Channel names must be strings"))
- .required(),
- channel_regex_enabled: Yup.boolean().required(),
- })}
- initialValues={{
- workspace: "",
- channels: [],
- channel_regex_enabled: false,
- }}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={slackCredential.id}
- />
-
- ) : (
-
- Please provide your slack bot token in Step 1 first! Once done with
- that, you can then specify which Slack channels you want to make
- searchable.
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Slack" />
-
-
-
- );
-}
diff --git a/web/src/app/admin/connectors/teams/page.tsx b/web/src/app/admin/connectors/teams/page.tsx
deleted file mode 100644
index 530d430abb6..00000000000
--- a/web/src/app/admin/connectors/teams/page.tsx
+++ /dev/null
@@ -1,275 +0,0 @@
-"use client";
-
-import * as Yup from "yup";
-import { TrashIcon, TeamsIcon } from "@/components/icons/icons"; // Make sure you have a Document360 icon
-import { errorHandlingFetcher } from "@/lib/fetcher";
-import useSWR, { useSWRConfig } from "swr";
-import { LoadingAnimation } from "@/components/Loading";
-import { HealthCheckBanner } from "@/components/health/healthcheck";
-import {
- TeamsConfig,
- TeamsCredentialJson,
- ConnectorIndexingStatus,
- Credential,
-} from "@/lib/types"; // Modify or create these types as required
-import { adminDeleteCredential, linkCredential } from "@/lib/credential";
-import { CredentialForm } from "@/components/admin/connectors/CredentialForm";
-import {
- TextFormField,
- TextArrayFieldBuilder,
-} from "@/components/admin/connectors/Field";
-import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable";
-import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
-import { usePublicCredentials } from "@/lib/hooks";
-import { AdminPageTitle } from "@/components/admin/Title";
-import { Card, Text, Title } from "@tremor/react";
-import { ErrorCallout } from "@/components/ErrorCallout";
-
-const MainSection = () => {
- const { mutate } = useSWRConfig();
- const {
- data: connectorIndexingStatuses,
- isLoading: isConnectorIndexingStatusesLoading,
- error: connectorIndexingStatusesError,
- } = useSWR
[]>(
- "/api/manage/admin/connector/indexing-status",
- errorHandlingFetcher
- );
-
- const {
- data: credentialsData,
- isLoading: isCredentialsLoading,
- error: credentialsError,
- refreshCredentials,
- } = usePublicCredentials();
-
- if (
- (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) ||
- (!credentialsData && isCredentialsLoading)
- ) {
- return ;
- }
-
- if (connectorIndexingStatusesError || !connectorIndexingStatuses) {
- return (
-
- );
- }
-
- if (credentialsError || !credentialsData) {
- return (
-
- );
- }
-
- const teamsConnectorIndexingStatuses: ConnectorIndexingStatus<
- TeamsConfig,
- TeamsCredentialJson
- >[] = connectorIndexingStatuses.filter(
- (connectorIndexingStatus) =>
- connectorIndexingStatus.connector.source === "teams"
- );
-
- const teamsCredential: Credential | undefined =
- credentialsData.find(
- (credential) => credential.credential_json?.teams_client_id
- );
-
- return (
- <>
-
- The Teams connector allows you to index and search through your Teams
- channels. Once setup, all messages from the channels contained in the
- specified teams will be queryable within Danswer.
-
-
-
- Step 1: Provide Teams credentials
-
- {teamsCredential ? (
- <>
-
- Existing Azure AD Client ID:
-
- {teamsCredential.credential_json.teams_client_id}
-
-
-
-
- ) : (
- <>
-
- As a first step, please provide Application (client) ID, Directory
- (tenant) ID, and Client Secret. You can follow the guide{" "}
-
- here
- {" "}
- to create an Azure AD application and obtain these values.
-
-
-
- formBody={
- <>
-
-
-
-
- }
- validationSchema={Yup.object().shape({
- teams_client_id: Yup.string().required(
- "Please enter your Application (client) ID"
- ),
- teams_directory_id: Yup.string().required(
- "Please enter your Directory (tenant) ID"
- ),
- teams_client_secret: Yup.string().required(
- "Please enter your Client Secret"
- ),
- })}
- initialValues={{
- teams_client_id: "",
- teams_directory_id: "",
- teams_client_secret: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Manage Teams Connector
-
-
- {teamsConnectorIndexingStatuses.length > 0 && (
- <>
-
- The latest messages from the specified teams are fetched every 10
- minutes.
-
-
-
- connectorIndexingStatuses={teamsConnectorIndexingStatuses}
- liveCredential={teamsCredential}
- getCredential={(credential) =>
- credential.credential_json.teams_directory_id
- }
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- onCredentialLink={async (connectorId) => {
- if (teamsCredential) {
- await linkCredential(connectorId, teamsCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- specialColumns={[
- {
- header: "Connectors",
- key: "connectors",
- getValue: (ccPairStatus) => {
- const connectorConfig =
- ccPairStatus.connector.connector_specific_config;
- return `${connectorConfig.teams}`;
- },
- },
- ]}
- includeName
- />
-
-
- )}
-
- {teamsCredential ? (
-
-
- nameBuilder={(values) =>
- values.teams && values.teams.length > 0
- ? `Teams-${values.teams.join("-")}`
- : "Teams"
- }
- ccPairNameBuilder={(values) =>
- values.teams && values.teams.length > 0
- ? `Teams-${values.teams.join("-")}`
- : "Teams"
- }
- source="teams"
- inputType="poll"
- // formBody={<>}
- formBodyBuilder={TextArrayFieldBuilder({
- name: "teams",
- label: "Teams:",
- subtext:
- "Specify 0 or more Teams to index. " +
- "For example, specifying the Team 'Support' for the 'danswerai' Org will cause " +
- "us to only index messages sent in channels belonging to the 'Support' Team. " +
- "If no Teams are specified, all Teams in your organization will be indexed.",
- })}
- validationSchema={Yup.object().shape({
- teams: Yup.array()
- .of(Yup.string().required("Team names must be strings"))
- .required(),
- })}
- initialValues={{
- teams: [],
- }}
- credentialId={teamsCredential.id}
- refreshFreq={10 * 60} // 10 minutes
- />
-
- ) : (
-
- Please provide all Azure info in Step 1 first! Once you're done
- with that, you can then specify which teams you want to make
- searchable.
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Teams" />
-
-
-
-
-
-
-
-
} title="Web" />
-
-
- Step 1: Specify which websites to index
-
-
- We re-fetch the latest state of the website once a day.
-
-
-
- nameBuilder={(values) => `WebConnector-${values.base_url}`}
- ccPairNameBuilder={(values) => values.base_url}
- // Since there is no "real" credential associated with a web connector
- // we create a dummy one here so that we can associate the CC Pair with a
- // user. This is needed since the user for a CC Pair is found via the credential
- // associated with it.
- shouldCreateEmptyCredentialForConnector={true}
- source="web"
- inputType="load_state"
- formBody={
- <>
-
-
-
-
-
- }
- validationSchema={Yup.object().shape({
- base_url: Yup.string().required(
- "Please enter the website URL to scrape e.g. https://docs.danswer.dev/"
- ),
- web_connector_type: Yup.string()
- .oneOf(["recursive", "single", "sitemap"])
- .optional(),
- })}
- initialValues={{
- base_url: "",
- web_connector_type: undefined,
- }}
- refreshFreq={60 * 60 * 24} // 1 day
- pruneFreq={0} // Don't prune
- />
-
-
-
- Already Indexed Websites
-
- {isConnectorIndexingStatusesLoading ? (
-
- ) : connectorIndexingStatusesError || !connectorIndexingStatuses ? (
-
Error loading indexing history
- ) : webIndexingStatuses.length > 0 ? (
-
- connectorIndexingStatuses={webIndexingStatuses}
- specialColumns={[
- {
- header: "Base URL",
- key: "base_url",
- getValue: (
- ccPairStatus: ConnectorIndexingStatus
- ) => {
- const connectorConfig =
- ccPairStatus.connector.connector_specific_config;
- return (
-
- );
- },
- },
- {
- header: "Scrape Method",
- key: "web_connector_type",
- getValue: (ccPairStatus) => {
- const connectorConfig =
- ccPairStatus.connector.connector_specific_config;
- return connectorConfig.web_connector_type
- ? SCRAPE_TYPE_TO_PRETTY_NAME[
- connectorConfig.web_connector_type
- ]
- : "Recursive";
- },
- },
- ]}
- onUpdate={() => mutate("/api/manage/admin/connector/indexing-status")}
- />
- ) : (
- No indexed websites found
- )}
-
-
- connectorIndexingStatuses={wikipediaConnectorIndexingStatuses}
- liveCredential={wikipediaCredential}
- getCredential={(credential) => {
- return ;
- }}
- onCredentialLink={async (connectorId) => {
- if (wikipediaCredential) {
- await linkCredential(connectorId, wikipediaCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- />
- Create Connection
-
- Press connect below to start the connection to your Wikipedia
- instance.
-
-
- nameBuilder={(values) =>
- `WikipediaConnector-${values.connector_name}`
- }
- ccPairNameBuilder={(values) =>
- `WikipediaConnector-${values.connector_name}`
- }
- source="wikipedia"
- inputType="poll"
- formBodyBuilder={(values) => (
-
-
-
- {TextArrayFieldBuilder({
- name: "pages",
- label: "Pages to index:",
- subtext:
- "Specify 0 or more names of pages to index. Only specify the name of the page, not its url.",
- })(values)}
- {TextArrayFieldBuilder({
- name: "categories",
- label: "Categories to index:",
- subtext:
- "Specify 0 or more names of categories to index. These are pages" +
- " with a name of the form 'Category: XYZ', that are lists of other pages/categories. Only" +
- " specify the name of the category, not its url.",
- })(values)}
-
-
- )}
- validationSchema={Yup.object().shape({
- connector_name: Yup.string().required(
- "Please enter a name for your Wikipedia connector."
- ),
- language_code: Yup.string().default("en"),
- categories: Yup.array().of(
- Yup.string().required(
- "Please enter categories to index from your Wikipedia site"
- )
- ),
- pages: Yup.array().of(
- Yup.string().required(
- "Please enter pages to index from your Wikipedia site"
- )
- ),
- recurse_depth: Yup.number().required(
- "Please enter the recursion depth for your Wikipedia site."
- ),
- })}
- initialValues={{
- connector_name: "",
- language_code: "en",
- categories: [],
- pages: [],
- recurse_depth: 0,
- }}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={wikipediaCredential.id}
- />
-
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Wikipedia" />
-
-
-
-
Existing API Token:
-
- {zendeskCredential.credential_json?.zendesk_token}
-
-
-
-
- connectorIndexingStatuses={zendeskConnectorIndexingStatuses}
- liveCredential={zendeskCredential}
- getCredential={(credential) => {
- return (
-
-
{credential.credential_json.zendesk_token}
-
Create Connection
-
- Press connect below to start the connection to your Zendesk
- instance.
-
-
- nameBuilder={(values) => `ZendeskConnector`}
- ccPairNameBuilder={(values) => `ZendeskConnector`}
- source="zendesk"
- inputType="poll"
- formBody={<>}
- validationSchema={Yup.object().shape({})}
- initialValues={{}}
- refreshFreq={10 * 60} // 10 minutes
- credentialId={zendeskCredential.id}
- />
-
-
- )}
-
- {!zendeskCredential && (
- <>
-
- Please provide your API details in Step 1 first! Once done with
- that, you'll be able to start the connection then see indexing
- status.
-
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Zendesk" />
-
-
-
- Existing zuliprc file content:
-
- {zulipCredential.credential_json.zuliprc_content}
- {" "}
-
-
-
- ) : (
- <>
-
- To use the Zulip connector, you must first provide content of the
- zuliprc config file. For more details on setting up the Danswer
- Zulip connector, see the{" "}
-
- docs
-
- .
-
-
-
- formBody={
- <>
-
-
- }
- validationSchema={Yup.object().shape({
- zuliprc_content: Yup.string().required(
- "Please enter content of the zuliprc file"
- ),
- })}
- initialValues={{
- zuliprc_content: "",
- }}
- onSubmit={(isSuccess) => {
- if (isSuccess) {
- refreshCredentials();
- }
- }}
- />
-
-
- )}
-
-
- Step 2: Which workspaces do you want to make searchable?
-
-
- {zulipCredential ? (
- <>
- {zulipConnectorIndexingStatuses.length > 0 && (
- <>
-
- We pull the latest messages from each workspace listed below
- every 10 minutes.
-
-
-
- credential.credential_json.zuliprc_content
- }
- specialColumns={[
- {
- header: "Realm name",
- key: "realm_name",
- getValue: (ccPairStatus) =>
- ccPairStatus.connector.connector_specific_config
- .realm_name,
- },
- {
- header: "Realm url",
- key: "realm_url",
- getValue: (ccPairStatus) =>
- ccPairStatus.connector.connector_specific_config
- .realm_url,
- },
- ]}
- onUpdate={() =>
- mutate("/api/manage/admin/connector/indexing-status")
- }
- onCredentialLink={async (connectorId) => {
- if (zulipCredential) {
- await linkCredential(connectorId, zulipCredential.id);
- mutate("/api/manage/admin/connector/indexing-status");
- }
- }}
- />
-
-
-
- )}
-
-
- Connect to a New Realm
-
- nameBuilder={(values) => `ZulipConnector-${values.realm_name}`}
- ccPairNameBuilder={(values) => values.realm_name}
- source="zulip"
- inputType="poll"
- credentialId={zulipCredential.id}
- formBody={
- <>
-
-
-
- }
- validationSchema={Yup.object().shape({
- realm_name: Yup.string().required(
- "Please enter the realm name"
- ),
- realm_url: Yup.string().required("Please enter the realm url"),
- })}
- initialValues={{
- realm_name: "",
- realm_url: "",
- }}
- refreshFreq={10 * 60} // 10 minutes
- />
-
-
- ) : (
-
- Please provide your Zulip credentials in Step 1 first! Once done with
- that, you can then specify which Zulip realms you want to make
- searchable.
-
- )}
-
- );
-};
-
-export default function Page() {
- return (
-
-
-
-
-
-
} title="Zulip" />
-
-
-
-
initialValues={{
- name: existingDocumentSet ? existingDocumentSet.name : "",
- description: existingDocumentSet
- ? existingDocumentSet.description
- : "",
- cc_pair_ids: existingDocumentSet
- ? existingDocumentSet.cc_pair_descriptors.map(
- (ccPairDescriptor) => {
- return ccPairDescriptor.id;
- }
- )
- : ([] as number[]),
- is_public: existingDocumentSet ? existingDocumentSet.is_public : true,
- users: existingDocumentSet ? existingDocumentSet.users : [],
- groups: existingDocumentSet ? existingDocumentSet.groups : [],
+ name: existingDocumentSet?.name ?? "",
+ description: existingDocumentSet?.description ?? "",
+ cc_pair_ids:
+ existingDocumentSet?.cc_pair_descriptors.map(
+ (ccPairDescriptor) => ccPairDescriptor.id
+ ) ?? [],
+ is_public: existingDocumentSet?.is_public ?? true,
+ users: existingDocumentSet?.users ?? [],
+ groups: existingDocumentSet?.groups ?? [],
}}
validationSchema={Yup.object().shape({
name: Yup.string().required("Please enter a name for the set"),
@@ -74,6 +84,7 @@ export const DocumentSetCreationForm = ({
response = await updateDocumentSet({
id: existingDocumentSet.id,
...processedValues,
+ users: processedValues.users,
});
} else {
response = await createDocumentSet(processedValues);
@@ -98,129 +109,90 @@ export const DocumentSetCreationForm = ({
}
}}
>
- {({ isSubmitting, values }) => (
- 
diff --git a/backend/Dockerfile b/backend/Dockerfile
index 7f9daad94a3..17e0be8c239 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -68,7 +68,9 @@ RUN apt-get update && \
rm -f /usr/local/lib/python3.11/site-packages/tornado/test/test.key
# Pre-downloading models for setups with limited egress
-RUN python -c "from transformers import AutoTokenizer; AutoTokenizer.from_pretrained('intfloat/e5-base-v2')"
+RUN python -c "from tokenizers import Tokenizer; \
+Tokenizer.from_pretrained('nomic-ai/nomic-embed-text-v1')"
+
# Pre-downloading NLTK for setups with limited egress
RUN python -c "import nltk; \
diff --git a/backend/Dockerfile.model_server b/backend/Dockerfile.model_server
index 89f24e2ac26..f2fb1ca44d0 100644
--- a/backend/Dockerfile.model_server
+++ b/backend/Dockerfile.model_server
@@ -18,14 +18,22 @@ RUN apt-get remove -y --allow-remove-essential perl-base && \
apt-get autoremove -y
# Pre-downloading models for setups with limited egress
-RUN python -c "from transformers import AutoModel, AutoTokenizer, TFDistilBertForSequenceClassification; \
-from huggingface_hub import snapshot_download; \
-AutoTokenizer.from_pretrained('danswer/intent-model'); \
-AutoTokenizer.from_pretrained('intfloat/e5-base-v2'); \
+# Download tokenizers, distilbert for the Danswer model
+# Download model weights
+# Run Nomic to pull in the custom architecture and have it cached locally
+RUN python -c "from transformers import AutoTokenizer; \
+AutoTokenizer.from_pretrained('distilbert-base-uncased'); \
AutoTokenizer.from_pretrained('mixedbread-ai/mxbai-rerank-xsmall-v1'); \
-snapshot_download('danswer/intent-model'); \
-snapshot_download('intfloat/e5-base-v2'); \
-snapshot_download('mixedbread-ai/mxbai-rerank-xsmall-v1')"
+from huggingface_hub import snapshot_download; \
+snapshot_download(repo_id='danswer/hybrid-intent-token-classifier', revision='v1.0.3'); \
+snapshot_download('nomic-ai/nomic-embed-text-v1'); \
+snapshot_download('mixedbread-ai/mxbai-rerank-xsmall-v1'); \
+from sentence_transformers import SentenceTransformer; \
+SentenceTransformer(model_name_or_path='nomic-ai/nomic-embed-text-v1', trust_remote_code=True);"
+
+# In case the user has volumes mounted to /root/.cache/huggingface that they've downloaded while
+# running Danswer, don't overwrite it with the built in cache folder
+RUN mv /root/.cache/huggingface /root/.cache/temp_huggingface
WORKDIR /app
diff --git a/backend/alembic/env.py b/backend/alembic/env.py
index 8a944689de4..8c028202bfc 100644
--- a/backend/alembic/env.py
+++ b/backend/alembic/env.py
@@ -8,6 +8,7 @@
from sqlalchemy.engine import Connection
from sqlalchemy.ext.asyncio import create_async_engine
from celery.backends.database.session import ResultModelBase # type: ignore
+from sqlalchemy.schema import SchemaItem
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
@@ -29,6 +30,20 @@
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
+EXCLUDE_TABLES = {"kombu_queue", "kombu_message"}
+
+
+def include_object(
+ object: SchemaItem,
+ name: str,
+ type_: str,
+ reflected: bool,
+ compare_to: SchemaItem | None,
+) -> bool:
+ if type_ == "table" and name in EXCLUDE_TABLES:
+ return False
+ return True
+
def run_migrations_offline() -> None:
"""Run migrations in 'offline' mode.
@@ -55,7 +70,11 @@ def run_migrations_offline() -> None:
def do_run_migrations(connection: Connection) -> None:
- context.configure(connection=connection, target_metadata=target_metadata) # type: ignore
+ context.configure(
+ connection=connection,
+ target_metadata=target_metadata, # type: ignore
+ include_object=include_object,
+ ) # type: ignore
with context.begin_transaction():
context.run_migrations()
diff --git a/backend/alembic/versions/0568ccf46a6b_add_thread_specific_model_selection.py b/backend/alembic/versions/0568ccf46a6b_add_thread_specific_model_selection.py
index 2d8912d9bcf..d0b90da0232 100644
--- a/backend/alembic/versions/0568ccf46a6b_add_thread_specific_model_selection.py
+++ b/backend/alembic/versions/0568ccf46a6b_add_thread_specific_model_selection.py
@@ -17,15 +17,11 @@
def upgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
op.add_column(
"chat_session",
sa.Column("current_alternate_model", sa.String(), nullable=True),
)
- # ### end Alembic commands ###
def downgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
op.drop_column("chat_session", "current_alternate_model")
- # ### end Alembic commands ###
diff --git a/backend/alembic/versions/05c07bf07c00_add_search_doc_relevance_details.py b/backend/alembic/versions/05c07bf07c00_add_search_doc_relevance_details.py
new file mode 100644
index 00000000000..69eec4c108e
--- /dev/null
+++ b/backend/alembic/versions/05c07bf07c00_add_search_doc_relevance_details.py
@@ -0,0 +1,32 @@
+"""add search doc relevance details
+
+Revision ID: 05c07bf07c00
+Revises: b896bbd0d5a7
+Create Date: 2024-07-10 17:48:15.886653
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "05c07bf07c00"
+down_revision = "b896bbd0d5a7"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.add_column(
+ "search_doc",
+ sa.Column("is_relevant", sa.Boolean(), nullable=True),
+ )
+ op.add_column(
+ "search_doc",
+ sa.Column("relevance_explanation", sa.String(), nullable=True),
+ )
+
+
+def downgrade() -> None:
+ op.drop_column("search_doc", "relevance_explanation")
+ op.drop_column("search_doc", "is_relevant")
diff --git a/backend/alembic/versions/08a1eda20fe1_add_earliest_indexing_to_connector.py b/backend/alembic/versions/08a1eda20fe1_add_earliest_indexing_to_connector.py
new file mode 100644
index 00000000000..3f4893bd12c
--- /dev/null
+++ b/backend/alembic/versions/08a1eda20fe1_add_earliest_indexing_to_connector.py
@@ -0,0 +1,26 @@
+"""add_indexing_start_to_connector
+
+Revision ID: 08a1eda20fe1
+Revises: 8a87bd6ec550
+Create Date: 2024-07-23 11:12:39.462397
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "08a1eda20fe1"
+down_revision = "8a87bd6ec550"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.add_column(
+ "connector", sa.Column("indexing_start", sa.DateTime(), nullable=True)
+ )
+
+
+def downgrade() -> None:
+ op.drop_column("connector", "indexing_start")
diff --git a/backend/alembic/versions/1f60f60c3401_embedding_model_search_settings.py b/backend/alembic/versions/1f60f60c3401_embedding_model_search_settings.py
new file mode 100644
index 00000000000..42f4c22ed78
--- /dev/null
+++ b/backend/alembic/versions/1f60f60c3401_embedding_model_search_settings.py
@@ -0,0 +1,135 @@
+"""embedding model -> search settings
+
+Revision ID: 1f60f60c3401
+Revises: f17bf3b0d9f1
+Create Date: 2024-08-25 12:39:51.731632
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+from danswer.configs.chat_configs import NUM_POSTPROCESSED_RESULTS
+
+# revision identifiers, used by Alembic.
+revision = "1f60f60c3401"
+down_revision = "f17bf3b0d9f1"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.drop_constraint(
+ "index_attempt__embedding_model_fk", "index_attempt", type_="foreignkey"
+ )
+ # Rename the table
+ op.rename_table("embedding_model", "search_settings")
+
+ # Add new columns
+ op.add_column(
+ "search_settings",
+ sa.Column(
+ "multipass_indexing", sa.Boolean(), nullable=False, server_default="true"
+ ),
+ )
+ op.add_column(
+ "search_settings",
+ sa.Column(
+ "multilingual_expansion",
+ postgresql.ARRAY(sa.String()),
+ nullable=False,
+ server_default="{}",
+ ),
+ )
+ op.add_column(
+ "search_settings",
+ sa.Column(
+ "disable_rerank_for_streaming",
+ sa.Boolean(),
+ nullable=False,
+ server_default="false",
+ ),
+ )
+ op.add_column(
+ "search_settings", sa.Column("rerank_model_name", sa.String(), nullable=True)
+ )
+ op.add_column(
+ "search_settings", sa.Column("rerank_provider_type", sa.String(), nullable=True)
+ )
+ op.add_column(
+ "search_settings", sa.Column("rerank_api_key", sa.String(), nullable=True)
+ )
+ op.add_column(
+ "search_settings",
+ sa.Column(
+ "num_rerank",
+ sa.Integer(),
+ nullable=False,
+ server_default=str(NUM_POSTPROCESSED_RESULTS),
+ ),
+ )
+
+ # Add the new column as nullable initially
+ op.add_column(
+ "index_attempt", sa.Column("search_settings_id", sa.Integer(), nullable=True)
+ )
+
+ # Populate the new column with data from the existing embedding_model_id
+ op.execute("UPDATE index_attempt SET search_settings_id = embedding_model_id")
+
+ # Create the foreign key constraint
+ op.create_foreign_key(
+ "fk_index_attempt_search_settings",
+ "index_attempt",
+ "search_settings",
+ ["search_settings_id"],
+ ["id"],
+ )
+
+ # Make the new column non-nullable
+ op.alter_column("index_attempt", "search_settings_id", nullable=False)
+
+ # Drop the old embedding_model_id column
+ op.drop_column("index_attempt", "embedding_model_id")
+
+
+def downgrade() -> None:
+ # Add back the embedding_model_id column
+ op.add_column(
+ "index_attempt", sa.Column("embedding_model_id", sa.Integer(), nullable=True)
+ )
+
+ # Populate the old column with data from search_settings_id
+ op.execute("UPDATE index_attempt SET embedding_model_id = search_settings_id")
+
+ # Make the old column non-nullable
+ op.alter_column("index_attempt", "embedding_model_id", nullable=False)
+
+ # Drop the foreign key constraint
+ op.drop_constraint(
+ "fk_index_attempt_search_settings", "index_attempt", type_="foreignkey"
+ )
+
+ # Drop the new search_settings_id column
+ op.drop_column("index_attempt", "search_settings_id")
+
+ # Rename the table back
+ op.rename_table("search_settings", "embedding_model")
+
+ # Remove added columns
+ op.drop_column("embedding_model", "num_rerank")
+ op.drop_column("embedding_model", "rerank_api_key")
+ op.drop_column("embedding_model", "rerank_provider_type")
+ op.drop_column("embedding_model", "rerank_model_name")
+ op.drop_column("embedding_model", "disable_rerank_for_streaming")
+ op.drop_column("embedding_model", "multilingual_expansion")
+ op.drop_column("embedding_model", "multipass_indexing")
+
+ op.create_foreign_key(
+ "index_attempt__embedding_model_fk",
+ "index_attempt",
+ "embedding_model",
+ ["embedding_model_id"],
+ ["id"],
+ )
diff --git a/backend/alembic/versions/213fd978c6d8_notifications.py b/backend/alembic/versions/213fd978c6d8_notifications.py
new file mode 100644
index 00000000000..563556ea50a
--- /dev/null
+++ b/backend/alembic/versions/213fd978c6d8_notifications.py
@@ -0,0 +1,44 @@
+"""notifications
+
+Revision ID: 213fd978c6d8
+Revises: 5fc1f54cc252
+Create Date: 2024-08-10 11:13:36.070790
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "213fd978c6d8"
+down_revision = "5fc1f54cc252"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "notification",
+ sa.Column("id", sa.Integer(), nullable=False),
+ sa.Column(
+ "notif_type",
+ sa.String(),
+ nullable=False,
+ ),
+ sa.Column(
+ "user_id",
+ sa.UUID(),
+ nullable=True,
+ ),
+ sa.Column("dismissed", sa.Boolean(), nullable=False),
+ sa.Column("last_shown", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("first_shown", sa.DateTime(timezone=True), nullable=False),
+ sa.ForeignKeyConstraint(
+ ["user_id"],
+ ["user.id"],
+ ),
+ sa.PrimaryKeyConstraint("id"),
+ )
+
+
+def downgrade() -> None:
+ op.drop_table("notification")
diff --git a/backend/alembic/versions/23957775e5f5_remove_feedback_foreignkey_constraint.py b/backend/alembic/versions/23957775e5f5_remove_feedback_foreignkey_constraint.py
index 07b5601013b..10d094e0da2 100644
--- a/backend/alembic/versions/23957775e5f5_remove_feedback_foreignkey_constraint.py
+++ b/backend/alembic/versions/23957775e5f5_remove_feedback_foreignkey_constraint.py
@@ -79,7 +79,7 @@ def downgrade() -> None:
)
op.create_foreign_key(
"document_retrieval_feedback__chat_message_fk",
- "document_retrieval",
+ "document_retrieval_feedback",
"chat_message",
["chat_message_id"],
["id"],
diff --git a/backend/alembic/versions/27c6ecc08586_permission_framework.py b/backend/alembic/versions/27c6ecc08586_permission_framework.py
index cd869e2ba6d..ff41d2f5cff 100644
--- a/backend/alembic/versions/27c6ecc08586_permission_framework.py
+++ b/backend/alembic/versions/27c6ecc08586_permission_framework.py
@@ -160,12 +160,28 @@ def downgrade() -> None:
nullable=False,
),
)
- op.drop_constraint(
- "fk_index_attempt_credential_id", "index_attempt", type_="foreignkey"
- )
- op.drop_constraint(
- "fk_index_attempt_connector_id", "index_attempt", type_="foreignkey"
- )
+
+ # Check if the constraint exists before dropping
+ conn = op.get_bind()
+ inspector = sa.inspect(conn)
+ constraints = inspector.get_foreign_keys("index_attempt")
+
+ if any(
+ constraint["name"] == "fk_index_attempt_credential_id"
+ for constraint in constraints
+ ):
+ op.drop_constraint(
+ "fk_index_attempt_credential_id", "index_attempt", type_="foreignkey"
+ )
+
+ if any(
+ constraint["name"] == "fk_index_attempt_connector_id"
+ for constraint in constraints
+ ):
+ op.drop_constraint(
+ "fk_index_attempt_connector_id", "index_attempt", type_="foreignkey"
+ )
+
op.drop_column("index_attempt", "credential_id")
op.drop_column("index_attempt", "connector_id")
op.drop_table("connector_credential_pair")
diff --git a/backend/alembic/versions/2d2304e27d8c_add_above_below_to_persona.py b/backend/alembic/versions/2d2304e27d8c_add_above_below_to_persona.py
new file mode 100644
index 00000000000..cab166531ae
--- /dev/null
+++ b/backend/alembic/versions/2d2304e27d8c_add_above_below_to_persona.py
@@ -0,0 +1,32 @@
+"""Add Above Below to Persona
+
+Revision ID: 2d2304e27d8c
+Revises: 4b08d97e175a
+Create Date: 2024-08-21 19:15:15.762948
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "2d2304e27d8c"
+down_revision = "4b08d97e175a"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.add_column("persona", sa.Column("chunks_above", sa.Integer(), nullable=True))
+ op.add_column("persona", sa.Column("chunks_below", sa.Integer(), nullable=True))
+
+ op.execute(
+ "UPDATE persona SET chunks_above = 1, chunks_below = 1 WHERE chunks_above IS NULL AND chunks_below IS NULL"
+ )
+
+ op.alter_column("persona", "chunks_above", nullable=False)
+ op.alter_column("persona", "chunks_below", nullable=False)
+
+
+def downgrade() -> None:
+ op.drop_column("persona", "chunks_below")
+ op.drop_column("persona", "chunks_above")
diff --git a/backend/alembic/versions/325975216eb3_add_icon_color_and_icon_shape_to_persona.py b/backend/alembic/versions/325975216eb3_add_icon_color_and_icon_shape_to_persona.py
new file mode 100644
index 00000000000..46beab05f5c
--- /dev/null
+++ b/backend/alembic/versions/325975216eb3_add_icon_color_and_icon_shape_to_persona.py
@@ -0,0 +1,70 @@
+"""Add icon_color and icon_shape to Persona
+
+Revision ID: 325975216eb3
+Revises: 91ffac7e65b3
+Create Date: 2024-07-24 21:29:31.784562
+
+"""
+import random
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.sql import table, column, select
+
+# revision identifiers, used by Alembic.
+revision = "325975216eb3"
+down_revision = "91ffac7e65b3"
+branch_labels: None = None
+depends_on: None = None
+
+
+colorOptions = [
+ "#FF6FBF",
+ "#6FB1FF",
+ "#B76FFF",
+ "#FFB56F",
+ "#6FFF8D",
+ "#FF6F6F",
+ "#6FFFFF",
+]
+
+
+# Function to generate a random shape ensuring at least 3 of the middle 4 squares are filled
+def generate_random_shape() -> int:
+ center_squares = [12, 10, 6, 14, 13, 11, 7, 15]
+ center_fill = random.choice(center_squares)
+ remaining_squares = [i for i in range(16) if not (center_fill & (1 << i))]
+ random.shuffle(remaining_squares)
+ for i in range(10 - bin(center_fill).count("1")):
+ center_fill |= 1 << remaining_squares[i]
+ return center_fill
+
+
+def upgrade() -> None:
+ op.add_column("persona", sa.Column("icon_color", sa.String(), nullable=True))
+ op.add_column("persona", sa.Column("icon_shape", sa.Integer(), nullable=True))
+ op.add_column("persona", sa.Column("uploaded_image_id", sa.String(), nullable=True))
+
+ persona = table(
+ "persona",
+ column("id", sa.Integer),
+ column("icon_color", sa.String),
+ column("icon_shape", sa.Integer),
+ )
+
+ conn = op.get_bind()
+ personas = conn.execute(select(persona.c.id))
+
+ for persona_id in personas:
+ random_color = random.choice(colorOptions)
+ random_shape = generate_random_shape()
+ conn.execute(
+ persona.update()
+ .where(persona.c.id == persona_id[0])
+ .values(icon_color=random_color, icon_shape=random_shape)
+ )
+
+
+def downgrade() -> None:
+ op.drop_column("persona", "icon_shape")
+ op.drop_column("persona", "uploaded_image_id")
+ op.drop_column("persona", "icon_color")
diff --git a/backend/alembic/versions/351faebd379d_add_curator_fields.py b/backend/alembic/versions/351faebd379d_add_curator_fields.py
new file mode 100644
index 00000000000..b3254d26c16
--- /dev/null
+++ b/backend/alembic/versions/351faebd379d_add_curator_fields.py
@@ -0,0 +1,90 @@
+"""Add curator fields
+
+Revision ID: 351faebd379d
+Revises: ee3f4b47fad5
+Create Date: 2024-08-15 22:37:08.397052
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "351faebd379d"
+down_revision = "ee3f4b47fad5"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ # Add is_curator column to User__UserGroup table
+ op.add_column(
+ "user__user_group",
+ sa.Column("is_curator", sa.Boolean(), nullable=False, server_default="false"),
+ )
+
+ # Use batch mode to modify the enum type
+ with op.batch_alter_table("user", schema=None) as batch_op:
+ batch_op.alter_column( # type: ignore[attr-defined]
+ "role",
+ type_=sa.Enum(
+ "BASIC",
+ "ADMIN",
+ "CURATOR",
+ "GLOBAL_CURATOR",
+ name="userrole",
+ native_enum=False,
+ ),
+ existing_type=sa.Enum("BASIC", "ADMIN", name="userrole", native_enum=False),
+ existing_nullable=False,
+ )
+ # Create the association table
+ op.create_table(
+ "credential__user_group",
+ sa.Column("credential_id", sa.Integer(), nullable=False),
+ sa.Column("user_group_id", sa.Integer(), nullable=False),
+ sa.ForeignKeyConstraint(
+ ["credential_id"],
+ ["credential.id"],
+ ),
+ sa.ForeignKeyConstraint(
+ ["user_group_id"],
+ ["user_group.id"],
+ ),
+ sa.PrimaryKeyConstraint("credential_id", "user_group_id"),
+ )
+ op.add_column(
+ "credential",
+ sa.Column(
+ "curator_public", sa.Boolean(), nullable=False, server_default="false"
+ ),
+ )
+
+
+def downgrade() -> None:
+ # Update existing records to ensure they fit within the BASIC/ADMIN roles
+ op.execute(
+ "UPDATE \"user\" SET role = 'ADMIN' WHERE role IN ('CURATOR', 'GLOBAL_CURATOR')"
+ )
+
+ # Remove is_curator column from User__UserGroup table
+ op.drop_column("user__user_group", "is_curator")
+
+ with op.batch_alter_table("user", schema=None) as batch_op:
+ batch_op.alter_column( # type: ignore[attr-defined]
+ "role",
+ type_=sa.Enum(
+ "BASIC", "ADMIN", name="userrole", native_enum=False, length=20
+ ),
+ existing_type=sa.Enum(
+ "BASIC",
+ "ADMIN",
+ "CURATOR",
+ "GLOBAL_CURATOR",
+ name="userrole",
+ native_enum=False,
+ ),
+ existing_nullable=False,
+ )
+ # Drop the association table
+ op.drop_table("credential__user_group")
+ op.drop_column("credential", "curator_public")
diff --git a/backend/alembic/versions/3a7802814195_add_alternate_assistant_to_chat_message.py b/backend/alembic/versions/3a7802814195_add_alternate_assistant_to_chat_message.py
index 5e50c02c8bf..bfde0162ba2 100644
--- a/backend/alembic/versions/3a7802814195_add_alternate_assistant_to_chat_message.py
+++ b/backend/alembic/versions/3a7802814195_add_alternate_assistant_to_chat_message.py
@@ -18,7 +18,6 @@
def upgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
op.add_column(
"chat_message", sa.Column("alternate_assistant_id", sa.Integer(), nullable=True)
)
@@ -29,10 +28,8 @@ def upgrade() -> None:
["alternate_assistant_id"],
["id"],
)
- # ### end Alembic commands ###
def downgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
op.drop_constraint("fk_chat_message_persona", "chat_message", type_="foreignkey")
op.drop_column("chat_message", "alternate_assistant_id")
diff --git a/backend/alembic/versions/43cbbb3f5e6a_rename_index_origin_to_index_recursively.py b/backend/alembic/versions/43cbbb3f5e6a_rename_index_origin_to_index_recursively.py
new file mode 100644
index 00000000000..6aa2ffca0a6
--- /dev/null
+++ b/backend/alembic/versions/43cbbb3f5e6a_rename_index_origin_to_index_recursively.py
@@ -0,0 +1,42 @@
+"""Rename index_origin to index_recursively
+
+Revision ID: 1d6ad76d1f37
+Revises: e1392f05e840
+Create Date: 2024-08-01 12:38:54.466081
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "1d6ad76d1f37"
+down_revision = "e1392f05e840"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.execute(
+ """
+ UPDATE connector
+ SET connector_specific_config = jsonb_set(
+ connector_specific_config,
+ '{index_recursively}',
+ 'true'::jsonb
+ ) - 'index_origin'
+ WHERE connector_specific_config ? 'index_origin'
+ """
+ )
+
+
+def downgrade() -> None:
+ op.execute(
+ """
+ UPDATE connector
+ SET connector_specific_config = jsonb_set(
+ connector_specific_config,
+ '{index_origin}',
+ connector_specific_config->'index_recursively'
+ ) - 'index_recursively'
+ WHERE connector_specific_config ? 'index_recursively'
+ """
+ )
diff --git a/backend/alembic/versions/44f856ae2a4a_add_cloud_embedding_model.py b/backend/alembic/versions/44f856ae2a4a_add_cloud_embedding_model.py
new file mode 100644
index 00000000000..2d0e1a32f98
--- /dev/null
+++ b/backend/alembic/versions/44f856ae2a4a_add_cloud_embedding_model.py
@@ -0,0 +1,65 @@
+"""add cloud embedding model and update embedding_model
+
+Revision ID: 44f856ae2a4a
+Revises: d716b0791ddd
+Create Date: 2024-06-28 20:01:05.927647
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "44f856ae2a4a"
+down_revision = "d716b0791ddd"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ # Create embedding_provider table
+ op.create_table(
+ "embedding_provider",
+ sa.Column("id", sa.Integer(), nullable=False),
+ sa.Column("name", sa.String(), nullable=False),
+ sa.Column("api_key", sa.LargeBinary(), nullable=True),
+ sa.Column("default_model_id", sa.Integer(), nullable=True),
+ sa.PrimaryKeyConstraint("id"),
+ sa.UniqueConstraint("name"),
+ )
+
+ # Add cloud_provider_id to embedding_model table
+ op.add_column(
+ "embedding_model", sa.Column("cloud_provider_id", sa.Integer(), nullable=True)
+ )
+
+ # Add foreign key constraints
+ op.create_foreign_key(
+ "fk_embedding_model_cloud_provider",
+ "embedding_model",
+ "embedding_provider",
+ ["cloud_provider_id"],
+ ["id"],
+ )
+ op.create_foreign_key(
+ "fk_embedding_provider_default_model",
+ "embedding_provider",
+ "embedding_model",
+ ["default_model_id"],
+ ["id"],
+ )
+
+
+def downgrade() -> None:
+ # Remove foreign key constraints
+ op.drop_constraint(
+ "fk_embedding_model_cloud_provider", "embedding_model", type_="foreignkey"
+ )
+ op.drop_constraint(
+ "fk_embedding_provider_default_model", "embedding_provider", type_="foreignkey"
+ )
+
+ # Remove cloud_provider_id column
+ op.drop_column("embedding_model", "cloud_provider_id")
+
+ # Drop embedding_provider table
+ op.drop_table("embedding_provider")
diff --git a/backend/alembic/versions/473a1a7ca408_add_display_model_names_to_llm_provider.py b/backend/alembic/versions/473a1a7ca408_add_display_model_names_to_llm_provider.py
new file mode 100644
index 00000000000..2e3f377e372
--- /dev/null
+++ b/backend/alembic/versions/473a1a7ca408_add_display_model_names_to_llm_provider.py
@@ -0,0 +1,49 @@
+"""Add display_model_names to llm_provider
+
+Revision ID: 473a1a7ca408
+Revises: 325975216eb3
+Create Date: 2024-07-25 14:31:02.002917
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "473a1a7ca408"
+down_revision = "325975216eb3"
+branch_labels: None = None
+depends_on: None = None
+
+default_models_by_provider = {
+ "openai": ["gpt-4", "gpt-4o", "gpt-4o-mini"],
+ "bedrock": [
+ "meta.llama3-1-70b-instruct-v1:0",
+ "meta.llama3-1-8b-instruct-v1:0",
+ "anthropic.claude-3-opus-20240229-v1:0",
+ "mistral.mistral-large-2402-v1:0",
+ "anthropic.claude-3-5-sonnet-20240620-v1:0",
+ ],
+ "anthropic": ["claude-3-opus-20240229", "claude-3-5-sonnet-20240620"],
+}
+
+
+def upgrade() -> None:
+ op.add_column(
+ "llm_provider",
+ sa.Column("display_model_names", postgresql.ARRAY(sa.String()), nullable=True),
+ )
+
+ connection = op.get_bind()
+ for provider, models in default_models_by_provider.items():
+ connection.execute(
+ sa.text(
+ "UPDATE llm_provider SET display_model_names = :models WHERE provider = :provider"
+ ),
+ {"models": models, "provider": provider},
+ )
+
+
+def downgrade() -> None:
+ op.drop_column("llm_provider", "display_model_names")
diff --git a/backend/alembic/versions/4a951134c801_moved_status_to_connector_credential_.py b/backend/alembic/versions/4a951134c801_moved_status_to_connector_credential_.py
new file mode 100644
index 00000000000..3deebaecd39
--- /dev/null
+++ b/backend/alembic/versions/4a951134c801_moved_status_to_connector_credential_.py
@@ -0,0 +1,80 @@
+"""Moved status to connector credential pair
+
+Revision ID: 4a951134c801
+Revises: 7477a5f5d728
+Create Date: 2024-08-10 19:20:34.527559
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "4a951134c801"
+down_revision = "7477a5f5d728"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.add_column(
+ "connector_credential_pair",
+ sa.Column(
+ "status",
+ sa.Enum(
+ "ACTIVE",
+ "PAUSED",
+ "DELETING",
+ name="connectorcredentialpairstatus",
+ native_enum=False,
+ ),
+ nullable=True,
+ ),
+ )
+
+ # Update status of connector_credential_pair based on connector's disabled status
+ op.execute(
+ """
+ UPDATE connector_credential_pair
+ SET status = CASE
+ WHEN (
+ SELECT disabled
+ FROM connector
+ WHERE connector.id = connector_credential_pair.connector_id
+ ) = FALSE THEN 'ACTIVE'
+ ELSE 'PAUSED'
+ END
+ """
+ )
+
+ # Make the status column not nullable after setting values
+ op.alter_column("connector_credential_pair", "status", nullable=False)
+
+ op.drop_column("connector", "disabled")
+
+
+def downgrade() -> None:
+ op.add_column(
+ "connector",
+ sa.Column("disabled", sa.BOOLEAN(), autoincrement=False, nullable=True),
+ )
+
+ # Update disabled status of connector based on connector_credential_pair's status
+ op.execute(
+ """
+ UPDATE connector
+ SET disabled = CASE
+ WHEN EXISTS (
+ SELECT 1
+ FROM connector_credential_pair
+ WHERE connector_credential_pair.connector_id = connector.id
+ AND connector_credential_pair.status = 'ACTIVE'
+ ) THEN FALSE
+ ELSE TRUE
+ END
+ """
+ )
+
+ # Make the disabled column not nullable after setting values
+ op.alter_column("connector", "disabled", nullable=False)
+
+ op.drop_column("connector_credential_pair", "status")
diff --git a/backend/alembic/versions/4b08d97e175a_change_default_prune_freq.py b/backend/alembic/versions/4b08d97e175a_change_default_prune_freq.py
new file mode 100644
index 00000000000..29316adb1df
--- /dev/null
+++ b/backend/alembic/versions/4b08d97e175a_change_default_prune_freq.py
@@ -0,0 +1,34 @@
+"""change default prune_freq
+
+Revision ID: 4b08d97e175a
+Revises: d9ec13955951
+Create Date: 2024-08-20 15:28:52.993827
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "4b08d97e175a"
+down_revision = "d9ec13955951"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.execute(
+ """
+ UPDATE connector
+ SET prune_freq = 2592000
+ WHERE prune_freq = 86400
+ """
+ )
+
+
+def downgrade() -> None:
+ op.execute(
+ """
+ UPDATE connector
+ SET prune_freq = 86400
+ WHERE prune_freq = 2592000
+ """
+ )
diff --git a/backend/alembic/versions/4ea2c93919c1_add_type_to_credentials.py b/backend/alembic/versions/4ea2c93919c1_add_type_to_credentials.py
new file mode 100644
index 00000000000..8077b24b095
--- /dev/null
+++ b/backend/alembic/versions/4ea2c93919c1_add_type_to_credentials.py
@@ -0,0 +1,72 @@
+"""Add type to credentials
+
+Revision ID: 4ea2c93919c1
+Revises: 473a1a7ca408
+Create Date: 2024-07-18 13:07:13.655895
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "4ea2c93919c1"
+down_revision = "473a1a7ca408"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ # Add the new 'source' column to the 'credential' table
+ op.add_column(
+ "credential",
+ sa.Column(
+ "source",
+ sa.String(length=100), # Use String instead of Enum
+ nullable=True, # Initially allow NULL values
+ ),
+ )
+ op.add_column(
+ "credential",
+ sa.Column(
+ "name",
+ sa.String(),
+ nullable=True,
+ ),
+ )
+
+ # Create a temporary table that maps each credential to a single connector source.
+ # This is needed because a credential can be associated with multiple connectors,
+ # but we want to assign a single source to each credential.
+ # We use DISTINCT ON to ensure we only get one row per credential_id.
+ op.execute(
+ """
+ CREATE TEMPORARY TABLE temp_connector_credential AS
+ SELECT DISTINCT ON (cc.credential_id)
+ cc.credential_id,
+ c.source AS connector_source
+ FROM connector_credential_pair cc
+ JOIN connector c ON cc.connector_id = c.id
+ """
+ )
+
+ # Update the 'source' column in the 'credential' table
+ op.execute(
+ """
+ UPDATE credential cred
+ SET source = COALESCE(
+ (SELECT connector_source
+ FROM temp_connector_credential temp
+ WHERE cred.id = temp.credential_id),
+ 'NOT_APPLICABLE'
+ )
+ """
+ )
+ # If no exception was raised, alter the column
+ op.alter_column("credential", "source", nullable=True) # TODO modify
+ # # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ op.drop_column("credential", "source")
+ op.drop_column("credential", "name")
diff --git a/backend/alembic/versions/5fc1f54cc252_hybrid_enum.py b/backend/alembic/versions/5fc1f54cc252_hybrid_enum.py
new file mode 100644
index 00000000000..63b1e7875a9
--- /dev/null
+++ b/backend/alembic/versions/5fc1f54cc252_hybrid_enum.py
@@ -0,0 +1,25 @@
+"""hybrid-enum
+
+Revision ID: 5fc1f54cc252
+Revises: 1d6ad76d1f37
+Create Date: 2024-08-06 15:35:40.278485
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "5fc1f54cc252"
+down_revision = "1d6ad76d1f37"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.drop_column("persona", "search_type")
+
+
+def downgrade() -> None:
+ op.add_column("persona", sa.Column("search_type", sa.String(), nullable=True))
+ op.execute("UPDATE persona SET search_type = 'SEMANTIC'")
+ op.alter_column("persona", "search_type", nullable=False)
diff --git a/backend/alembic/versions/7477a5f5d728_added_model_defaults_for_users.py b/backend/alembic/versions/7477a5f5d728_added_model_defaults_for_users.py
new file mode 100644
index 00000000000..6efb9840526
--- /dev/null
+++ b/backend/alembic/versions/7477a5f5d728_added_model_defaults_for_users.py
@@ -0,0 +1,24 @@
+"""Added model defaults for users
+
+Revision ID: 7477a5f5d728
+Revises: 213fd978c6d8
+Create Date: 2024-08-04 19:00:04.512634
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "7477a5f5d728"
+down_revision = "213fd978c6d8"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.add_column("user", sa.Column("default_model", sa.Text(), nullable=True))
+
+
+def downgrade() -> None:
+ op.drop_column("user", "default_model")
diff --git a/backend/alembic/versions/76b60d407dfb_cc_pair_name_not_unique.py b/backend/alembic/versions/76b60d407dfb_cc_pair_name_not_unique.py
index c609ca4ae04..1dfbb9365d8 100644
--- a/backend/alembic/versions/76b60d407dfb_cc_pair_name_not_unique.py
+++ b/backend/alembic/versions/76b60d407dfb_cc_pair_name_not_unique.py
@@ -28,5 +28,9 @@ def upgrade() -> None:
def downgrade() -> None:
- # This wasn't really required by the code either, no good reason to make it unique again
- pass
+ op.create_unique_constraint(
+ "connector_credential_pair__name__key", "connector_credential_pair", ["name"]
+ )
+ op.alter_column(
+ "connector_credential_pair", "name", existing_type=sa.String(), nullable=True
+ )
diff --git a/backend/alembic/versions/776b3bbe9092_remove_remaining_enums.py b/backend/alembic/versions/776b3bbe9092_remove_remaining_enums.py
index 6fb8c5ac0e1..c2ba10b3875 100644
--- a/backend/alembic/versions/776b3bbe9092_remove_remaining_enums.py
+++ b/backend/alembic/versions/776b3bbe9092_remove_remaining_enums.py
@@ -10,7 +10,7 @@
from danswer.db.models import IndexModelStatus
from danswer.search.enums import RecencyBiasSetting
-from danswer.search.models import SearchType
+from danswer.search.enums import SearchType
# revision identifiers, used by Alembic.
revision = "776b3bbe9092"
diff --git a/backend/alembic/versions/795b20b85b4b_add_llm_group_permissions_control.py b/backend/alembic/versions/795b20b85b4b_add_llm_group_permissions_control.py
new file mode 100644
index 00000000000..8b7fb9a2b8d
--- /dev/null
+++ b/backend/alembic/versions/795b20b85b4b_add_llm_group_permissions_control.py
@@ -0,0 +1,41 @@
+"""add_llm_group_permissions_control
+
+Revision ID: 795b20b85b4b
+Revises: 05c07bf07c00
+Create Date: 2024-07-19 11:54:35.701558
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = "795b20b85b4b"
+down_revision = "05c07bf07c00"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "llm_provider__user_group",
+ sa.Column("llm_provider_id", sa.Integer(), nullable=False),
+ sa.Column("user_group_id", sa.Integer(), nullable=False),
+ sa.ForeignKeyConstraint(
+ ["llm_provider_id"],
+ ["llm_provider.id"],
+ ),
+ sa.ForeignKeyConstraint(
+ ["user_group_id"],
+ ["user_group.id"],
+ ),
+ sa.PrimaryKeyConstraint("llm_provider_id", "user_group_id"),
+ )
+ op.add_column(
+ "llm_provider",
+ sa.Column("is_public", sa.Boolean(), nullable=False, server_default="true"),
+ )
+
+
+def downgrade() -> None:
+ op.drop_table("llm_provider__user_group")
+ op.drop_column("llm_provider", "is_public")
diff --git a/backend/alembic/versions/7aea705850d5_added_slack_auto_filter.py b/backend/alembic/versions/7aea705850d5_added_slack_auto_filter.py
index a07b94bd925..b41e18f856c 100644
--- a/backend/alembic/versions/7aea705850d5_added_slack_auto_filter.py
+++ b/backend/alembic/versions/7aea705850d5_added_slack_auto_filter.py
@@ -10,8 +10,8 @@
revision = "7aea705850d5"
down_revision = "4505fd7302e1"
-branch_labels = None
-depends_on = None
+branch_labels: None = None
+depends_on: None = None
def upgrade() -> None:
diff --git a/backend/alembic/versions/8a87bd6ec550_associate_index_attempts_with_ccpair.py b/backend/alembic/versions/8a87bd6ec550_associate_index_attempts_with_ccpair.py
new file mode 100644
index 00000000000..166c4b7ba18
--- /dev/null
+++ b/backend/alembic/versions/8a87bd6ec550_associate_index_attempts_with_ccpair.py
@@ -0,0 +1,107 @@
+"""associate index attempts with ccpair
+
+Revision ID: 8a87bd6ec550
+Revises: 4ea2c93919c1
+Create Date: 2024-07-22 15:15:52.558451
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "8a87bd6ec550"
+down_revision = "4ea2c93919c1"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ # Add the new connector_credential_pair_id column
+ op.add_column(
+ "index_attempt",
+ sa.Column("connector_credential_pair_id", sa.Integer(), nullable=True),
+ )
+
+ # Create a foreign key constraint to the connector_credential_pair table
+ op.create_foreign_key(
+ "fk_index_attempt_connector_credential_pair_id",
+ "index_attempt",
+ "connector_credential_pair",
+ ["connector_credential_pair_id"],
+ ["id"],
+ )
+
+ # Populate the new connector_credential_pair_id column using existing connector_id and credential_id
+ op.execute(
+ """
+ UPDATE index_attempt ia
+ SET connector_credential_pair_id = (
+ SELECT id FROM connector_credential_pair ccp
+ WHERE
+ (ia.connector_id IS NULL OR ccp.connector_id = ia.connector_id)
+ AND (ia.credential_id IS NULL OR ccp.credential_id = ia.credential_id)
+ LIMIT 1
+ )
+ WHERE ia.connector_id IS NOT NULL OR ia.credential_id IS NOT NULL
+ """
+ )
+
+ # For good measure
+ op.execute(
+ """
+ DELETE FROM index_attempt
+ WHERE connector_credential_pair_id IS NULL
+ """
+ )
+
+ # Make the new connector_credential_pair_id column non-nullable
+ op.alter_column("index_attempt", "connector_credential_pair_id", nullable=False)
+
+ # Drop the old connector_id and credential_id columns
+ op.drop_column("index_attempt", "connector_id")
+ op.drop_column("index_attempt", "credential_id")
+
+ # Update the index to use connector_credential_pair_id
+ op.create_index(
+ "ix_index_attempt_latest_for_connector_credential_pair",
+ "index_attempt",
+ ["connector_credential_pair_id", "time_created"],
+ )
+
+
+def downgrade() -> None:
+ # Add back the old connector_id and credential_id columns
+ op.add_column(
+ "index_attempt", sa.Column("connector_id", sa.Integer(), nullable=True)
+ )
+ op.add_column(
+ "index_attempt", sa.Column("credential_id", sa.Integer(), nullable=True)
+ )
+
+ # Populate the old connector_id and credential_id columns using the connector_credential_pair_id
+ op.execute(
+ """
+ UPDATE index_attempt ia
+ SET connector_id = ccp.connector_id, credential_id = ccp.credential_id
+ FROM connector_credential_pair ccp
+ WHERE ia.connector_credential_pair_id = ccp.id
+ """
+ )
+
+ # Make the old connector_id and credential_id columns non-nullable
+ op.alter_column("index_attempt", "connector_id", nullable=False)
+ op.alter_column("index_attempt", "credential_id", nullable=False)
+
+ # Drop the new connector_credential_pair_id column
+ op.drop_constraint(
+ "fk_index_attempt_connector_credential_pair_id",
+ "index_attempt",
+ type_="foreignkey",
+ )
+ op.drop_column("index_attempt", "connector_credential_pair_id")
+
+ op.create_index(
+ "ix_index_attempt_latest_for_connector_credential_pair",
+ "index_attempt",
+ ["connector_id", "credential_id", "time_created"],
+ )
diff --git a/backend/alembic/versions/91ffac7e65b3_add_expiry_time.py b/backend/alembic/versions/91ffac7e65b3_add_expiry_time.py
new file mode 100644
index 00000000000..7c029b3c9cf
--- /dev/null
+++ b/backend/alembic/versions/91ffac7e65b3_add_expiry_time.py
@@ -0,0 +1,26 @@
+"""add expiry time
+
+Revision ID: 91ffac7e65b3
+Revises: bc9771dccadf
+Create Date: 2024-06-24 09:39:56.462242
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "91ffac7e65b3"
+down_revision = "795b20b85b4b"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.add_column(
+ "user", sa.Column("oidc_expiry", sa.DateTime(timezone=True), nullable=True)
+ )
+
+
+def downgrade() -> None:
+ op.drop_column("user", "oidc_expiry")
diff --git a/backend/alembic/versions/b082fec533f0_make_last_attempt_status_nullable.py b/backend/alembic/versions/b082fec533f0_make_last_attempt_status_nullable.py
index cdf1722b3a5..a6938e365c6 100644
--- a/backend/alembic/versions/b082fec533f0_make_last_attempt_status_nullable.py
+++ b/backend/alembic/versions/b082fec533f0_make_last_attempt_status_nullable.py
@@ -16,7 +16,6 @@
def upgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
op.alter_column(
"connector_credential_pair",
"last_attempt_status",
@@ -29,11 +28,9 @@ def upgrade() -> None:
),
nullable=True,
)
- # ### end Alembic commands ###
def downgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
op.alter_column(
"connector_credential_pair",
"last_attempt_status",
@@ -46,4 +43,3 @@ def downgrade() -> None:
),
nullable=False,
)
- # ### end Alembic commands ###
diff --git a/backend/alembic/versions/b896bbd0d5a7_backfill_is_internet_data_to_false.py b/backend/alembic/versions/b896bbd0d5a7_backfill_is_internet_data_to_false.py
new file mode 100644
index 00000000000..9deac574b28
--- /dev/null
+++ b/backend/alembic/versions/b896bbd0d5a7_backfill_is_internet_data_to_false.py
@@ -0,0 +1,23 @@
+"""backfill is_internet data to False
+
+Revision ID: b896bbd0d5a7
+Revises: 44f856ae2a4a
+Create Date: 2024-07-16 15:21:05.718571
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "b896bbd0d5a7"
+down_revision = "44f856ae2a4a"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.execute("UPDATE search_doc SET is_internet = FALSE WHERE is_internet IS NULL")
+
+
+def downgrade() -> None:
+ pass
diff --git a/backend/alembic/versions/c5b692fa265c_add_index_attempt_errors_table.py b/backend/alembic/versions/c5b692fa265c_add_index_attempt_errors_table.py
new file mode 100644
index 00000000000..e4808042aae
--- /dev/null
+++ b/backend/alembic/versions/c5b692fa265c_add_index_attempt_errors_table.py
@@ -0,0 +1,57 @@
+"""Add index_attempt_errors table
+
+Revision ID: c5b692fa265c
+Revises: 4a951134c801
+Create Date: 2024-08-08 14:06:39.581972
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "c5b692fa265c"
+down_revision = "4a951134c801"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "index_attempt_errors",
+ sa.Column("id", sa.Integer(), nullable=False),
+ sa.Column("index_attempt_id", sa.Integer(), nullable=True),
+ sa.Column("batch", sa.Integer(), nullable=True),
+ sa.Column(
+ "doc_summaries",
+ postgresql.JSONB(astext_type=sa.Text()),
+ nullable=False,
+ ),
+ sa.Column("error_msg", sa.Text(), nullable=True),
+ sa.Column("traceback", sa.Text(), nullable=True),
+ sa.Column(
+ "time_created",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("now()"),
+ nullable=False,
+ ),
+ sa.ForeignKeyConstraint(
+ ["index_attempt_id"],
+ ["index_attempt.id"],
+ ),
+ sa.PrimaryKeyConstraint("id"),
+ )
+ op.create_index(
+ "index_attempt_id",
+ "index_attempt_errors",
+ ["time_created"],
+ unique=False,
+ )
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_index("index_attempt_id", table_name="index_attempt_errors")
+ op.drop_table("index_attempt_errors")
+ # ### end Alembic commands ###
diff --git a/backend/alembic/versions/d5645c915d0e_remove_deletion_attempt_table.py b/backend/alembic/versions/d5645c915d0e_remove_deletion_attempt_table.py
index aa4e7c71c1e..5ef63ed331c 100644
--- a/backend/alembic/versions/d5645c915d0e_remove_deletion_attempt_table.py
+++ b/backend/alembic/versions/d5645c915d0e_remove_deletion_attempt_table.py
@@ -19,6 +19,9 @@
def upgrade() -> None:
op.drop_table("deletion_attempt")
+ # Remove the DeletionStatus enum
+ op.execute("DROP TYPE IF EXISTS deletionstatus;")
+
def downgrade() -> None:
op.create_table(
diff --git a/backend/alembic/versions/d716b0791ddd_combined_slack_id_fields.py b/backend/alembic/versions/d716b0791ddd_combined_slack_id_fields.py
new file mode 100644
index 00000000000..6510d8b39da
--- /dev/null
+++ b/backend/alembic/versions/d716b0791ddd_combined_slack_id_fields.py
@@ -0,0 +1,45 @@
+"""combined slack id fields
+
+Revision ID: d716b0791ddd
+Revises: 7aea705850d5
+Create Date: 2024-07-10 17:57:45.630550
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "d716b0791ddd"
+down_revision = "7aea705850d5"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.execute(
+ """
+ UPDATE slack_bot_config
+ SET channel_config = jsonb_set(
+ channel_config,
+ '{respond_member_group_list}',
+ coalesce(channel_config->'respond_team_member_list', '[]'::jsonb) ||
+ coalesce(channel_config->'respond_slack_group_list', '[]'::jsonb)
+ ) - 'respond_team_member_list' - 'respond_slack_group_list'
+ """
+ )
+
+
+def downgrade() -> None:
+ op.execute(
+ """
+ UPDATE slack_bot_config
+ SET channel_config = jsonb_set(
+ jsonb_set(
+ channel_config - 'respond_member_group_list',
+ '{respond_team_member_list}',
+ '[]'::jsonb
+ ),
+ '{respond_slack_group_list}',
+ '[]'::jsonb
+ )
+ """
+ )
diff --git a/backend/alembic/versions/d9ec13955951_remove__dim_suffix_from_model_name.py b/backend/alembic/versions/d9ec13955951_remove__dim_suffix_from_model_name.py
new file mode 100644
index 00000000000..0e84d5fe85a
--- /dev/null
+++ b/backend/alembic/versions/d9ec13955951_remove__dim_suffix_from_model_name.py
@@ -0,0 +1,31 @@
+"""Remove _alt suffix from model_name
+
+Revision ID: d9ec13955951
+Revises: da4c21c69164
+Create Date: 2024-08-20 16:31:32.955686
+
+"""
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "d9ec13955951"
+down_revision = "da4c21c69164"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.execute(
+ """
+ UPDATE embedding_model
+ SET model_name = regexp_replace(model_name, '__danswer_alt_index$', '')
+ WHERE model_name LIKE '%__danswer_alt_index'
+ """
+ )
+
+
+def downgrade() -> None:
+ # We can't reliably add the __danswer_alt_index suffix back, so we'll leave this empty
+ pass
diff --git a/backend/alembic/versions/da4c21c69164_chosen_assistants_changed_to_jsonb.py b/backend/alembic/versions/da4c21c69164_chosen_assistants_changed_to_jsonb.py
new file mode 100644
index 00000000000..95b53cbeb41
--- /dev/null
+++ b/backend/alembic/versions/da4c21c69164_chosen_assistants_changed_to_jsonb.py
@@ -0,0 +1,65 @@
+"""chosen_assistants changed to jsonb
+
+Revision ID: da4c21c69164
+Revises: c5b692fa265c
+Create Date: 2024-08-18 19:06:47.291491
+
+"""
+import json
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "da4c21c69164"
+down_revision = "c5b692fa265c"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ conn = op.get_bind()
+ existing_ids_and_chosen_assistants = conn.execute(
+ sa.text("select id, chosen_assistants from public.user")
+ )
+ op.drop_column(
+ "user",
+ "chosen_assistants",
+ )
+ op.add_column(
+ "user",
+ sa.Column(
+ "chosen_assistants",
+ postgresql.JSONB(astext_type=sa.Text()),
+ nullable=True,
+ ),
+ )
+ for id, chosen_assistants in existing_ids_and_chosen_assistants:
+ conn.execute(
+ sa.text(
+ "update public.user set chosen_assistants = :chosen_assistants where id = :id"
+ ),
+ {"chosen_assistants": json.dumps(chosen_assistants), "id": id},
+ )
+
+
+def downgrade() -> None:
+ conn = op.get_bind()
+ existing_ids_and_chosen_assistants = conn.execute(
+ sa.text("select id, chosen_assistants from public.user")
+ )
+ op.drop_column(
+ "user",
+ "chosen_assistants",
+ )
+ op.add_column(
+ "user",
+ sa.Column("chosen_assistants", postgresql.ARRAY(sa.Integer()), nullable=True),
+ )
+ for id, chosen_assistants in existing_ids_and_chosen_assistants:
+ conn.execute(
+ sa.text(
+ "update public.user set chosen_assistants = :chosen_assistants where id = :id"
+ ),
+ {"chosen_assistants": chosen_assistants, "id": id},
+ )
diff --git a/backend/alembic/versions/dbaa756c2ccf_embedding_models.py b/backend/alembic/versions/dbaa756c2ccf_embedding_models.py
index a7c9b8f5aee..6274b3e1334 100644
--- a/backend/alembic/versions/dbaa756c2ccf_embedding_models.py
+++ b/backend/alembic/versions/dbaa756c2ccf_embedding_models.py
@@ -9,7 +9,7 @@
import sqlalchemy as sa
from sqlalchemy import table, column, String, Integer, Boolean
-from danswer.db.embedding_model import (
+from danswer.db.search_settings import (
get_new_default_embedding_model,
get_old_default_embedding_model,
user_has_overridden_embedding_model,
@@ -71,14 +71,14 @@ def upgrade() -> None:
"query_prefix": old_embedding_model.query_prefix,
"passage_prefix": old_embedding_model.passage_prefix,
"index_name": old_embedding_model.index_name,
- "status": old_embedding_model.status,
+ "status": IndexModelStatus.PRESENT,
}
],
)
# if the user has not overridden the default embedding model via env variables,
# insert the new default model into the database to auto-upgrade them
if not user_has_overridden_embedding_model():
- new_embedding_model = get_new_default_embedding_model(is_present=False)
+ new_embedding_model = get_new_default_embedding_model()
op.bulk_insert(
EmbeddingModel,
[
@@ -136,4 +136,4 @@ def downgrade() -> None:
)
op.drop_column("index_attempt", "embedding_model_id")
op.drop_table("embedding_model")
- op.execute("DROP TYPE indexmodelstatus;")
+ op.execute("DROP TYPE IF EXISTS indexmodelstatus;")
diff --git a/backend/alembic/versions/e1392f05e840_added_input_prompts.py b/backend/alembic/versions/e1392f05e840_added_input_prompts.py
new file mode 100644
index 00000000000..dd358220f7e
--- /dev/null
+++ b/backend/alembic/versions/e1392f05e840_added_input_prompts.py
@@ -0,0 +1,58 @@
+"""Added input prompts
+
+Revision ID: e1392f05e840
+Revises: 08a1eda20fe1
+Create Date: 2024-07-13 19:09:22.556224
+
+"""
+
+import fastapi_users_db_sqlalchemy
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "e1392f05e840"
+down_revision = "08a1eda20fe1"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "inputprompt",
+ sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+ sa.Column("prompt", sa.String(), nullable=False),
+ sa.Column("content", sa.String(), nullable=False),
+ sa.Column("active", sa.Boolean(), nullable=False),
+ sa.Column("is_public", sa.Boolean(), nullable=False),
+ sa.Column(
+ "user_id",
+ fastapi_users_db_sqlalchemy.generics.GUID(),
+ nullable=True,
+ ),
+ sa.ForeignKeyConstraint(
+ ["user_id"],
+ ["user.id"],
+ ),
+ sa.PrimaryKeyConstraint("id"),
+ )
+ op.create_table(
+ "inputprompt__user",
+ sa.Column("input_prompt_id", sa.Integer(), nullable=False),
+ sa.Column("user_id", sa.Integer(), nullable=False),
+ sa.ForeignKeyConstraint(
+ ["input_prompt_id"],
+ ["inputprompt.id"],
+ ),
+ sa.ForeignKeyConstraint(
+ ["user_id"],
+ ["inputprompt.id"],
+ ),
+ sa.PrimaryKeyConstraint("input_prompt_id", "user_id"),
+ )
+
+
+def downgrade() -> None:
+ op.drop_table("inputprompt__user")
+ op.drop_table("inputprompt")
diff --git a/backend/alembic/versions/ee3f4b47fad5_added_alternate_model_to_chat_message.py b/backend/alembic/versions/ee3f4b47fad5_added_alternate_model_to_chat_message.py
new file mode 100644
index 00000000000..64ffdad25f7
--- /dev/null
+++ b/backend/alembic/versions/ee3f4b47fad5_added_alternate_model_to_chat_message.py
@@ -0,0 +1,28 @@
+"""Added alternate model to chat message
+
+Revision ID: ee3f4b47fad5
+Revises: 2d2304e27d8c
+Create Date: 2024-08-12 00:11:50.915845
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "ee3f4b47fad5"
+down_revision = "2d2304e27d8c"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ op.add_column(
+ "chat_message",
+ sa.Column("overridden_model", sa.String(length=255), nullable=True),
+ )
+
+
+def downgrade() -> None:
+ op.drop_column("chat_message", "overridden_model")
diff --git a/backend/alembic/versions/f17bf3b0d9f1_embedding_provider_by_provider_type.py b/backend/alembic/versions/f17bf3b0d9f1_embedding_provider_by_provider_type.py
new file mode 100644
index 00000000000..a141f946d2a
--- /dev/null
+++ b/backend/alembic/versions/f17bf3b0d9f1_embedding_provider_by_provider_type.py
@@ -0,0 +1,172 @@
+"""embedding provider by provider type
+
+Revision ID: f17bf3b0d9f1
+Revises: 351faebd379d
+Create Date: 2024-08-21 13:13:31.120460
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "f17bf3b0d9f1"
+down_revision = "351faebd379d"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+ # Add provider_type column to embedding_provider
+ op.add_column(
+ "embedding_provider",
+ sa.Column("provider_type", sa.String(50), nullable=True),
+ )
+
+ # Update provider_type with existing name values
+ op.execute("UPDATE embedding_provider SET provider_type = UPPER(name)")
+
+ # Make provider_type not nullable
+ op.alter_column("embedding_provider", "provider_type", nullable=False)
+
+ # Drop the foreign key constraint in embedding_model table
+ op.drop_constraint(
+ "fk_embedding_model_cloud_provider", "embedding_model", type_="foreignkey"
+ )
+
+ # Drop the existing primary key constraint
+ op.drop_constraint("embedding_provider_pkey", "embedding_provider", type_="primary")
+
+ # Create a new primary key constraint on provider_type
+ op.create_primary_key(
+ "embedding_provider_pkey", "embedding_provider", ["provider_type"]
+ )
+
+ # Add provider_type column to embedding_model
+ op.add_column(
+ "embedding_model",
+ sa.Column("provider_type", sa.String(50), nullable=True),
+ )
+
+ # Update provider_type for existing embedding models
+ op.execute(
+ """
+ UPDATE embedding_model
+ SET provider_type = (
+ SELECT provider_type
+ FROM embedding_provider
+ WHERE embedding_provider.id = embedding_model.cloud_provider_id
+ )
+ """
+ )
+
+ # Drop the old id column from embedding_provider
+ op.drop_column("embedding_provider", "id")
+
+ # Drop the name column from embedding_provider
+ op.drop_column("embedding_provider", "name")
+
+ # Drop the default_model_id column from embedding_provider
+ op.drop_column("embedding_provider", "default_model_id")
+
+ # Drop the old cloud_provider_id column from embedding_model
+ op.drop_column("embedding_model", "cloud_provider_id")
+
+ # Create the new foreign key constraint
+ op.create_foreign_key(
+ "fk_embedding_model_cloud_provider",
+ "embedding_model",
+ "embedding_provider",
+ ["provider_type"],
+ ["provider_type"],
+ )
+
+
+def downgrade() -> None:
+ # Drop the foreign key constraint in embedding_model table
+ op.drop_constraint(
+ "fk_embedding_model_cloud_provider", "embedding_model", type_="foreignkey"
+ )
+
+ # Add back the cloud_provider_id column to embedding_model
+ op.add_column(
+ "embedding_model", sa.Column("cloud_provider_id", sa.Integer(), nullable=True)
+ )
+ op.add_column("embedding_provider", sa.Column("id", sa.Integer(), nullable=True))
+
+ # Assign incrementing IDs to embedding providers
+ op.execute(
+ """
+ CREATE SEQUENCE IF NOT EXISTS embedding_provider_id_seq;"""
+ )
+ op.execute(
+ """
+ UPDATE embedding_provider SET id = nextval('embedding_provider_id_seq');
+ """
+ )
+
+ # Update cloud_provider_id based on provider_type
+ op.execute(
+ """
+ UPDATE embedding_model
+ SET cloud_provider_id = CASE
+ WHEN provider_type IS NULL THEN NULL
+ ELSE (
+ SELECT id
+ FROM embedding_provider
+ WHERE embedding_provider.provider_type = embedding_model.provider_type
+ )
+ END
+ """
+ )
+
+ # Drop the provider_type column from embedding_model
+ op.drop_column("embedding_model", "provider_type")
+
+ # Add back the columns to embedding_provider
+ op.add_column("embedding_provider", sa.Column("name", sa.String(50), nullable=True))
+ op.add_column(
+ "embedding_provider", sa.Column("default_model_id", sa.Integer(), nullable=True)
+ )
+
+ # Drop the existing primary key constraint on provider_type
+ op.drop_constraint("embedding_provider_pkey", "embedding_provider", type_="primary")
+
+ # Create the original primary key constraint on id
+ op.create_primary_key("embedding_provider_pkey", "embedding_provider", ["id"])
+
+ # Update name with existing provider_type values
+ op.execute(
+ """
+ UPDATE embedding_provider
+ SET name = CASE
+ WHEN provider_type = 'OPENAI' THEN 'OpenAI'
+ WHEN provider_type = 'COHERE' THEN 'Cohere'
+ WHEN provider_type = 'GOOGLE' THEN 'Google'
+ WHEN provider_type = 'VOYAGE' THEN 'Voyage'
+ ELSE provider_type
+ END
+ """
+ )
+
+ # Drop the provider_type column from embedding_provider
+ op.drop_column("embedding_provider", "provider_type")
+
+ # Recreate the foreign key constraint in embedding_model table
+ op.create_foreign_key(
+ "fk_embedding_model_cloud_provider",
+ "embedding_model",
+ "embedding_provider",
+ ["cloud_provider_id"],
+ ["id"],
+ )
+
+ # Recreate the foreign key constraint in embedding_model table
+ op.create_foreign_key(
+ "fk_embedding_provider_default_model",
+ "embedding_provider",
+ "embedding_model",
+ ["default_model_id"],
+ ["id"],
+ )
diff --git a/backend/danswer/access/access.py b/backend/danswer/access/access.py
index 51f5a300c49..5501980ab48 100644
--- a/backend/danswer/access/access.py
+++ b/backend/danswer/access/access.py
@@ -5,19 +5,16 @@
from danswer.configs.constants import PUBLIC_DOC_PAT
from danswer.db.document import get_acccess_info_for_documents
from danswer.db.models import User
-from danswer.server.documents.models import ConnectorCredentialPairIdentifier
from danswer.utils.variable_functionality import fetch_versioned_implementation
def _get_access_for_documents(
document_ids: list[str],
db_session: Session,
- cc_pair_to_delete: ConnectorCredentialPairIdentifier | None = None,
) -> dict[str, DocumentAccess]:
document_access_info = get_acccess_info_for_documents(
db_session=db_session,
document_ids=document_ids,
- cc_pair_to_delete=cc_pair_to_delete,
)
return {
document_id: DocumentAccess.build(user_ids, [], is_public)
@@ -28,14 +25,13 @@ def _get_access_for_documents(
def get_access_for_documents(
document_ids: list[str],
db_session: Session,
- cc_pair_to_delete: ConnectorCredentialPairIdentifier | None = None,
) -> dict[str, DocumentAccess]:
"""Fetches all access information for the given documents."""
versioned_get_access_for_documents_fn = fetch_versioned_implementation(
"danswer.access.access", "_get_access_for_documents"
)
return versioned_get_access_for_documents_fn(
- document_ids, db_session, cc_pair_to_delete
+ document_ids, db_session
) # type: ignore
diff --git a/backend/danswer/auth/invited_users.py b/backend/danswer/auth/invited_users.py
index 56a02fc60c4..efce858f265 100644
--- a/backend/danswer/auth/invited_users.py
+++ b/backend/danswer/auth/invited_users.py
@@ -1,21 +1,20 @@
from typing import cast
+from danswer.configs.constants import KV_USER_STORE_KEY
from danswer.dynamic_configs.factory import get_dynamic_config_store
from danswer.dynamic_configs.interface import ConfigNotFoundError
from danswer.dynamic_configs.interface import JSON_ro
-USER_STORE_KEY = "INVITED_USERS"
-
def get_invited_users() -> list[str]:
try:
store = get_dynamic_config_store()
- return cast(list, store.load(USER_STORE_KEY))
+ return cast(list, store.load(KV_USER_STORE_KEY))
except ConfigNotFoundError:
return list()
def write_invited_users(emails: list[str]) -> int:
store = get_dynamic_config_store()
- store.store(USER_STORE_KEY, cast(JSON_ro, emails))
+ store.store(KV_USER_STORE_KEY, cast(JSON_ro, emails))
return len(emails)
diff --git a/backend/danswer/auth/noauth_user.py b/backend/danswer/auth/noauth_user.py
index 4744c4a6488..9520ef41c23 100644
--- a/backend/danswer/auth/noauth_user.py
+++ b/backend/danswer/auth/noauth_user.py
@@ -3,29 +3,27 @@
from typing import cast
from danswer.auth.schemas import UserRole
+from danswer.configs.constants import KV_NO_AUTH_USER_PREFERENCES_KEY
from danswer.dynamic_configs.store import ConfigNotFoundError
from danswer.dynamic_configs.store import DynamicConfigStore
from danswer.server.manage.models import UserInfo
from danswer.server.manage.models import UserPreferences
-NO_AUTH_USER_PREFERENCES_KEY = "no_auth_user_preferences"
-
-
def set_no_auth_user_preferences(
store: DynamicConfigStore, preferences: UserPreferences
) -> None:
- store.store(NO_AUTH_USER_PREFERENCES_KEY, preferences.dict())
+ store.store(KV_NO_AUTH_USER_PREFERENCES_KEY, preferences.model_dump())
def load_no_auth_user_preferences(store: DynamicConfigStore) -> UserPreferences:
try:
preferences_data = cast(
- Mapping[str, Any], store.load(NO_AUTH_USER_PREFERENCES_KEY)
+ Mapping[str, Any], store.load(KV_NO_AUTH_USER_PREFERENCES_KEY)
)
return UserPreferences(**preferences_data)
except ConfigNotFoundError:
- return UserPreferences(chosen_assistants=None)
+ return UserPreferences(chosen_assistants=None, default_model=None)
def fetch_no_auth_user(store: DynamicConfigStore) -> UserInfo:
diff --git a/backend/danswer/auth/schemas.py b/backend/danswer/auth/schemas.py
index 79d9a7f8098..9e0553991cc 100644
--- a/backend/danswer/auth/schemas.py
+++ b/backend/danswer/auth/schemas.py
@@ -5,8 +5,20 @@
class UserRole(str, Enum):
+ """
+ User roles
+ - Basic can't perform any admin actions
+ - Admin can perform all admin actions
+ - Curator can perform admin actions for
+ groups they are curators of
+ - Global Curator can perform admin actions
+ for all groups they are a member of
+ """
+
BASIC = "basic"
ADMIN = "admin"
+ CURATOR = "curator"
+ GLOBAL_CURATOR = "global_curator"
class UserStatus(str, Enum):
diff --git a/backend/danswer/auth/users.py b/backend/danswer/auth/users.py
index 479cf07df3d..c3851ff1990 100644
--- a/backend/danswer/auth/users.py
+++ b/backend/danswer/auth/users.py
@@ -1,11 +1,15 @@
import smtplib
import uuid
from collections.abc import AsyncGenerator
+from datetime import datetime
+from datetime import timezone
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from typing import Optional
from typing import Tuple
+from email_validator import EmailNotValidError
+from email_validator import validate_email
from fastapi import APIRouter
from fastapi import Depends
from fastapi import HTTPException
@@ -38,6 +42,7 @@
from danswer.configs.app_configs import SMTP_PORT
from danswer.configs.app_configs import SMTP_SERVER
from danswer.configs.app_configs import SMTP_USER
+from danswer.configs.app_configs import TRACK_EXTERNAL_IDP_EXPIRY
from danswer.configs.app_configs import USER_AUTH_SECRET
from danswer.configs.app_configs import VALID_EMAIL_DOMAINS
from danswer.configs.app_configs import WEB_DOMAIN
@@ -50,26 +55,50 @@
from danswer.db.auth import get_user_count
from danswer.db.auth import get_user_db
from danswer.db.engine import get_session
+from danswer.db.engine import get_sqlalchemy_engine
from danswer.db.models import AccessToken
from danswer.db.models import User
+from danswer.db.users import get_user_by_email
from danswer.utils.logger import setup_logger
from danswer.utils.telemetry import optional_telemetry
from danswer.utils.telemetry import RecordType
-from danswer.utils.variable_functionality import (
- fetch_versioned_implementation,
-)
-
+from danswer.utils.variable_functionality import fetch_versioned_implementation
logger = setup_logger()
+def validate_curator_request(groups: list | None, is_public: bool) -> None:
+ if is_public:
+ detail = "Curators cannot create public objects"
+ logger.error(detail)
+ raise HTTPException(
+ status_code=401,
+ detail=detail,
+ )
+ if not groups:
+ detail = "Curators must specify 1+ groups"
+ logger.error(detail)
+ raise HTTPException(
+ status_code=401,
+ detail=detail,
+ )
+
+
+def is_user_admin(user: User | None) -> bool:
+ if AUTH_TYPE == AuthType.DISABLED:
+ return True
+ if user and user.role == UserRole.ADMIN:
+ return True
+ return False
+
+
def verify_auth_setting() -> None:
if AUTH_TYPE not in [AuthType.DISABLED, AuthType.BASIC, AuthType.GOOGLE_OAUTH]:
raise ValueError(
"User must choose a valid user authentication method: "
"disabled, basic, or google_oauth"
)
- logger.info(f"Using Auth Type: {AUTH_TYPE.value}")
+ logger.notice(f"Using Auth Type: {AUTH_TYPE.value}")
def get_display_email(email: str | None, space_less: bool = False) -> str:
@@ -92,10 +121,36 @@ def user_needs_to_be_verified() -> bool:
return AUTH_TYPE != AuthType.BASIC or REQUIRE_EMAIL_VERIFICATION
-def verify_email_in_whitelist(email: str) -> None:
+def verify_email_is_invited(email: str) -> None:
whitelist = get_invited_users()
- if (whitelist and email not in whitelist) or not email:
- raise PermissionError("User not on allowed user whitelist")
+ if not whitelist:
+ return
+
+ if not email:
+ raise PermissionError("Email must be specified")
+
+ email_info = validate_email(email) # can raise EmailNotValidError
+
+ for email_whitelist in whitelist:
+ try:
+ # normalized emails are now being inserted into the db
+ # we can remove this normalization on read after some time has passed
+ email_info_whitelist = validate_email(email_whitelist)
+ except EmailNotValidError:
+ continue
+
+ # oddly, normalization does not include lowercasing the user part of the
+ # email address ... which we want to allow
+ if email_info.normalized.lower() == email_info_whitelist.normalized.lower():
+ return
+
+ raise PermissionError("User not on allowed user whitelist")
+
+
+def verify_email_in_whitelist(email: str) -> None:
+ with Session(get_sqlalchemy_engine()) as db_session:
+ if not get_user_by_email(email, db_session):
+ verify_email_is_invited(email)
def verify_email_domain(email: str) -> None:
@@ -147,7 +202,7 @@ async def create(
safe: bool = False,
request: Optional[Request] = None,
) -> models.UP:
- verify_email_in_whitelist(user_create.email)
+ verify_email_is_invited(user_create.email)
verify_email_domain(user_create.email)
if hasattr(user_create, "role"):
user_count = await get_user_count()
@@ -172,7 +227,7 @@ async def oauth_callback(
) -> models.UOAP:
verify_email_domain(account_email)
- return await super().oauth_callback( # type: ignore
+ user = await super().oauth_callback( # type: ignore
oauth_name=oauth_name,
access_token=access_token,
account_id=account_id,
@@ -184,10 +239,23 @@ async def oauth_callback(
is_verified_by_default=is_verified_by_default,
)
+ # NOTE: Most IdPs have very short expiry times, and we don't want to force the user to
+ # re-authenticate that frequently, so by default this is disabled
+ if expires_at and TRACK_EXTERNAL_IDP_EXPIRY:
+ oidc_expiry = datetime.fromtimestamp(expires_at, tz=timezone.utc)
+ await self.user_db.update(user, update_dict={"oidc_expiry": oidc_expiry})
+
+ # this is needed if an organization goes from `TRACK_EXTERNAL_IDP_EXPIRY=true` to `false`
+ # otherwise, the oidc expiry will always be old, and the user will never be able to login
+ if user.oidc_expiry and not TRACK_EXTERNAL_IDP_EXPIRY:
+ await self.user_db.update(user, update_dict={"oidc_expiry": None})
+
+ return user
+
async def on_after_register(
self, user: User, request: Optional[Request] = None
) -> None:
- logger.info(f"User {user.id} has registered.")
+ logger.notice(f"User {user.id} has registered.")
optional_telemetry(
record_type=RecordType.SIGN_UP,
data={"action": "create"},
@@ -197,14 +265,14 @@ async def on_after_register(
async def on_after_forgot_password(
self, user: User, token: str, request: Optional[Request] = None
) -> None:
- logger.info(f"User {user.id} has forgot their password. Reset token: {token}")
+ logger.notice(f"User {user.id} has forgot their password. Reset token: {token}")
async def on_after_request_verify(
self, user: User, token: str, request: Optional[Request] = None
) -> None:
verify_email_domain(user.email)
- logger.info(
+ logger.notice(
f"Verification requested for user {user.id}. Verification token: {token}"
)
@@ -226,10 +294,12 @@ async def get_user_manager(
def get_database_strategy(
access_token_db: AccessTokenDatabase[AccessToken] = Depends(get_access_token_db),
) -> DatabaseStrategy:
- return DatabaseStrategy(
+ strategy = DatabaseStrategy(
access_token_db, lifetime_seconds=SESSION_EXPIRE_TIME_SECONDS # type: ignore
)
+ return strategy
+
auth_backend = AuthenticationBackend(
name="database",
@@ -326,6 +396,12 @@ async def double_check_user(
detail="Access denied. User is not verified.",
)
+ if user.oidc_expiry and user.oidc_expiry < datetime.now(timezone.utc):
+ raise HTTPException(
+ status_code=status.HTTP_403_FORBIDDEN,
+ detail="Access denied. User's OIDC token has expired.",
+ )
+
return user
@@ -335,6 +411,28 @@ async def current_user(
return await double_check_user(user)
+async def current_curator_or_admin_user(
+ user: User | None = Depends(current_user),
+) -> User | None:
+ if DISABLE_AUTH:
+ return None
+
+ if not user or not hasattr(user, "role"):
+ raise HTTPException(
+ status_code=status.HTTP_403_FORBIDDEN,
+ detail="Access denied. User is not authenticated or lacks role information.",
+ )
+
+ allowed_roles = {UserRole.GLOBAL_CURATOR, UserRole.CURATOR, UserRole.ADMIN}
+ if user.role not in allowed_roles:
+ raise HTTPException(
+ status_code=status.HTTP_403_FORBIDDEN,
+ detail="Access denied. User is not a curator or admin.",
+ )
+
+ return user
+
+
async def current_admin_user(user: User | None = Depends(current_user)) -> User | None:
if DISABLE_AUTH:
return None
@@ -342,6 +440,12 @@ async def current_admin_user(user: User | None = Depends(current_user)) -> User
if not user or not hasattr(user, "role") or user.role != UserRole.ADMIN:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
- detail="Access denied. User is not an admin.",
+ detail="Access denied. User must be an admin to perform this action.",
)
+
return user
+
+
+def get_default_admin_user_emails_() -> list[str]:
+ # No default seeding available for Danswer MIT
+ return []
diff --git a/backend/danswer/background/celery/celery_app.py b/backend/danswer/background/celery/celery_app.py
index 1a678ea11fa..1c0e949d068 100644
--- a/backend/danswer/background/celery/celery_app.py
+++ b/backend/danswer/background/celery/celery_app.py
@@ -1,10 +1,16 @@
+import json
from datetime import timedelta
+from typing import Any
from typing import cast
from celery import Celery # type: ignore
+from celery.contrib.abortable import AbortableTask # type: ignore
+from celery.exceptions import TaskRevokedError
+from sqlalchemy import text
from sqlalchemy.orm import Session
from danswer.background.celery.celery_utils import extract_ids_from_runnable_connector
+from danswer.background.celery.celery_utils import should_kick_off_deletion_of_cc_pair
from danswer.background.celery.celery_utils import should_prune_cc_pair
from danswer.background.celery.celery_utils import should_sync_doc_set
from danswer.background.connector_deletion import delete_connector_credential_pair
@@ -14,6 +20,8 @@
from danswer.background.task_utils import name_cc_prune_task
from danswer.background.task_utils import name_document_set_sync_task
from danswer.configs.app_configs import JOB_TIMEOUT
+from danswer.configs.constants import POSTGRES_CELERY_APP_NAME
+from danswer.configs.constants import PostgresAdvisoryLocks
from danswer.connectors.factory import instantiate_connector
from danswer.connectors.models import InputType
from danswer.db.connector_credential_pair import get_connector_credential_pair
@@ -38,7 +46,9 @@
logger = setup_logger()
-connection_string = build_connection_string(db_api=SYNC_DB_API)
+connection_string = build_connection_string(
+ db_api=SYNC_DB_API, app_name=POSTGRES_CELERY_APP_NAME
+)
celery_broker_url = f"sqla+{connection_string}"
celery_backend_url = f"db+{connection_string}"
celery_app = Celery(__name__, broker=celery_broker_url, backend=celery_backend_url)
@@ -100,7 +110,7 @@ def cleanup_connector_credential_pair_task(
@build_celery_task_wrapper(name_cc_prune_task)
@celery_app.task(soft_time_limit=JOB_TIMEOUT)
def prune_documents_task(connector_id: int, credential_id: int) -> None:
- """connector pruning task. For a cc pair, this task pulls all docuement IDs from the source
+ """connector pruning task. For a cc pair, this task pulls all document IDs from the source
and compares those IDs to locally stored documents and deletes all locally stored IDs missing
from the most recently pulled document ID list"""
with Session(get_sqlalchemy_engine()) as db_session:
@@ -278,6 +288,141 @@ def check_for_document_sets_sync_task() -> None:
)
+@celery_app.task(
+ name="check_for_cc_pair_deletion_task",
+ soft_time_limit=JOB_TIMEOUT,
+)
+def check_for_cc_pair_deletion_task() -> None:
+ """Runs periodically to check if any deletion tasks should be run"""
+ with Session(get_sqlalchemy_engine()) as db_session:
+ # check if any document sets are not synced
+ cc_pairs = get_connector_credential_pairs(db_session)
+ for cc_pair in cc_pairs:
+ if should_kick_off_deletion_of_cc_pair(cc_pair, db_session):
+ logger.notice(f"Deleting the {cc_pair.name} connector credential pair")
+ cleanup_connector_credential_pair_task.apply_async(
+ kwargs=dict(
+ connector_id=cc_pair.connector.id,
+ credential_id=cc_pair.credential.id,
+ ),
+ )
+
+
+@celery_app.task(
+ name="kombu_message_cleanup_task",
+ soft_time_limit=JOB_TIMEOUT,
+ bind=True,
+ base=AbortableTask,
+)
+def kombu_message_cleanup_task(self: Any) -> int:
+ """Runs periodically to clean up the kombu_message table"""
+
+ # we will select messages older than this amount to clean up
+ KOMBU_MESSAGE_CLEANUP_AGE = 7 # days
+ KOMBU_MESSAGE_CLEANUP_PAGE_LIMIT = 1000
+
+ ctx = {}
+ ctx["last_processed_id"] = 0
+ ctx["deleted"] = 0
+ ctx["cleanup_age"] = KOMBU_MESSAGE_CLEANUP_AGE
+ ctx["page_limit"] = KOMBU_MESSAGE_CLEANUP_PAGE_LIMIT
+ with Session(get_sqlalchemy_engine()) as db_session:
+ # Exit the task if we can't take the advisory lock
+ result = db_session.execute(
+ text("SELECT pg_try_advisory_lock(:id)"),
+ {"id": PostgresAdvisoryLocks.KOMBU_MESSAGE_CLEANUP_LOCK_ID.value},
+ ).scalar()
+ if not result:
+ return 0
+
+ while True:
+ if self.is_aborted():
+ raise TaskRevokedError("kombu_message_cleanup_task was aborted.")
+
+ b = kombu_message_cleanup_task_helper(ctx, db_session)
+ if not b:
+ break
+
+ db_session.commit()
+
+ if ctx["deleted"] > 0:
+ logger.info(f"Deleted {ctx['deleted']} orphaned messages from kombu_message.")
+
+ return ctx["deleted"]
+
+
+def kombu_message_cleanup_task_helper(ctx: dict, db_session: Session) -> bool:
+ """
+ Helper function to clean up old messages from the `kombu_message` table that are no longer relevant.
+
+ This function retrieves messages from the `kombu_message` table that are no longer visible and
+ older than a specified interval. It checks if the corresponding task_id exists in the
+ `celery_taskmeta` table. If the task_id does not exist, the message is deleted.
+
+ Args:
+ ctx (dict): A context dictionary containing configuration parameters such as:
+ - 'cleanup_age' (int): The age in days after which messages are considered old.
+ - 'page_limit' (int): The maximum number of messages to process in one batch.
+ - 'last_processed_id' (int): The ID of the last processed message to handle pagination.
+ - 'deleted' (int): A counter to track the number of deleted messages.
+ db_session (Session): The SQLAlchemy database session for executing queries.
+
+ Returns:
+ bool: Returns True if there are more rows to process, False if not.
+ """
+
+ query = text(
+ """
+ SELECT id, timestamp, payload
+ FROM kombu_message WHERE visible = 'false'
+ AND timestamp < CURRENT_TIMESTAMP - INTERVAL :interval_days
+ AND id > :last_processed_id
+ ORDER BY id
+ LIMIT :page_limit
+"""
+ )
+ kombu_messages = db_session.execute(
+ query,
+ {
+ "interval_days": f"{ctx['cleanup_age']} days",
+ "page_limit": ctx["page_limit"],
+ "last_processed_id": ctx["last_processed_id"],
+ },
+ ).fetchall()
+
+ if len(kombu_messages) == 0:
+ return False
+
+ for msg in kombu_messages:
+ payload = json.loads(msg[2])
+ task_id = payload["headers"]["id"]
+
+ # Check if task_id exists in celery_taskmeta
+ task_exists = db_session.execute(
+ text("SELECT 1 FROM celery_taskmeta WHERE task_id = :task_id"),
+ {"task_id": task_id},
+ ).fetchone()
+
+ # If task_id does not exist, delete the message
+ if not task_exists:
+ result = db_session.execute(
+ text("DELETE FROM kombu_message WHERE id = :message_id"),
+ {"message_id": msg[0]},
+ )
+ if result.rowcount > 0: # type: ignore
+ ctx["deleted"] += 1
+ else:
+ task_name = payload["headers"]["task"]
+ logger.warning(
+ f"Message found for task older than {ctx['cleanup_age']} days. "
+ f"id={task_id} name={task_name}"
+ )
+
+ ctx["last_processed_id"] = msg[0]
+
+ return True
+
+
@celery_app.task(
name="check_for_prune_task",
soft_time_limit=JOB_TIMEOUT,
@@ -313,6 +458,12 @@ def check_for_prune_task() -> None:
"task": "check_for_document_sets_sync_task",
"schedule": timedelta(seconds=5),
},
+ "check-for-cc-pair-deletion": {
+ "task": "check_for_cc_pair_deletion_task",
+ # don't need to check too often, since we kick off a deletion initially
+ # during the API call that actually marks the CC pair for deletion
+ "schedule": timedelta(minutes=1),
+ },
}
celery_app.conf.beat_schedule.update(
{
@@ -322,3 +473,11 @@ def check_for_prune_task() -> None:
},
}
)
+celery_app.conf.beat_schedule.update(
+ {
+ "kombu-message-cleanup": {
+ "task": "kombu_message_cleanup_task",
+ "schedule": timedelta(seconds=3600),
+ },
+ }
+)
diff --git a/backend/danswer/background/celery/celery_utils.py b/backend/danswer/background/celery/celery_utils.py
index 6b9b5a89603..e4d4d13bb1d 100644
--- a/backend/danswer/background/celery/celery_utils.py
+++ b/backend/danswer/background/celery/celery_utils.py
@@ -6,8 +6,8 @@
from danswer.background.task_utils import name_cc_cleanup_task
from danswer.background.task_utils import name_cc_prune_task
from danswer.background.task_utils import name_document_set_sync_task
+from danswer.configs.app_configs import ALLOW_SIMULTANEOUS_PRUNING
from danswer.configs.app_configs import MAX_PRUNING_DOCUMENT_RETRIEVAL_PER_MINUTE
-from danswer.configs.app_configs import PREVENT_SIMULTANEOUS_PRUNING
from danswer.connectors.cross_connector_utils.rate_limit_wrapper import (
rate_limit_builder,
)
@@ -16,10 +16,14 @@
from danswer.connectors.interfaces import LoadConnector
from danswer.connectors.interfaces import PollConnector
from danswer.connectors.models import Document
+from danswer.db.deletion_attempt import check_deletion_attempt_is_allowed
from danswer.db.engine import get_db_current_time
+from danswer.db.enums import ConnectorCredentialPairStatus
from danswer.db.models import Connector
+from danswer.db.models import ConnectorCredentialPair
from danswer.db.models import Credential
from danswer.db.models import DocumentSet
+from danswer.db.models import TaskQueueState
from danswer.db.tasks import check_task_is_live_and_not_timed_out
from danswer.db.tasks import get_latest_task
from danswer.db.tasks import get_latest_task_by_type
@@ -29,24 +33,54 @@
logger = setup_logger()
-def get_deletion_status(
+def _get_deletion_status(
connector_id: int, credential_id: int, db_session: Session
-) -> DeletionAttemptSnapshot | None:
+) -> TaskQueueState | None:
cleanup_task_name = name_cc_cleanup_task(
connector_id=connector_id, credential_id=credential_id
)
- task_state = get_latest_task(task_name=cleanup_task_name, db_session=db_session)
+ return get_latest_task(task_name=cleanup_task_name, db_session=db_session)
+
- if not task_state:
+def get_deletion_attempt_snapshot(
+ connector_id: int, credential_id: int, db_session: Session
+) -> DeletionAttemptSnapshot | None:
+ deletion_task = _get_deletion_status(connector_id, credential_id, db_session)
+ if not deletion_task:
return None
return DeletionAttemptSnapshot(
connector_id=connector_id,
credential_id=credential_id,
- status=task_state.status,
+ status=deletion_task.status,
)
+def should_kick_off_deletion_of_cc_pair(
+ cc_pair: ConnectorCredentialPair, db_session: Session
+) -> bool:
+ if cc_pair.status != ConnectorCredentialPairStatus.DELETING:
+ return False
+
+ if check_deletion_attempt_is_allowed(cc_pair, db_session):
+ return False
+
+ deletion_task = _get_deletion_status(
+ connector_id=cc_pair.connector_id,
+ credential_id=cc_pair.credential_id,
+ db_session=db_session,
+ )
+ if deletion_task and check_task_is_live_and_not_timed_out(
+ deletion_task,
+ db_session,
+ # 1 hour timeout
+ timeout=60 * 60,
+ ):
+ return False
+
+ return True
+
+
def should_sync_doc_set(document_set: DocumentSet, db_session: Session) -> bool:
if document_set.is_up_to_date:
return False
@@ -58,7 +92,7 @@ def should_sync_doc_set(document_set: DocumentSet, db_session: Session) -> bool:
logger.info(f"Document set '{document_set.id}' is already syncing. Skipping.")
return False
- logger.info(f"Document set {document_set.id} syncing now!")
+ logger.info(f"Document set {document_set.id} syncing now.")
return True
@@ -80,7 +114,7 @@ def should_prune_cc_pair(
return True
return False
- if PREVENT_SIMULTANEOUS_PRUNING:
+ if not ALLOW_SIMULTANEOUS_PRUNING:
pruning_type_task_name = name_cc_prune_task()
last_pruning_type_task = get_latest_task_by_type(
pruning_type_task_name, db_session
@@ -89,11 +123,9 @@ def should_prune_cc_pair(
if last_pruning_type_task and check_task_is_live_and_not_timed_out(
last_pruning_type_task, db_session
):
- logger.info("Another Connector is already pruning. Skipping.")
return False
if check_task_is_live_and_not_timed_out(last_pruning_task, db_session):
- logger.info(f"Connector '{connector.name}' is already pruning. Skipping.")
return False
if not last_pruning_task.start_time:
diff --git a/backend/danswer/background/connector_deletion.py b/backend/danswer/background/connector_deletion.py
index 28c58f02dfc..90883564910 100644
--- a/backend/danswer/background/connector_deletion.py
+++ b/backend/danswer/background/connector_deletion.py
@@ -10,8 +10,6 @@
connector / credential pair from the access list
(6) delete all relevant entries from postgres
"""
-import time
-
from sqlalchemy.orm import Session
from danswer.access.access import get_access_for_documents
@@ -24,10 +22,8 @@
from danswer.db.document import get_document_connector_cnts
from danswer.db.document import get_documents_for_connector_credential_pair
from danswer.db.document import prepare_to_modify_documents
-from danswer.db.document_set import get_document_sets_by_ids
-from danswer.db.document_set import (
- mark_cc_pair__document_set_relationships_to_be_deleted__no_commit,
-)
+from danswer.db.document_set import delete_document_set_cc_pair_relationship__no_commit
+from danswer.db.document_set import fetch_document_sets_for_documents
from danswer.db.engine import get_sqlalchemy_engine
from danswer.db.index_attempt import delete_index_attempts
from danswer.db.models import ConnectorCredentialPair
@@ -35,6 +31,10 @@
from danswer.document_index.interfaces import UpdateRequest
from danswer.server.documents.models import ConnectorCredentialPairIdentifier
from danswer.utils.logger import setup_logger
+from danswer.utils.variable_functionality import (
+ fetch_versioned_implementation_with_fallback,
+)
+from danswer.utils.variable_functionality import noop_fallback
logger = setup_logger()
@@ -78,25 +78,37 @@ def delete_connector_credential_pair_batch(
document_ids_to_update = [
document_id for document_id, cnt in document_connector_cnts if cnt > 1
]
+
+ # maps document id to list of document set names
+ new_doc_sets_for_documents: dict[str, set[str]] = {
+ document_id_and_document_set_names_tuple[0]: set(
+ document_id_and_document_set_names_tuple[1]
+ )
+ for document_id_and_document_set_names_tuple in fetch_document_sets_for_documents(
+ db_session=db_session,
+ document_ids=document_ids_to_update,
+ )
+ }
+
+ # determine future ACLs for documents in batch
access_for_documents = get_access_for_documents(
document_ids=document_ids_to_update,
db_session=db_session,
- cc_pair_to_delete=ConnectorCredentialPairIdentifier(
- connector_id=connector_id,
- credential_id=credential_id,
- ),
)
+
+ # update Vespa
+ logger.debug(f"Updating documents: {document_ids_to_update}")
update_requests = [
UpdateRequest(
document_ids=[document_id],
access=access,
+ document_sets=new_doc_sets_for_documents[document_id],
)
for document_id, access in access_for_documents.items()
]
- logger.debug(f"Updating documents: {document_ids_to_update}")
-
document_index.update(update_requests=update_requests)
+ # clean up Postgres
delete_document_by_connector_credential_pair__no_commit(
db_session=db_session,
document_ids=document_ids_to_update,
@@ -108,48 +120,6 @@ def delete_connector_credential_pair_batch(
db_session.commit()
-def cleanup_synced_entities(
- cc_pair: ConnectorCredentialPair, db_session: Session
-) -> None:
- """Updates the document sets associated with the connector / credential pair,
- then relies on the document set sync script to kick off Celery jobs which will
- sync these updates to Vespa.
-
- Waits until the document sets are synced before returning."""
- logger.info(f"Cleaning up Document Sets for CC Pair with ID: '{cc_pair.id}'")
- document_sets_ids_to_sync = list(
- mark_cc_pair__document_set_relationships_to_be_deleted__no_commit(
- cc_pair_id=cc_pair.id,
- db_session=db_session,
- )
- )
- db_session.commit()
-
- # wait till all document sets are synced before continuing
- while True:
- all_synced = True
- document_sets = get_document_sets_by_ids(
- db_session=db_session, document_set_ids=document_sets_ids_to_sync
- )
- for document_set in document_sets:
- if not document_set.is_up_to_date:
- all_synced = False
-
- if all_synced:
- break
-
- # wait for 30 seconds before checking again
- db_session.commit() # end transaction
- logger.info(
- f"Document sets '{document_sets_ids_to_sync}' not synced yet, waiting 30s"
- )
- time.sleep(30)
-
- logger.info(
- f"Finished cleaning up Document Sets for CC Pair with ID: '{cc_pair.id}'"
- )
-
-
def delete_connector_credential_pair(
db_session: Session,
document_index: DocumentIndex,
@@ -177,17 +147,33 @@ def delete_connector_credential_pair(
)
num_docs_deleted += len(documents)
- # Clean up document sets / access information from Postgres
- # and sync these updates to Vespa
- # TODO: add user group cleanup with `fetch_versioned_implementation`
- cleanup_synced_entities(cc_pair, db_session)
-
# clean up the rest of the related Postgres entities
+ # index attempts
delete_index_attempts(
db_session=db_session,
connector_id=connector_id,
credential_id=credential_id,
)
+
+ # document sets
+ delete_document_set_cc_pair_relationship__no_commit(
+ db_session=db_session,
+ connector_id=connector_id,
+ credential_id=credential_id,
+ )
+
+ # user groups
+ cleanup_user_groups = fetch_versioned_implementation_with_fallback(
+ "danswer.db.user_group",
+ "delete_user_group_cc_pair_relationship__no_commit",
+ noop_fallback,
+ )
+ cleanup_user_groups(
+ cc_pair_id=cc_pair.id,
+ db_session=db_session,
+ )
+
+ # finally, delete the cc-pair
delete_connector_credential_pair__no_commit(
db_session=db_session,
connector_id=connector_id,
@@ -199,11 +185,11 @@ def delete_connector_credential_pair(
connector_id=connector_id,
)
if not connector or not len(connector.credentials):
- logger.debug("Found no credentials left for connector, deleting connector")
+ logger.info("Found no credentials left for connector, deleting connector")
db_session.delete(connector)
db_session.commit()
- logger.info(
+ logger.notice(
"Successfully deleted connector_credential_pair with connector_id:"
f" '{connector_id}' and credential_id: '{credential_id}'. Deleted {num_docs_deleted} docs."
)
diff --git a/backend/danswer/background/indexing/job_client.py b/backend/danswer/background/indexing/job_client.py
index 72919d690a4..68d706895fd 100644
--- a/backend/danswer/background/indexing/job_client.py
+++ b/backend/danswer/background/indexing/job_client.py
@@ -41,6 +41,12 @@ def _initializer(
return func(*args, **kwargs)
+def _run_in_process(
+ func: Callable, args: list | tuple, kwargs: dict[str, Any] | None = None
+) -> None:
+ _initializer(func, args, kwargs)
+
+
@dataclass
class SimpleJob:
"""Drop in replacement for `dask.distributed.Future`"""
@@ -113,7 +119,7 @@ def submit(self, func: Callable, *args: Any, pure: bool = True) -> SimpleJob | N
job_id = self.job_id_counter
self.job_id_counter += 1
- process = Process(target=_initializer(func=func, args=args), daemon=True)
+ process = Process(target=_run_in_process, args=(func, args), daemon=True)
job = SimpleJob(id=job_id, process=process)
process.start()
diff --git a/backend/danswer/background/indexing/run_indexing.py b/backend/danswer/background/indexing/run_indexing.py
index fa684f020b6..a98f4e1f5ad 100644
--- a/backend/danswer/background/indexing/run_indexing.py
+++ b/backend/danswer/background/indexing/run_indexing.py
@@ -7,20 +7,21 @@
from sqlalchemy.orm import Session
from danswer.background.indexing.checkpointing import get_time_windows_for_index_attempt
+from danswer.background.indexing.tracer import DanswerTracer
+from danswer.configs.app_configs import INDEXING_SIZE_WARNING_THRESHOLD
+from danswer.configs.app_configs import INDEXING_TRACER_INTERVAL
from danswer.configs.app_configs import POLL_CONNECTOR_OFFSET
+from danswer.connectors.connector_runner import ConnectorRunner
from danswer.connectors.factory import instantiate_connector
-from danswer.connectors.interfaces import GenerateDocumentsOutput
-from danswer.connectors.interfaces import LoadConnector
-from danswer.connectors.interfaces import PollConnector
from danswer.connectors.models import IndexAttemptMetadata
-from danswer.connectors.models import InputType
-from danswer.db.connector import disable_connector
from danswer.db.connector_credential_pair import get_last_successful_attempt_time
from danswer.db.connector_credential_pair import update_connector_credential_pair
from danswer.db.engine import get_sqlalchemy_engine
+from danswer.db.enums import ConnectorCredentialPairStatus
from danswer.db.index_attempt import get_index_attempt
from danswer.db.index_attempt import mark_attempt_failed
-from danswer.db.index_attempt import mark_attempt_in_progress__no_commit
+from danswer.db.index_attempt import mark_attempt_in_progress
+from danswer.db.index_attempt import mark_attempt_partially_succeeded
from danswer.db.index_attempt import mark_attempt_succeeded
from danswer.db.index_attempt import update_docs_indexed
from danswer.db.models import IndexAttempt
@@ -35,13 +36,15 @@
logger = setup_logger()
+INDEXING_TRACER_NUM_PRINT_ENTRIES = 5
-def _get_document_generator(
+
+def _get_connector_runner(
db_session: Session,
attempt: IndexAttempt,
start_time: datetime,
end_time: datetime,
-) -> GenerateDocumentsOutput:
+) -> ConnectorRunner:
"""
NOTE: `start_time` and `end_time` are only used for poll connectors
@@ -49,43 +52,31 @@ def _get_document_generator(
are the complete list of existing documents of the connector. If the task
of type LOAD_STATE, the list will be considered complete and otherwise incomplete.
"""
- task = attempt.connector.input_type
+ task = attempt.connector_credential_pair.connector.input_type
try:
runnable_connector = instantiate_connector(
- attempt.connector.source,
+ attempt.connector_credential_pair.connector.source,
task,
- attempt.connector.connector_specific_config,
- attempt.credential,
+ attempt.connector_credential_pair.connector.connector_specific_config,
+ attempt.connector_credential_pair.credential,
db_session,
)
except Exception as e:
logger.exception(f"Unable to instantiate connector due to {e}")
- disable_connector(attempt.connector.id, db_session)
- raise e
-
- if task == InputType.LOAD_STATE:
- assert isinstance(runnable_connector, LoadConnector)
- doc_batch_generator = runnable_connector.load_from_state()
-
- elif task == InputType.POLL:
- assert isinstance(runnable_connector, PollConnector)
- if attempt.connector_id is None or attempt.credential_id is None:
- raise ValueError(
- f"Polling attempt {attempt.id} is missing connector_id or credential_id, "
- f"can't fetch time range."
- )
-
- logger.info(f"Polling for updates between {start_time} and {end_time}")
- doc_batch_generator = runnable_connector.poll_source(
- start=start_time.timestamp(), end=end_time.timestamp()
+ # since we failed to even instantiate the connector, we pause the CCPair since
+ # it will never succeed
+ update_connector_credential_pair(
+ db_session=db_session,
+ connector_id=attempt.connector_credential_pair.connector.id,
+ credential_id=attempt.connector_credential_pair.credential.id,
+ status=ConnectorCredentialPairStatus.PAUSED,
)
+ raise e
- else:
- # Event types cannot be handled by a background type
- raise RuntimeError(f"Invalid task type: {task}")
-
- return doc_batch_generator
+ return ConnectorRunner(
+ connector=runnable_connector, time_range=(start_time, end_time)
+ )
def _run_indexing(
@@ -99,46 +90,62 @@ def _run_indexing(
"""
start_time = time.time()
- db_embedding_model = index_attempt.embedding_model
- index_name = db_embedding_model.index_name
+ search_settings = index_attempt.search_settings
+ index_name = search_settings.index_name
# Only update cc-pair status for primary index jobs
# Secondary index syncs at the end when swapping
- is_primary = index_attempt.embedding_model.status == IndexModelStatus.PRESENT
+ is_primary = search_settings.status == IndexModelStatus.PRESENT
# Indexing is only done into one index at a time
document_index = get_default_document_index(
primary_index_name=index_name, secondary_index_name=None
)
- embedding_model = DefaultIndexingEmbedder(
- model_name=db_embedding_model.model_name,
- normalize=db_embedding_model.normalize,
- query_prefix=db_embedding_model.query_prefix,
- passage_prefix=db_embedding_model.passage_prefix,
+ embedding_model = DefaultIndexingEmbedder.from_db_search_settings(
+ search_settings=search_settings
)
indexing_pipeline = build_indexing_pipeline(
+ attempt_id=index_attempt.id,
embedder=embedding_model,
document_index=document_index,
ignore_time_skip=index_attempt.from_beginning
- or (db_embedding_model.status == IndexModelStatus.FUTURE),
+ or (search_settings.status == IndexModelStatus.FUTURE),
db_session=db_session,
)
- db_connector = index_attempt.connector
- db_credential = index_attempt.credential
+ db_cc_pair = index_attempt.connector_credential_pair
+ db_connector = index_attempt.connector_credential_pair.connector
+ db_credential = index_attempt.connector_credential_pair.credential
+
last_successful_index_time = (
- 0.0
- if index_attempt.from_beginning
- else get_last_successful_attempt_time(
- connector_id=db_connector.id,
- credential_id=db_credential.id,
- embedding_model=index_attempt.embedding_model,
- db_session=db_session,
+ db_connector.indexing_start.timestamp()
+ if index_attempt.from_beginning and db_connector.indexing_start is not None
+ else (
+ 0.0
+ if index_attempt.from_beginning
+ else get_last_successful_attempt_time(
+ connector_id=db_connector.id,
+ credential_id=db_credential.id,
+ search_settings=index_attempt.search_settings,
+ db_session=db_session,
+ )
)
)
+ if INDEXING_TRACER_INTERVAL > 0:
+ logger.debug(f"Memory tracer starting: interval={INDEXING_TRACER_INTERVAL}")
+ tracer = DanswerTracer()
+ tracer.start()
+ tracer.snap()
+
+ index_attempt_md = IndexAttemptMetadata(
+ connector_id=db_connector.id,
+ credential_id=db_credential.id,
+ )
+
+ batch_num = 0
net_doc_change = 0
document_count = 0
chunk_count = 0
@@ -157,7 +164,7 @@ def _run_indexing(
datetime(1970, 1, 1, tzinfo=timezone.utc),
)
- doc_batch_generator = _get_document_generator(
+ connector_runner = _get_connector_runner(
db_session=db_session,
attempt=index_attempt,
start_time=window_start,
@@ -165,15 +172,23 @@ def _run_indexing(
)
all_connector_doc_ids: set[str] = set()
- for doc_batch in doc_batch_generator:
+
+ tracer_counter = 0
+ if INDEXING_TRACER_INTERVAL > 0:
+ tracer.snap()
+ for doc_batch in connector_runner.run():
# Check if connector is disabled mid run and stop if so unless it's the secondary
# index being built. We want to populate it even for paused connectors
# Often paused connectors are sources that aren't updated frequently but the
# contents still need to be initially pulled.
db_session.refresh(db_connector)
if (
- db_connector.disabled
- and db_embedding_model.status != IndexModelStatus.FUTURE
+ (
+ db_cc_pair.status == ConnectorCredentialPairStatus.PAUSED
+ and search_settings.status != IndexModelStatus.FUTURE
+ )
+ # if it's deleting, we don't care if this is a secondary index
+ or db_cc_pair.status == ConnectorCredentialPairStatus.DELETING
):
# let the `except` block handle this
raise RuntimeError("Connector was disabled mid run")
@@ -183,17 +198,30 @@ def _run_indexing(
# Likely due to user manually disabling it or model swap
raise RuntimeError("Index Attempt was canceled")
- logger.debug(
- f"Indexing batch of documents: {[doc.to_short_descriptor() for doc in doc_batch]}"
- )
+ batch_description = []
+ for doc in doc_batch:
+ batch_description.append(doc.to_short_descriptor())
+
+ doc_size = 0
+ for section in doc.sections:
+ doc_size += len(section.text)
+ if doc_size > INDEXING_SIZE_WARNING_THRESHOLD:
+ logger.warning(
+ f"Document size: doc='{doc.to_short_descriptor()}' "
+ f"size={doc_size} "
+ f"threshold={INDEXING_SIZE_WARNING_THRESHOLD}"
+ )
+
+ logger.debug(f"Indexing batch of documents: {batch_description}")
+
+ index_attempt_md.batch_num = batch_num + 1 # use 1-index for this
new_docs, total_batch_chunks = indexing_pipeline(
- documents=doc_batch,
- index_attempt_metadata=IndexAttemptMetadata(
- connector_id=db_connector.id,
- credential_id=db_credential.id,
- ),
+ document_batch=doc_batch,
+ index_attempt_metadata=index_attempt_md,
)
+
+ batch_num += 1
net_doc_change += new_docs
chunk_count += total_batch_chunks
document_count += len(doc_batch)
@@ -215,6 +243,17 @@ def _run_indexing(
docs_removed_from_index=0,
)
+ tracer_counter += 1
+ if (
+ INDEXING_TRACER_INTERVAL > 0
+ and tracer_counter % INDEXING_TRACER_INTERVAL == 0
+ ):
+ logger.debug(
+ f"Running trace comparison for batch {tracer_counter}. interval={INDEXING_TRACER_INTERVAL}"
+ )
+ tracer.snap()
+ tracer.log_previous_diff(INDEXING_TRACER_NUM_PRINT_ENTRIES)
+
run_end_dt = window_end
if is_primary:
update_connector_credential_pair(
@@ -225,7 +264,7 @@ def _run_indexing(
run_dt=run_end_dt,
)
except Exception as e:
- logger.info(
+ logger.exception(
f"Connector run ran into exception after elapsed time: {time.time() - start_time} seconds"
)
# Only mark the attempt as a complete failure if this is the first indexing window.
@@ -237,7 +276,7 @@ def _run_indexing(
# to give better clarity in the UI, as the next run will never happen.
if (
ind == 0
- or db_connector.disabled
+ or not db_cc_pair.status.is_active()
or index_attempt.status != IndexingStatus.IN_PROGRESS
):
mark_attempt_failed(
@@ -249,17 +288,66 @@ def _run_indexing(
if is_primary:
update_connector_credential_pair(
db_session=db_session,
- connector_id=index_attempt.connector.id,
- credential_id=index_attempt.credential.id,
+ connector_id=db_connector.id,
+ credential_id=db_credential.id,
net_docs=net_doc_change,
)
+
+ if INDEXING_TRACER_INTERVAL > 0:
+ tracer.stop()
raise e
# break => similar to success case. As mentioned above, if the next run fails for the same
# reason it will then be marked as a failure
break
- mark_attempt_succeeded(index_attempt, db_session)
+ if INDEXING_TRACER_INTERVAL > 0:
+ logger.debug(
+ f"Running trace comparison between start and end of indexing. {tracer_counter} batches processed."
+ )
+ tracer.snap()
+ tracer.log_first_diff(INDEXING_TRACER_NUM_PRINT_ENTRIES)
+ tracer.stop()
+ logger.debug("Memory tracer stopped.")
+
+ if (
+ index_attempt_md.num_exceptions > 0
+ and index_attempt_md.num_exceptions >= batch_num
+ ):
+ mark_attempt_failed(
+ index_attempt,
+ db_session,
+ failure_reason="All batches exceptioned.",
+ )
+ if is_primary:
+ update_connector_credential_pair(
+ db_session=db_session,
+ connector_id=index_attempt.connector_credential_pair.connector.id,
+ credential_id=index_attempt.connector_credential_pair.credential.id,
+ )
+ raise Exception(
+ f"Connector failed - All batches exceptioned: batches={batch_num}"
+ )
+
+ elapsed_time = time.time() - start_time
+
+ if index_attempt_md.num_exceptions == 0:
+ mark_attempt_succeeded(index_attempt, db_session)
+ logger.info(
+ f"Connector succeeded: "
+ f"docs={document_count} chunks={chunk_count} elapsed={elapsed_time:.2f}s"
+ )
+ else:
+ mark_attempt_partially_succeeded(index_attempt, db_session)
+ logger.info(
+ f"Connector completed with some errors: "
+ f"exceptions={index_attempt_md.num_exceptions} "
+ f"batches={batch_num} "
+ f"docs={document_count} "
+ f"chunks={chunk_count} "
+ f"elapsed={elapsed_time:.2f}s"
+ )
+
if is_primary:
update_connector_credential_pair(
db_session=db_session,
@@ -268,13 +356,6 @@ def _run_indexing(
run_dt=run_end_dt,
)
- logger.info(
- f"Indexed or refreshed {document_count} total documents for a total of {chunk_count} indexed chunks"
- )
- logger.info(
- f"Connector successfully finished, elapsed time: {time.time() - start_time} seconds"
- )
-
def _prepare_index_attempt(db_session: Session, index_attempt_id: int) -> IndexAttempt:
# make sure that the index attempt can't change in between checking the
@@ -287,6 +368,7 @@ def _prepare_index_attempt(db_session: Session, index_attempt_id: int) -> IndexA
db_session=db_session,
index_attempt_id=index_attempt_id,
)
+
if attempt is None:
raise RuntimeError(f"Unable to find IndexAttempt for ID '{index_attempt_id}'")
@@ -297,9 +379,7 @@ def _prepare_index_attempt(db_session: Session, index_attempt_id: int) -> IndexA
)
# only commit once, to make sure this all happens in a single transaction
- mark_attempt_in_progress__no_commit(attempt)
- if attempt.embedding_model.status != IndexModelStatus.PRESENT:
- db_session.commit()
+ mark_attempt_in_progress(attempt, db_session)
return attempt
@@ -322,17 +402,19 @@ def run_indexing_entrypoint(index_attempt_id: int, is_ee: bool = False) -> None:
attempt = _prepare_index_attempt(db_session, index_attempt_id)
logger.info(
- f"Running indexing attempt for connector: '{attempt.connector.name}', "
- f"with config: '{attempt.connector.connector_specific_config}', and "
- f"with credentials: '{attempt.credential_id}'"
+ f"Indexing starting: "
+ f"connector='{attempt.connector_credential_pair.connector.name}' "
+ f"config='{attempt.connector_credential_pair.connector.connector_specific_config}' "
+ f"credentials='{attempt.connector_credential_pair.connector_id}'"
)
_run_indexing(db_session, attempt)
logger.info(
- f"Completed indexing attempt for connector: '{attempt.connector.name}', "
- f"with config: '{attempt.connector.connector_specific_config}', and "
- f"with credentials: '{attempt.credential_id}'"
+ f"Indexing finished: "
+ f"connector='{attempt.connector_credential_pair.connector.name}' "
+ f"config='{attempt.connector_credential_pair.connector.connector_specific_config}' "
+ f"credentials='{attempt.connector_credential_pair.connector_id}'"
)
except Exception as e:
logger.exception(f"Indexing job with ID '{index_attempt_id}' failed due to {e}")
diff --git a/backend/danswer/background/indexing/tracer.py b/backend/danswer/background/indexing/tracer.py
new file mode 100644
index 00000000000..baad9623087
--- /dev/null
+++ b/backend/danswer/background/indexing/tracer.py
@@ -0,0 +1,77 @@
+import tracemalloc
+
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+DANSWER_TRACEMALLOC_FRAMES = 10
+
+
+class DanswerTracer:
+ def __init__(self) -> None:
+ self.snapshot_first: tracemalloc.Snapshot | None = None
+ self.snapshot_prev: tracemalloc.Snapshot | None = None
+ self.snapshot: tracemalloc.Snapshot | None = None
+
+ def start(self) -> None:
+ tracemalloc.start(DANSWER_TRACEMALLOC_FRAMES)
+
+ def stop(self) -> None:
+ tracemalloc.stop()
+
+ def snap(self) -> None:
+ snapshot = tracemalloc.take_snapshot()
+ # Filter out irrelevant frames (e.g., from tracemalloc itself or importlib)
+ snapshot = snapshot.filter_traces(
+ (
+ tracemalloc.Filter(False, tracemalloc.__file__), # Exclude tracemalloc
+ tracemalloc.Filter(
+ False, "