rough impl leveraging existing fetch call

the `DATA_STRUCTURE` generic object can handle this concept of indexed view. Here I'm defining DATASET object. If the data type wants to handled the DATASET type and return an indexed view then it can. Added a mock of S3 Signed-off-by: Kawika Avilla <kavilla414@gmail.com>
opensearch-project · Nov 13, 2024 · 80d028b · 80d028b
1 parent 76cf823
commit 80d028b
Show file tree

Hide file tree

Showing 8 changed files with 240 additions and 6 deletions.
diff --git a/src/plugins/data/common/constants.ts b/src/plugins/data/common/constants.ts
@@ -52,6 +52,7 @@ export const DEFAULT_DATA = {
   SET_TYPES: {
     INDEX_PATTERN: 'INDEX_PATTERN',
     INDEX: 'INDEXES',
+    DATASET: 'DATASET',
   },
 
   SOURCE_TYPES: {

diff --git a/src/plugins/data/common/datasets/types.ts b/src/plugins/data/common/datasets/types.ts
@@ -30,6 +30,12 @@ export interface DataSourceMeta {
   sessionId?: string;
   /** Optional supportsTimeFilter determines if a time filter is needed */
   supportsTimeFilter?: boolean;
+  /** Optional reference to the original dataset */
+  ref?: {
+    id: string;
+    type: string;
+    title: string;
+  };
 }
 
 /**

diff --git a/src/plugins/data/public/query/query_string/dataset_service/dataset_service.mock.ts b/src/plugins/data/public/query/query_string/dataset_service/dataset_service.mock.ts
@@ -43,6 +43,9 @@ const createSetupDatasetServiceMock = (): jest.Mocked<DatasetServiceContract> =>
     fetchOptions: jest.fn(),
     getRecentDatasets: jest.fn(),
     addRecentDataset: jest.fn(),
+    clearCache: jest.fn(),
+    getLastCacheTime: jest.fn(),
+    removeFromRecentDatasets: jest.fn(),
   };
 };
 

diff --git a/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts b/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts
@@ -98,10 +98,11 @@ export class DatasetService {
     dataset: Dataset,
     services: Partial<IDataPluginServices>
   ): Promise<void> {
-    const type = this.getType(dataset?.type);
+    const datasetType = dataset.dataSource?.meta?.ref?.type || dataset.type;
+    const type = this.getType(datasetType);
     try {
       const asyncType = type?.meta.isFieldLoadAsync ?? false;
-      if (dataset && dataset.type !== DEFAULT_DATA.SET_TYPES.INDEX_PATTERN) {
+      if (datasetType !== DEFAULT_DATA.SET_TYPES.INDEX_PATTERN) {
         const fetchedFields = asyncType
           ? ({} as IndexPatternFieldMap)
           : await type?.fetchFields(dataset, services);

diff --git a/src/plugins/data/public/query/query_string/dataset_service/types.ts b/src/plugins/data/public/query/query_string/dataset_service/types.ts
@@ -34,6 +34,8 @@ export interface DatasetTypeConfig {
     searchOnLoad?: boolean;
     /** Optional supportsTimeFilter determines if a time filter is needed */
     supportsTimeFilter?: boolean;
+    /** Optional supportsIndexedViews determines if indexed views are supported */
+    supportsIndexedViews?: boolean;
     /** Optional isFieldLoadAsync determines if field loads are async */
     isFieldLoadAsync?: boolean;
     /** Optional cacheOptions determines if the data structure is cacheable. Defaults to false */

diff --git a/src/plugins/data/public/ui/dataset_selector/configurator.tsx b/src/plugins/data/public/ui/dataset_selector/configurator.tsx
@@ -19,7 +19,15 @@ import {
 import { i18n } from '@osd/i18n';
 import { FormattedMessage } from '@osd/i18n/react';
 import React, { useEffect, useMemo, useState } from 'react';
-import { BaseDataset, DEFAULT_DATA, Dataset, DatasetField, Query } from '../../../common';
+import {
+  BaseDataset,
+  DEFAULT_DATA,
+  DataStructure,
+  DataStructureMeta,
+  Dataset,
+  DatasetField,
+  Query,
+} from '../../../common';
 import { getIndexPatterns, getQueryService } from '../../services';
 import { IDataPluginServices } from '../../types';
 
@@ -52,6 +60,23 @@ export const Configurator = ({
       defaultMessage: "I don't want to use the time filter",
     }
   );
+  const [indexedViews, setIndexedViews] = useState<DataStructure[]>([]);
+  const [indexedView, setIndexedView] = useState<DataStructure | undefined>(
+    dataset.dataSource?.meta?.ref
+      ? ({
+          id: dataset.id,
+          title: dataset.title,
+          type: DEFAULT_DATA.SET_TYPES.INDEX,
+          meta: dataset.dataSource.meta,
+        } as DataStructure)
+      : undefined
+  );
+  const noIndexedView = i18n.translate(
+    'data.explorer.datasetSelector.advancedSelector.configurator.indexedView.noIndexedViewOptionLabel',
+    {
+      defaultMessage: "I don't want to use an indexed view",
+    }
+  );
   const [language, setLanguage] = useState<string>(() => {
     const currentLanguage = queryString.getQuery().language;
     if (languages.includes(currentLanguage)) {
@@ -91,6 +116,40 @@ export const Configurator = ({
     fetchFields();
   }, [baseDataset, indexPatternsService, queryString, timeFields.length]);
 
+  useEffect(() => {
+    const fetchViews = async () => {
+      if (type?.meta.supportsIndexedViews) {
+        try {
+          const dataSourceStructure: DataStructure = {
+            id: dataset.dataSource?.id || '',
+            title: dataset.dataSource?.title || '',
+            type: 'DATA_SOURCE',
+            children: [],
+          };
+
+          const datasetStructure: DataStructure = {
+            id: dataset.id,
+            title: dataset.title,
+            type: DEFAULT_DATA.SET_TYPES.DATASET,
+            meta: dataset.dataSource?.meta as DataStructureMeta,
+            children: [],
+          };
+
+          const path = dataset.dataSource
+            ? [dataSourceStructure, datasetStructure]
+            : [datasetStructure];
+
+          const result = await type.fetch(services, path);
+          setIndexedViews(result.children || []);
+        } catch (error) {
+          setIndexedViews([]);
+        }
+      }
+    };
+
+    fetchViews();
+  }, [dataset, type?.meta.supportsIndexedViews, services, type]);
+
   return (
     <>
       <EuiModalHeader>
@@ -185,6 +244,40 @@ export const Configurator = ({
                 />
               </EuiFormRow>
             ))}
+          <EuiFormRow
+            label={i18n.translate(
+              'data.explorer.datasetSelector.advancedSelector.configurator.indexedViewLabel',
+              {
+                defaultMessage: 'Available indexed views',
+              }
+            )}
+            helpText={i18n.translate(
+              'data.explorer.datasetSelector.advancedSelector.configurator.indexedViewHelpText',
+              {
+                defaultMessage: 'Select an indexed view to speed up your query.',
+              }
+            )}
+          >
+            <EuiSelect
+              options={[
+                ...indexedViews.map((view) => ({
+                  text: view.title,
+                  value: view.id,
+                })),
+                { text: '-----', value: '-----', disabled: true },
+                { text: noIndexedView, value: noIndexedView },
+              ]}
+              value={indexedView?.id || '-----'}
+              onChange={(e) => {
+                const value = e.target.value === noIndexedView ? undefined : e.target.value;
+                if (!dataset.dataSource) return;
+                // if the indexed views are properly structured we can just set it directly without building it here
+                // see s3 type mock response how we can return the index type and with the correct id
+                const view = indexedViews.find((v) => v.id === value);
+                setIndexedView(view);
+              }}
+            />
+          </EuiFormRow>
         </EuiForm>
       </EuiModalBody>
       <EuiModalFooter>
@@ -202,8 +295,28 @@ export const Configurator = ({
         </EuiButton>
         <EuiButton
           onClick={async () => {
-            await queryString.getDatasetService().cacheDataset(dataset, services);
-            onConfirm({ dataset, language });
+            let configuredDataset = { ...dataset };
+            if (indexedView) {
+              configuredDataset = {
+                id: indexedView.id,
+                title: indexedView.title,
+                type: DEFAULT_DATA.SET_TYPES.INDEX,
+                timeFieldName: dataset.timeFieldName,
+                dataSource: {
+                  ...dataset.dataSource!,
+                  meta: {
+                    ...dataset.dataSource?.meta,
+                    ...indexedView.meta, // This includes the ref to original dataset
+                  },
+                },
+              };
+            }
+
+            await queryString.getDatasetService().cacheDataset(configuredDataset, services);
+            onConfirm({
+              dataset: configuredDataset,
+              language,
+            });
           }}
           fill
           disabled={submitDisabled}

diff --git a/src/plugins/data/public/ui/dataset_selector/dataset_selector.tsx b/src/plugins/data/public/ui/dataset_selector/dataset_selector.tsx
@@ -82,7 +82,9 @@ export const DatasetSelector = ({
   const { overlays } = services;
   const datasetService = getQueryService().queryString.getDatasetService();
   const datasetIcon =
-    datasetService.getType(selectedDataset?.type || '')?.meta.icon.type || 'database';
+    datasetService.getType(
+      selectedDataset?.dataSource?.meta?.ref?.type || selectedDataset?.type || ''
+    )?.meta.icon.type || 'database';
 
   useEffect(() => {
     isMounted.current = true;

diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.ts b/src/plugins/query_enhancements/public/datasets/s3_type.ts
@@ -27,6 +27,8 @@ import {
 } from '../../common';
 import S3_ICON from '../assets/s3_mark.svg';
 
+const mockFetchIndexedViews = true;
+
 export const s3TypeConfig: DatasetTypeConfig = {
   id: DATASET.S3,
   title: 'S3 Connections',
@@ -35,6 +37,7 @@ export const s3TypeConfig: DatasetTypeConfig = {
     tooltip: 'Amazon S3 Connections',
     searchOnLoad: true,
     supportsTimeFilter: false,
+    supportsIndexedViews: true,
     isFieldLoadAsync: true,
     cacheOptions: true,
   },
@@ -98,6 +101,18 @@ export const s3TypeConfig: DatasetTypeConfig = {
           children: tables,
         };
       }
+      // Use dataset type (could be TABLE, QUERY, etc) to fetch indexed views
+      case 'DATASET': {
+        const indexedViews = !mockFetchIndexedViews
+          ? await fetchIndexedViews(http, path)
+          : fetchIndexedViewsMock(dataStructure);
+        return {
+          ...dataStructure,
+          columnHeader: 'Indexed Views',
+          hasNext: false,
+          children: indexedViews,
+        };
+      }
       default: {
         const dataSources = await fetchDataSources(client);
         return {
@@ -114,6 +129,7 @@ export const s3TypeConfig: DatasetTypeConfig = {
     dataset: Dataset,
     services?: Partial<IDataPluginServices>
   ): Promise<DatasetField[]> => {
+    if (mockFetchIndexedViews) return [];
     const http = services?.http;
     if (!http) return [];
     return await fetchFields(http, dataset);
@@ -294,6 +310,96 @@ const fetchTables = async (http: HttpSetup, path: DataStructure[]): Promise<Data
   return fetch(http, path, 'TABLE');
 };
 
+const fetchIndexedViewsMock = (dataStructure: DataStructure): DataStructure[] => {
+  return [
+    {
+      id: `${dataStructure.id}.logstash-2015.09.20`,
+      title: 'logstash-2015.09.20',
+      type: DEFAULT_DATA.SET_TYPES.INDEX,
+      parent: dataStructure,
+      meta: {
+        type: DATA_STRUCTURE_META_TYPES.CUSTOM,
+        sessionId: (dataStructure.meta as DataStructureCustomMeta)?.sessionId,
+        name: (dataStructure.meta as DataStructureCustomMeta)?.name,
+        ref: {
+          id: dataStructure.id,
+          type: DATASET.S3,
+          title: dataStructure.title,
+        },
+      } as DataStructureCustomMeta,
+    },
+    {
+      id: `${dataStructure.id}.logstash-2015.09.22`,
+      title: 'logstash-2015.09.22',
+      type: DEFAULT_DATA.SET_TYPES.INDEX,
+      parent: dataStructure,
+      meta: {
+        type: DATA_STRUCTURE_META_TYPES.CUSTOM,
+        sessionId: (dataStructure.meta as DataStructureCustomMeta)?.sessionId,
+        name: (dataStructure.meta as DataStructureCustomMeta)?.name,
+        ref: {
+          id: dataStructure.id,
+          type: DATASET.S3,
+          title: dataStructure.title,
+        },
+      } as DataStructureCustomMeta,
+    },
+  ];
+};
+
+const fetchIndexedViews = async (
+  http: HttpSetup,
+  path: DataStructure[]
+): Promise<DataStructure[]> => {
+  const abortController = new AbortController();
+  const dataSource = path.find((ds) => ds.type === 'DATA_SOURCE');
+  const dataStructure = path[path.length - 1];
+  const meta = dataStructure.meta as DataStructureCustomMeta;
+
+  try {
+    const response = await http.fetch({
+      method: 'POST',
+      path: trimEnd(API.DATA_SOURCE.ASYNC_JOBS),
+      body: JSON.stringify({
+        lang: 'sql',
+        query: `SHOW MATERIALIZED VIEWS FOR ${dataStructure.title}`,
+        datasource: meta.name,
+        ...(meta.sessionId && { sessionId: meta.sessionId }),
+      }),
+      query: {
+        id: dataSource?.id,
+      },
+      signal: abortController.signal,
+    });
+
+    const views = await handleQueryStatus({
+      fetchStatus: () =>
+        http.fetch({
+          method: 'GET',
+          path: trimEnd(API.DATA_SOURCE.ASYNC_JOBS),
+          query: {
+            id: dataSource?.id,
+            queryId: response.queryId,
+          },
+        }),
+    });
+
+    return views.datarows.map((view: string[]) => ({
+      id: `${dataStructure.id}.${view[0]}`,
+      title: view[0],
+      type: DEFAULT_DATA.SET_TYPES.INDEX,
+      parent: dataStructure,
+      meta: {
+        type: DATA_STRUCTURE_META_TYPES.CUSTOM,
+        sessionId: meta.sessionId,
+        name: meta.name,
+      } as DataStructureCustomMeta,
+    }));
+  } catch (err) {
+    return [];
+  }
+};
+
 /**
  *  Mapping function from S3_FIELD_TYPES to OSD_FIELD_TYPES
  *