Indexed view using DATA_STRUCTURE and fetch

src/plugins/data/common/constants.ts

@@ -52,6 +52,7 @@ export const DEFAULT_DATA = {
   SET_TYPES: {
     INDEX_PATTERN: 'INDEX_PATTERN',
     INDEX: 'INDEXES',
+    DATASET: 'DATASET',
   },
 
   SOURCE_TYPES: {

src/plugins/data/common/datasets/types.ts

@@ -30,6 +30,12 @@ export interface DataSourceMeta {
   sessionId?: string;
   /** Optional supportsTimeFilter determines if a time filter is needed */
   supportsTimeFilter?: boolean;
+  /** Optional reference to the original dataset */
+  ref?: {
+    id: string;
+    type: string;
+    title: string;
+  };
 }
 
 /**

src/plugins/data/public/query/query_string/dataset_service/dataset_service.mock.ts

@@ -43,6 +43,9 @@ const createSetupDatasetServiceMock = (): jest.Mocked<DatasetServiceContract> =>
     fetchOptions: jest.fn(),
     getRecentDatasets: jest.fn(),
     addRecentDataset: jest.fn(),
+    clearCache: jest.fn(),
+    getLastCacheTime: jest.fn(),
+    removeFromRecentDatasets: jest.fn(),
   };
 };
 

src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts

@@ -98,10 +98,11 @@ export class DatasetService {
     dataset: Dataset,
     services: Partial<IDataPluginServices>
   ): Promise<void> {
-    const type = this.getType(dataset?.type);
+    const datasetType = dataset.dataSource?.meta?.ref?.type || dataset.type;
+    const type = this.getType(datasetType);
     try {
       const asyncType = type?.meta.isFieldLoadAsync ?? false;
-      if (dataset && dataset.type !== DEFAULT_DATA.SET_TYPES.INDEX_PATTERN) {
+      if (datasetType !== DEFAULT_DATA.SET_TYPES.INDEX_PATTERN) {
         const fetchedFields = asyncType
           ? ({} as IndexPatternFieldMap)
           : await type?.fetchFields(dataset, services);

src/plugins/data/public/query/query_string/dataset_service/types.ts

@@ -34,6 +34,8 @@ export interface DatasetTypeConfig {
     searchOnLoad?: boolean;
     /** Optional supportsTimeFilter determines if a time filter is needed */
     supportsTimeFilter?: boolean;
+    /** Optional supportsIndexedViews determines if indexed views are supported */
+    supportsIndexedViews?: boolean;
     /** Optional isFieldLoadAsync determines if field loads are async */
     isFieldLoadAsync?: boolean;
     /** Optional cacheOptions determines if the data structure is cacheable. Defaults to false */

src/plugins/data/public/ui/dataset_selector/configurator.tsx

@@ -19,7 +19,15 @@ import {
 import { i18n } from '@osd/i18n';
 import { FormattedMessage } from '@osd/i18n/react';
 import React, { useEffect, useMemo, useState } from 'react';
-import { BaseDataset, DEFAULT_DATA, Dataset, DatasetField, Query } from '../../../common';
+import {
+  BaseDataset,
+  DEFAULT_DATA,
+  DataStructure,
+  DataStructureMeta,
+  Dataset,
+  DatasetField,
+  Query,
+} from '../../../common';
 import { getIndexPatterns, getQueryService } from '../../services';
 import { IDataPluginServices } from '../../types';
 
@@ -52,6 +60,23 @@ export const Configurator = ({
       defaultMessage: "I don't want to use the time filter",
     }
   );
+  const [indexedViews, setIndexedViews] = useState<DataStructure[]>([]);
+  const [indexedView, setIndexedView] = useState<DataStructure | undefined>(
+    dataset.dataSource?.meta?.ref
+      ? ({
+          id: dataset.id,
+          title: dataset.title,
+          type: DEFAULT_DATA.SET_TYPES.INDEX,
+          meta: dataset.dataSource.meta,
+        } as DataStructure)
+      : undefined
+  );
+  const noIndexedView = i18n.translate(
+    'data.explorer.datasetSelector.advancedSelector.configurator.indexedView.noIndexedViewOptionLabel',
+    {
+      defaultMessage: "I don't want to use an indexed view",
+    }
+  );
   const [language, setLanguage] = useState<string>(() => {
     const currentLanguage = queryString.getQuery().language;
     if (languages.includes(currentLanguage)) {
@@ -91,6 +116,40 @@ export const Configurator = ({
     fetchFields();
   }, [baseDataset, indexPatternsService, queryString, timeFields.length]);
 
+  useEffect(() => {
+    const fetchViews = async () => {
+      if (type?.meta.supportsIndexedViews) {
+        try {
+          const dataSourceStructure: DataStructure = {
+            id: dataset.dataSource?.id || '',
+            title: dataset.dataSource?.title || '',
+            type: 'DATA_SOURCE',
+            children: [],
+          };
+
+          const datasetStructure: DataStructure = {
+            id: dataset.id,
+            title: dataset.title,
+            type: DEFAULT_DATA.SET_TYPES.DATASET,
+            meta: dataset.dataSource?.meta as DataStructureMeta,
+            children: [],
+          };
+
+          const path = dataset.dataSource
+            ? [dataSourceStructure, datasetStructure]
+            : [datasetStructure];
+
+          const result = await type.fetch(services, path);
+          setIndexedViews(result.children || []);
+        } catch (error) {
+          setIndexedViews([]);
+        }
+      }
+    };
+
+    fetchViews();
+  }, [dataset, type?.meta.supportsIndexedViews, services, type]);
+
   return (
     <>
       <EuiModalHeader>
@@ -185,6 +244,40 @@ export const Configurator = ({
                 />
               </EuiFormRow>
             ))}
+          <EuiFormRow
+            label={i18n.translate(
+              'data.explorer.datasetSelector.advancedSelector.configurator.indexedViewLabel',
+              {
+                defaultMessage: 'Available indexed views',
+              }
+            )}
+            helpText={i18n.translate(
+              'data.explorer.datasetSelector.advancedSelector.configurator.indexedViewHelpText',
+              {
+                defaultMessage: 'Select an indexed view to speed up your query.',
+              }
+            )}
+          >
+            <EuiSelect
+              options={[
+                ...indexedViews.map((view) => ({
+                  text: view.title,
+                  value: view.id,
+                })),
+                { text: '-----', value: '-----', disabled: true },
+                { text: noIndexedView, value: noIndexedView },
+              ]}
+              value={indexedView?.id || '-----'}
+              onChange={(e) => {
+                const value = e.target.value === noIndexedView ? undefined : e.target.value;
+                if (!dataset.dataSource) return;
+                // if the indexed views are properly structured we can just set it directly without building it here
+                // see s3 type mock response how we can return the index type and with the correct id
+                const view = indexedViews.find((v) => v.id === value);
+                setIndexedView(view);
+              }}
+            />
+          </EuiFormRow>
         </EuiForm>
       </EuiModalBody>
       <EuiModalFooter>
@@ -202,8 +295,28 @@ export const Configurator = ({
         </EuiButton>
         <EuiButton
           onClick={async () => {
-            await queryString.getDatasetService().cacheDataset(dataset, services);
-            onConfirm({ dataset, language });
+            let configuredDataset = { ...dataset };
+            if (indexedView) {
+              configuredDataset = {
+                id: indexedView.id,
+                title: indexedView.title,
+                type: DEFAULT_DATA.SET_TYPES.INDEX,
+                timeFieldName: dataset.timeFieldName,
+                dataSource: {
+                  ...dataset.dataSource!,
+                  meta: {
+                    ...dataset.dataSource?.meta,
+                    ...indexedView.meta, // This includes the ref to original dataset
+                  },
+                },
+              };
+            }
+
+            await queryString.getDatasetService().cacheDataset(configuredDataset, services);
+            onConfirm({
+              dataset: configuredDataset,
+              language,
+            });
           }}
           fill
           disabled={submitDisabled}

src/plugins/data/public/ui/dataset_selector/dataset_selector.tsx

@@ -82,7 +82,9 @@ export const DatasetSelector = ({
   const { overlays } = services;
   const datasetService = getQueryService().queryString.getDatasetService();
   const datasetIcon =
-    datasetService.getType(selectedDataset?.type || '')?.meta.icon.type || 'database';
+    datasetService.getType(
+      selectedDataset?.dataSource?.meta?.ref?.type || selectedDataset?.type || ''
+    )?.meta.icon.type || 'database';
 
   useEffect(() => {
     isMounted.current = true;

src/plugins/query_enhancements/public/datasets/s3_type.ts

@@ -27,6 +27,8 @@ import {
 } from '../../common';
 import S3_ICON from '../assets/s3_mark.svg';
 
+const mockFetchIndexedViews = true;
+
 export const s3TypeConfig: DatasetTypeConfig = {
   id: DATASET.S3,
   title: 'S3 Connections',
@@ -35,6 +37,7 @@ export const s3TypeConfig: DatasetTypeConfig = {
     tooltip: 'Amazon S3 Connections',
     searchOnLoad: true,
     supportsTimeFilter: false,
+    supportsIndexedViews: true,
     isFieldLoadAsync: true,
     cacheOptions: true,
   },
@@ -98,6 +101,18 @@ export const s3TypeConfig: DatasetTypeConfig = {
           children: tables,
         };
       }
+      // Use dataset type (could be TABLE, QUERY, etc) to fetch indexed views
+      case 'DATASET': {
+        const indexedViews = !mockFetchIndexedViews
+          ? await fetchIndexedViews(http, path)
+          : fetchIndexedViewsMock(dataStructure);
+        return {
+          ...dataStructure,
+          columnHeader: 'Indexed Views',
+          hasNext: false,
+          children: indexedViews,
+        };
+      }
       default: {
         const dataSources = await fetchDataSources(client);
         return {
@@ -114,6 +129,7 @@ export const s3TypeConfig: DatasetTypeConfig = {
     dataset: Dataset,
     services?: Partial<IDataPluginServices>
   ): Promise<DatasetField[]> => {
+    if (mockFetchIndexedViews) return [];
     const http = services?.http;
     if (!http) return [];
     return await fetchFields(http, dataset);
@@ -294,6 +310,96 @@ const fetchTables = async (http: HttpSetup, path: DataStructure[]): Promise<Data
   return fetch(http, path, 'TABLE');
 };
 
+const fetchIndexedViewsMock = (dataStructure: DataStructure): DataStructure[] => {
+  return [
+    {
+      id: `${dataStructure.id}.logstash-2015.09.20`,
+      title: 'logstash-2015.09.20',
+      type: DEFAULT_DATA.SET_TYPES.INDEX,
+      parent: dataStructure,
+      meta: {
+        type: DATA_STRUCTURE_META_TYPES.CUSTOM,
+        sessionId: (dataStructure.meta as DataStructureCustomMeta)?.sessionId,
+        name: (dataStructure.meta as DataStructureCustomMeta)?.name,
+        ref: {
+          id: dataStructure.id,
+          type: DATASET.S3,
+          title: dataStructure.title,
+        },
+      } as DataStructureCustomMeta,
+    },
+    {
+      id: `${dataStructure.id}.logstash-2015.09.22`,
+      title: 'logstash-2015.09.22',
+      type: DEFAULT_DATA.SET_TYPES.INDEX,
+      parent: dataStructure,
+      meta: {
+        type: DATA_STRUCTURE_META_TYPES.CUSTOM,
+        sessionId: (dataStructure.meta as DataStructureCustomMeta)?.sessionId,
+        name: (dataStructure.meta as DataStructureCustomMeta)?.name,
+        ref: {
+          id: dataStructure.id,
+          type: DATASET.S3,
+          title: dataStructure.title,
+        },
+      } as DataStructureCustomMeta,
+    },
+  ];
+};
+
+const fetchIndexedViews = async (
+  http: HttpSetup,
+  path: DataStructure[]
+): Promise<DataStructure[]> => {
+  const abortController = new AbortController();
+  const dataSource = path.find((ds) => ds.type === 'DATA_SOURCE');
+  const dataStructure = path[path.length - 1];
+  const meta = dataStructure.meta as DataStructureCustomMeta;
+
+  try {
+    const response = await http.fetch({
+      method: 'POST',
+      path: trimEnd(API.DATA_SOURCE.ASYNC_JOBS),
+      body: JSON.stringify({
+        lang: 'sql',
+        query: `SHOW MATERIALIZED VIEWS FOR ${dataStructure.title}`,
+        datasource: meta.name,
+        ...(meta.sessionId && { sessionId: meta.sessionId }),
+      }),
+      query: {
+        id: dataSource?.id,
+      },
+      signal: abortController.signal,
+    });
+
+    const views = await handleQueryStatus({
+      fetchStatus: () =>
+        http.fetch({
+          method: 'GET',
+          path: trimEnd(API.DATA_SOURCE.ASYNC_JOBS),
+          query: {
+            id: dataSource?.id,
+            queryId: response.queryId,
+          },
+        }),
+    });
+
+    return views.datarows.map((view: string[]) => ({
+      id: `${dataStructure.id}.${view[0]}`,
+      title: view[0],
+      type: DEFAULT_DATA.SET_TYPES.INDEX,
+      parent: dataStructure,
+      meta: {
+        type: DATA_STRUCTURE_META_TYPES.CUSTOM,
+        sessionId: meta.sessionId,
+        name: meta.name,
+      } as DataStructureCustomMeta,
+    }));
+  } catch (err) {
+    return [];
+  }
+};
+
 /**
  *  Mapping function from S3_FIELD_TYPES to OSD_FIELD_TYPES
  *