diff --git a/src/plugins/data/common/constants.ts b/src/plugins/data/common/constants.ts index 65d144531fce..da6d43002987 100644 --- a/src/plugins/data/common/constants.ts +++ b/src/plugins/data/common/constants.ts @@ -52,6 +52,7 @@ export const DEFAULT_DATA = { SET_TYPES: { INDEX_PATTERN: 'INDEX_PATTERN', INDEX: 'INDEXES', + DATASET: 'DATASET', }, SOURCE_TYPES: { diff --git a/src/plugins/data/common/datasets/types.ts b/src/plugins/data/common/datasets/types.ts index e777eb8a45e8..fbd3d27364e4 100644 --- a/src/plugins/data/common/datasets/types.ts +++ b/src/plugins/data/common/datasets/types.ts @@ -30,6 +30,12 @@ export interface DataSourceMeta { sessionId?: string; /** Optional supportsTimeFilter determines if a time filter is needed */ supportsTimeFilter?: boolean; + /** Optional reference to the original dataset */ + ref?: { + id: string; + type: string; + title: string; + }; } /** diff --git a/src/plugins/data/public/query/query_string/dataset_service/dataset_service.mock.ts b/src/plugins/data/public/query/query_string/dataset_service/dataset_service.mock.ts index df5521078feb..ba491cb51191 100644 --- a/src/plugins/data/public/query/query_string/dataset_service/dataset_service.mock.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/dataset_service.mock.ts @@ -43,6 +43,9 @@ const createSetupDatasetServiceMock = (): jest.Mocked => fetchOptions: jest.fn(), getRecentDatasets: jest.fn(), addRecentDataset: jest.fn(), + clearCache: jest.fn(), + getLastCacheTime: jest.fn(), + removeFromRecentDatasets: jest.fn(), }; }; diff --git a/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts b/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts index 9f47448c5324..695a609ca804 100644 --- a/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts @@ -98,10 +98,11 @@ export class DatasetService { dataset: Dataset, services: Partial ): Promise { - const type = this.getType(dataset?.type); + const datasetType = dataset.dataSource?.meta?.ref?.type || dataset.type; + const type = this.getType(datasetType); try { const asyncType = type?.meta.isFieldLoadAsync ?? false; - if (dataset && dataset.type !== DEFAULT_DATA.SET_TYPES.INDEX_PATTERN) { + if (datasetType !== DEFAULT_DATA.SET_TYPES.INDEX_PATTERN) { const fetchedFields = asyncType ? ({} as IndexPatternFieldMap) : await type?.fetchFields(dataset, services); diff --git a/src/plugins/data/public/query/query_string/dataset_service/types.ts b/src/plugins/data/public/query/query_string/dataset_service/types.ts index f303fa6af56d..127853d4a15c 100644 --- a/src/plugins/data/public/query/query_string/dataset_service/types.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/types.ts @@ -34,6 +34,8 @@ export interface DatasetTypeConfig { searchOnLoad?: boolean; /** Optional supportsTimeFilter determines if a time filter is needed */ supportsTimeFilter?: boolean; + /** Optional supportsIndexedViews determines if indexed views are supported */ + supportsIndexedViews?: boolean; /** Optional isFieldLoadAsync determines if field loads are async */ isFieldLoadAsync?: boolean; /** Optional cacheOptions determines if the data structure is cacheable. Defaults to false */ diff --git a/src/plugins/data/public/ui/dataset_selector/configurator.tsx b/src/plugins/data/public/ui/dataset_selector/configurator.tsx index 2940c6b2baf0..149ed2a2a2d9 100644 --- a/src/plugins/data/public/ui/dataset_selector/configurator.tsx +++ b/src/plugins/data/public/ui/dataset_selector/configurator.tsx @@ -19,7 +19,15 @@ import { import { i18n } from '@osd/i18n'; import { FormattedMessage } from '@osd/i18n/react'; import React, { useEffect, useMemo, useState } from 'react'; -import { BaseDataset, DEFAULT_DATA, Dataset, DatasetField, Query } from '../../../common'; +import { + BaseDataset, + DEFAULT_DATA, + DataStructure, + DataStructureMeta, + Dataset, + DatasetField, + Query, +} from '../../../common'; import { getIndexPatterns, getQueryService } from '../../services'; import { IDataPluginServices } from '../../types'; @@ -52,6 +60,23 @@ export const Configurator = ({ defaultMessage: "I don't want to use the time filter", } ); + const [indexedViews, setIndexedViews] = useState([]); + const [indexedView, setIndexedView] = useState( + dataset.dataSource?.meta?.ref + ? ({ + id: dataset.id, + title: dataset.title, + type: DEFAULT_DATA.SET_TYPES.INDEX, + meta: dataset.dataSource.meta, + } as DataStructure) + : undefined + ); + const noIndexedView = i18n.translate( + 'data.explorer.datasetSelector.advancedSelector.configurator.indexedView.noIndexedViewOptionLabel', + { + defaultMessage: "I don't want to use an indexed view", + } + ); const [language, setLanguage] = useState(() => { const currentLanguage = queryString.getQuery().language; if (languages.includes(currentLanguage)) { @@ -91,6 +116,40 @@ export const Configurator = ({ fetchFields(); }, [baseDataset, indexPatternsService, queryString, timeFields.length]); + useEffect(() => { + const fetchViews = async () => { + if (type?.meta.supportsIndexedViews) { + try { + const dataSourceStructure: DataStructure = { + id: dataset.dataSource?.id || '', + title: dataset.dataSource?.title || '', + type: 'DATA_SOURCE', + children: [], + }; + + const datasetStructure: DataStructure = { + id: dataset.id, + title: dataset.title, + type: DEFAULT_DATA.SET_TYPES.DATASET, + meta: dataset.dataSource?.meta as DataStructureMeta, + children: [], + }; + + const path = dataset.dataSource + ? [dataSourceStructure, datasetStructure] + : [datasetStructure]; + + const result = await type.fetch(services, path); + setIndexedViews(result.children || []); + } catch (error) { + setIndexedViews([]); + } + } + }; + + fetchViews(); + }, [dataset, type?.meta.supportsIndexedViews, services, type]); + return ( <> @@ -185,6 +244,40 @@ export const Configurator = ({ /> ))} + + ({ + text: view.title, + value: view.id, + })), + { text: '-----', value: '-----', disabled: true }, + { text: noIndexedView, value: noIndexedView }, + ]} + value={indexedView?.id || '-----'} + onChange={(e) => { + const value = e.target.value === noIndexedView ? undefined : e.target.value; + if (!dataset.dataSource) return; + // if the indexed views are properly structured we can just set it directly without building it here + // see s3 type mock response how we can return the index type and with the correct id + const view = indexedViews.find((v) => v.id === value); + setIndexedView(view); + }} + /> + @@ -202,8 +295,28 @@ export const Configurator = ({ { - await queryString.getDatasetService().cacheDataset(dataset, services); - onConfirm({ dataset, language }); + let configuredDataset = { ...dataset }; + if (indexedView) { + configuredDataset = { + id: indexedView.id, + title: indexedView.title, + type: DEFAULT_DATA.SET_TYPES.INDEX, + timeFieldName: dataset.timeFieldName, + dataSource: { + ...dataset.dataSource!, + meta: { + ...dataset.dataSource?.meta, + ...indexedView.meta, // This includes the ref to original dataset + }, + }, + }; + } + + await queryString.getDatasetService().cacheDataset(configuredDataset, services); + onConfirm({ + dataset: configuredDataset, + language, + }); }} fill disabled={submitDisabled} diff --git a/src/plugins/data/public/ui/dataset_selector/dataset_selector.tsx b/src/plugins/data/public/ui/dataset_selector/dataset_selector.tsx index 75ea695a2083..f66c1d768f5f 100644 --- a/src/plugins/data/public/ui/dataset_selector/dataset_selector.tsx +++ b/src/plugins/data/public/ui/dataset_selector/dataset_selector.tsx @@ -82,7 +82,9 @@ export const DatasetSelector = ({ const { overlays } = services; const datasetService = getQueryService().queryString.getDatasetService(); const datasetIcon = - datasetService.getType(selectedDataset?.type || '')?.meta.icon.type || 'database'; + datasetService.getType( + selectedDataset?.dataSource?.meta?.ref?.type || selectedDataset?.type || '' + )?.meta.icon.type || 'database'; useEffect(() => { isMounted.current = true; diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.ts b/src/plugins/query_enhancements/public/datasets/s3_type.ts index c13b5e898670..f8b5936af5c9 100644 --- a/src/plugins/query_enhancements/public/datasets/s3_type.ts +++ b/src/plugins/query_enhancements/public/datasets/s3_type.ts @@ -27,6 +27,8 @@ import { } from '../../common'; import S3_ICON from '../assets/s3_mark.svg'; +const mockFetchIndexedViews = true; + export const s3TypeConfig: DatasetTypeConfig = { id: DATASET.S3, title: 'S3 Connections', @@ -35,6 +37,7 @@ export const s3TypeConfig: DatasetTypeConfig = { tooltip: 'Amazon S3 Connections', searchOnLoad: true, supportsTimeFilter: false, + supportsIndexedViews: true, isFieldLoadAsync: true, cacheOptions: true, }, @@ -98,6 +101,18 @@ export const s3TypeConfig: DatasetTypeConfig = { children: tables, }; } + // Use dataset type (could be TABLE, QUERY, etc) to fetch indexed views + case 'DATASET': { + const indexedViews = !mockFetchIndexedViews + ? await fetchIndexedViews(http, path) + : fetchIndexedViewsMock(dataStructure); + return { + ...dataStructure, + columnHeader: 'Indexed Views', + hasNext: false, + children: indexedViews, + }; + } default: { const dataSources = await fetchDataSources(client); return { @@ -114,6 +129,7 @@ export const s3TypeConfig: DatasetTypeConfig = { dataset: Dataset, services?: Partial ): Promise => { + if (mockFetchIndexedViews) return []; const http = services?.http; if (!http) return []; return await fetchFields(http, dataset); @@ -294,6 +310,96 @@ const fetchTables = async (http: HttpSetup, path: DataStructure[]): Promise { + return [ + { + id: `${dataStructure.id}.logstash-2015.09.20`, + title: 'logstash-2015.09.20', + type: DEFAULT_DATA.SET_TYPES.INDEX, + parent: dataStructure, + meta: { + type: DATA_STRUCTURE_META_TYPES.CUSTOM, + sessionId: (dataStructure.meta as DataStructureCustomMeta)?.sessionId, + name: (dataStructure.meta as DataStructureCustomMeta)?.name, + ref: { + id: dataStructure.id, + type: DATASET.S3, + title: dataStructure.title, + }, + } as DataStructureCustomMeta, + }, + { + id: `${dataStructure.id}.logstash-2015.09.22`, + title: 'logstash-2015.09.22', + type: DEFAULT_DATA.SET_TYPES.INDEX, + parent: dataStructure, + meta: { + type: DATA_STRUCTURE_META_TYPES.CUSTOM, + sessionId: (dataStructure.meta as DataStructureCustomMeta)?.sessionId, + name: (dataStructure.meta as DataStructureCustomMeta)?.name, + ref: { + id: dataStructure.id, + type: DATASET.S3, + title: dataStructure.title, + }, + } as DataStructureCustomMeta, + }, + ]; +}; + +const fetchIndexedViews = async ( + http: HttpSetup, + path: DataStructure[] +): Promise => { + const abortController = new AbortController(); + const dataSource = path.find((ds) => ds.type === 'DATA_SOURCE'); + const dataStructure = path[path.length - 1]; + const meta = dataStructure.meta as DataStructureCustomMeta; + + try { + const response = await http.fetch({ + method: 'POST', + path: trimEnd(API.DATA_SOURCE.ASYNC_JOBS), + body: JSON.stringify({ + lang: 'sql', + query: `SHOW MATERIALIZED VIEWS FOR ${dataStructure.title}`, + datasource: meta.name, + ...(meta.sessionId && { sessionId: meta.sessionId }), + }), + query: { + id: dataSource?.id, + }, + signal: abortController.signal, + }); + + const views = await handleQueryStatus({ + fetchStatus: () => + http.fetch({ + method: 'GET', + path: trimEnd(API.DATA_SOURCE.ASYNC_JOBS), + query: { + id: dataSource?.id, + queryId: response.queryId, + }, + }), + }); + + return views.datarows.map((view: string[]) => ({ + id: `${dataStructure.id}.${view[0]}`, + title: view[0], + type: DEFAULT_DATA.SET_TYPES.INDEX, + parent: dataStructure, + meta: { + type: DATA_STRUCTURE_META_TYPES.CUSTOM, + sessionId: meta.sessionId, + name: meta.name, + } as DataStructureCustomMeta, + })); + } catch (err) { + return []; + } +}; + /** * Mapping function from S3_FIELD_TYPES to OSD_FIELD_TYPES *