From 80d028bccef3f548e8b877afbc911da1b2dc771c Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Wed, 13 Nov 2024 12:29:19 +0000 Subject: [PATCH] rough impl leveraging existing `fetch` call the `DATA_STRUCTURE` generic object can handle this concept of indexed view. Here I'm defining DATASET object. If the data type wants to handled the DATASET type and return an indexed view then it can. Added a mock of S3 Signed-off-by: Kawika Avilla --- src/plugins/data/common/constants.ts | 1 + src/plugins/data/common/datasets/types.ts | 6 + .../dataset_service/dataset_service.mock.ts | 3 + .../dataset_service/dataset_service.ts | 5 +- .../query_string/dataset_service/types.ts | 2 + .../ui/dataset_selector/configurator.tsx | 119 +++++++++++++++++- .../ui/dataset_selector/dataset_selector.tsx | 4 +- .../public/datasets/s3_type.ts | 106 ++++++++++++++++ 8 files changed, 240 insertions(+), 6 deletions(-) diff --git a/src/plugins/data/common/constants.ts b/src/plugins/data/common/constants.ts index 65d144531fce..da6d43002987 100644 --- a/src/plugins/data/common/constants.ts +++ b/src/plugins/data/common/constants.ts @@ -52,6 +52,7 @@ export const DEFAULT_DATA = { SET_TYPES: { INDEX_PATTERN: 'INDEX_PATTERN', INDEX: 'INDEXES', + DATASET: 'DATASET', }, SOURCE_TYPES: { diff --git a/src/plugins/data/common/datasets/types.ts b/src/plugins/data/common/datasets/types.ts index e777eb8a45e8..fbd3d27364e4 100644 --- a/src/plugins/data/common/datasets/types.ts +++ b/src/plugins/data/common/datasets/types.ts @@ -30,6 +30,12 @@ export interface DataSourceMeta { sessionId?: string; /** Optional supportsTimeFilter determines if a time filter is needed */ supportsTimeFilter?: boolean; + /** Optional reference to the original dataset */ + ref?: { + id: string; + type: string; + title: string; + }; } /** diff --git a/src/plugins/data/public/query/query_string/dataset_service/dataset_service.mock.ts b/src/plugins/data/public/query/query_string/dataset_service/dataset_service.mock.ts index df5521078feb..ba491cb51191 100644 --- a/src/plugins/data/public/query/query_string/dataset_service/dataset_service.mock.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/dataset_service.mock.ts @@ -43,6 +43,9 @@ const createSetupDatasetServiceMock = (): jest.Mocked => fetchOptions: jest.fn(), getRecentDatasets: jest.fn(), addRecentDataset: jest.fn(), + clearCache: jest.fn(), + getLastCacheTime: jest.fn(), + removeFromRecentDatasets: jest.fn(), }; }; diff --git a/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts b/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts index 9f47448c5324..695a609ca804 100644 --- a/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts @@ -98,10 +98,11 @@ export class DatasetService { dataset: Dataset, services: Partial ): Promise { - const type = this.getType(dataset?.type); + const datasetType = dataset.dataSource?.meta?.ref?.type || dataset.type; + const type = this.getType(datasetType); try { const asyncType = type?.meta.isFieldLoadAsync ?? false; - if (dataset && dataset.type !== DEFAULT_DATA.SET_TYPES.INDEX_PATTERN) { + if (datasetType !== DEFAULT_DATA.SET_TYPES.INDEX_PATTERN) { const fetchedFields = asyncType ? ({} as IndexPatternFieldMap) : await type?.fetchFields(dataset, services); diff --git a/src/plugins/data/public/query/query_string/dataset_service/types.ts b/src/plugins/data/public/query/query_string/dataset_service/types.ts index f303fa6af56d..127853d4a15c 100644 --- a/src/plugins/data/public/query/query_string/dataset_service/types.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/types.ts @@ -34,6 +34,8 @@ export interface DatasetTypeConfig { searchOnLoad?: boolean; /** Optional supportsTimeFilter determines if a time filter is needed */ supportsTimeFilter?: boolean; + /** Optional supportsIndexedViews determines if indexed views are supported */ + supportsIndexedViews?: boolean; /** Optional isFieldLoadAsync determines if field loads are async */ isFieldLoadAsync?: boolean; /** Optional cacheOptions determines if the data structure is cacheable. Defaults to false */ diff --git a/src/plugins/data/public/ui/dataset_selector/configurator.tsx b/src/plugins/data/public/ui/dataset_selector/configurator.tsx index 2940c6b2baf0..149ed2a2a2d9 100644 --- a/src/plugins/data/public/ui/dataset_selector/configurator.tsx +++ b/src/plugins/data/public/ui/dataset_selector/configurator.tsx @@ -19,7 +19,15 @@ import { import { i18n } from '@osd/i18n'; import { FormattedMessage } from '@osd/i18n/react'; import React, { useEffect, useMemo, useState } from 'react'; -import { BaseDataset, DEFAULT_DATA, Dataset, DatasetField, Query } from '../../../common'; +import { + BaseDataset, + DEFAULT_DATA, + DataStructure, + DataStructureMeta, + Dataset, + DatasetField, + Query, +} from '../../../common'; import { getIndexPatterns, getQueryService } from '../../services'; import { IDataPluginServices } from '../../types'; @@ -52,6 +60,23 @@ export const Configurator = ({ defaultMessage: "I don't want to use the time filter", } ); + const [indexedViews, setIndexedViews] = useState([]); + const [indexedView, setIndexedView] = useState( + dataset.dataSource?.meta?.ref + ? ({ + id: dataset.id, + title: dataset.title, + type: DEFAULT_DATA.SET_TYPES.INDEX, + meta: dataset.dataSource.meta, + } as DataStructure) + : undefined + ); + const noIndexedView = i18n.translate( + 'data.explorer.datasetSelector.advancedSelector.configurator.indexedView.noIndexedViewOptionLabel', + { + defaultMessage: "I don't want to use an indexed view", + } + ); const [language, setLanguage] = useState(() => { const currentLanguage = queryString.getQuery().language; if (languages.includes(currentLanguage)) { @@ -91,6 +116,40 @@ export const Configurator = ({ fetchFields(); }, [baseDataset, indexPatternsService, queryString, timeFields.length]); + useEffect(() => { + const fetchViews = async () => { + if (type?.meta.supportsIndexedViews) { + try { + const dataSourceStructure: DataStructure = { + id: dataset.dataSource?.id || '', + title: dataset.dataSource?.title || '', + type: 'DATA_SOURCE', + children: [], + }; + + const datasetStructure: DataStructure = { + id: dataset.id, + title: dataset.title, + type: DEFAULT_DATA.SET_TYPES.DATASET, + meta: dataset.dataSource?.meta as DataStructureMeta, + children: [], + }; + + const path = dataset.dataSource + ? [dataSourceStructure, datasetStructure] + : [datasetStructure]; + + const result = await type.fetch(services, path); + setIndexedViews(result.children || []); + } catch (error) { + setIndexedViews([]); + } + } + }; + + fetchViews(); + }, [dataset, type?.meta.supportsIndexedViews, services, type]); + return ( <> @@ -185,6 +244,40 @@ export const Configurator = ({ /> ))} + + ({ + text: view.title, + value: view.id, + })), + { text: '-----', value: '-----', disabled: true }, + { text: noIndexedView, value: noIndexedView }, + ]} + value={indexedView?.id || '-----'} + onChange={(e) => { + const value = e.target.value === noIndexedView ? undefined : e.target.value; + if (!dataset.dataSource) return; + // if the indexed views are properly structured we can just set it directly without building it here + // see s3 type mock response how we can return the index type and with the correct id + const view = indexedViews.find((v) => v.id === value); + setIndexedView(view); + }} + /> + @@ -202,8 +295,28 @@ export const Configurator = ({ { - await queryString.getDatasetService().cacheDataset(dataset, services); - onConfirm({ dataset, language }); + let configuredDataset = { ...dataset }; + if (indexedView) { + configuredDataset = { + id: indexedView.id, + title: indexedView.title, + type: DEFAULT_DATA.SET_TYPES.INDEX, + timeFieldName: dataset.timeFieldName, + dataSource: { + ...dataset.dataSource!, + meta: { + ...dataset.dataSource?.meta, + ...indexedView.meta, // This includes the ref to original dataset + }, + }, + }; + } + + await queryString.getDatasetService().cacheDataset(configuredDataset, services); + onConfirm({ + dataset: configuredDataset, + language, + }); }} fill disabled={submitDisabled} diff --git a/src/plugins/data/public/ui/dataset_selector/dataset_selector.tsx b/src/plugins/data/public/ui/dataset_selector/dataset_selector.tsx index 75ea695a2083..f66c1d768f5f 100644 --- a/src/plugins/data/public/ui/dataset_selector/dataset_selector.tsx +++ b/src/plugins/data/public/ui/dataset_selector/dataset_selector.tsx @@ -82,7 +82,9 @@ export const DatasetSelector = ({ const { overlays } = services; const datasetService = getQueryService().queryString.getDatasetService(); const datasetIcon = - datasetService.getType(selectedDataset?.type || '')?.meta.icon.type || 'database'; + datasetService.getType( + selectedDataset?.dataSource?.meta?.ref?.type || selectedDataset?.type || '' + )?.meta.icon.type || 'database'; useEffect(() => { isMounted.current = true; diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.ts b/src/plugins/query_enhancements/public/datasets/s3_type.ts index c13b5e898670..f8b5936af5c9 100644 --- a/src/plugins/query_enhancements/public/datasets/s3_type.ts +++ b/src/plugins/query_enhancements/public/datasets/s3_type.ts @@ -27,6 +27,8 @@ import { } from '../../common'; import S3_ICON from '../assets/s3_mark.svg'; +const mockFetchIndexedViews = true; + export const s3TypeConfig: DatasetTypeConfig = { id: DATASET.S3, title: 'S3 Connections', @@ -35,6 +37,7 @@ export const s3TypeConfig: DatasetTypeConfig = { tooltip: 'Amazon S3 Connections', searchOnLoad: true, supportsTimeFilter: false, + supportsIndexedViews: true, isFieldLoadAsync: true, cacheOptions: true, }, @@ -98,6 +101,18 @@ export const s3TypeConfig: DatasetTypeConfig = { children: tables, }; } + // Use dataset type (could be TABLE, QUERY, etc) to fetch indexed views + case 'DATASET': { + const indexedViews = !mockFetchIndexedViews + ? await fetchIndexedViews(http, path) + : fetchIndexedViewsMock(dataStructure); + return { + ...dataStructure, + columnHeader: 'Indexed Views', + hasNext: false, + children: indexedViews, + }; + } default: { const dataSources = await fetchDataSources(client); return { @@ -114,6 +129,7 @@ export const s3TypeConfig: DatasetTypeConfig = { dataset: Dataset, services?: Partial ): Promise => { + if (mockFetchIndexedViews) return []; const http = services?.http; if (!http) return []; return await fetchFields(http, dataset); @@ -294,6 +310,96 @@ const fetchTables = async (http: HttpSetup, path: DataStructure[]): Promise { + return [ + { + id: `${dataStructure.id}.logstash-2015.09.20`, + title: 'logstash-2015.09.20', + type: DEFAULT_DATA.SET_TYPES.INDEX, + parent: dataStructure, + meta: { + type: DATA_STRUCTURE_META_TYPES.CUSTOM, + sessionId: (dataStructure.meta as DataStructureCustomMeta)?.sessionId, + name: (dataStructure.meta as DataStructureCustomMeta)?.name, + ref: { + id: dataStructure.id, + type: DATASET.S3, + title: dataStructure.title, + }, + } as DataStructureCustomMeta, + }, + { + id: `${dataStructure.id}.logstash-2015.09.22`, + title: 'logstash-2015.09.22', + type: DEFAULT_DATA.SET_TYPES.INDEX, + parent: dataStructure, + meta: { + type: DATA_STRUCTURE_META_TYPES.CUSTOM, + sessionId: (dataStructure.meta as DataStructureCustomMeta)?.sessionId, + name: (dataStructure.meta as DataStructureCustomMeta)?.name, + ref: { + id: dataStructure.id, + type: DATASET.S3, + title: dataStructure.title, + }, + } as DataStructureCustomMeta, + }, + ]; +}; + +const fetchIndexedViews = async ( + http: HttpSetup, + path: DataStructure[] +): Promise => { + const abortController = new AbortController(); + const dataSource = path.find((ds) => ds.type === 'DATA_SOURCE'); + const dataStructure = path[path.length - 1]; + const meta = dataStructure.meta as DataStructureCustomMeta; + + try { + const response = await http.fetch({ + method: 'POST', + path: trimEnd(API.DATA_SOURCE.ASYNC_JOBS), + body: JSON.stringify({ + lang: 'sql', + query: `SHOW MATERIALIZED VIEWS FOR ${dataStructure.title}`, + datasource: meta.name, + ...(meta.sessionId && { sessionId: meta.sessionId }), + }), + query: { + id: dataSource?.id, + }, + signal: abortController.signal, + }); + + const views = await handleQueryStatus({ + fetchStatus: () => + http.fetch({ + method: 'GET', + path: trimEnd(API.DATA_SOURCE.ASYNC_JOBS), + query: { + id: dataSource?.id, + queryId: response.queryId, + }, + }), + }); + + return views.datarows.map((view: string[]) => ({ + id: `${dataStructure.id}.${view[0]}`, + title: view[0], + type: DEFAULT_DATA.SET_TYPES.INDEX, + parent: dataStructure, + meta: { + type: DATA_STRUCTURE_META_TYPES.CUSTOM, + sessionId: meta.sessionId, + name: meta.name, + } as DataStructureCustomMeta, + })); + } catch (err) { + return []; + } +}; + /** * Mapping function from S3_FIELD_TYPES to OSD_FIELD_TYPES *