From 939d23d14c7f1a2e21c67f12b40396c24ed1c3ff Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Thu, 14 Dec 2023 15:52:17 +0800 Subject: [PATCH 1/9] feat: adds engines table to information_schema --- src/catalog/src/information_schema.rs | 5 + src/catalog/src/information_schema/engines.rs | 147 ++++++++++++++++++ src/common/catalog/src/consts.rs | 2 + .../common/system/information_schema.result | 17 ++ .../common/system/information_schema.sql | 7 + 5 files changed, 178 insertions(+) create mode 100644 src/catalog/src/information_schema/engines.rs diff --git a/src/catalog/src/information_schema.rs b/src/catalog/src/information_schema.rs index 484529cd57a4..5c236943a63c 100644 --- a/src/catalog/src/information_schema.rs +++ b/src/catalog/src/information_schema.rs @@ -13,6 +13,7 @@ // limitations under the License. mod columns; +mod engines; mod tables; use std::collections::HashMap; @@ -35,11 +36,13 @@ use table::TableRef; use self::columns::InformationSchemaColumns; use crate::error::Result; +use crate::information_schema::engines::InformationSchemaEngines; use crate::information_schema::tables::InformationSchemaTables; use crate::CatalogManager; pub const TABLES: &str = "tables"; pub const COLUMNS: &str = "columns"; +pub const ENGINES: &str = "engines"; pub struct InformationSchemaProvider { catalog_name: String, @@ -65,6 +68,7 @@ impl InformationSchemaProvider { let mut schema = HashMap::new(); schema.insert(TABLES.to_owned(), provider.table(TABLES).unwrap()); schema.insert(COLUMNS.to_owned(), provider.table(COLUMNS).unwrap()); + schema.insert(ENGINES.to_owned(), provider.table(ENGINES).unwrap()); schema } @@ -89,6 +93,7 @@ impl InformationSchemaProvider { self.catalog_name.clone(), self.catalog_manager.clone(), )) as _), + ENGINES => Some(Arc::new(InformationSchemaEngines::new()) as _), _ => None, } } diff --git a/src/catalog/src/information_schema/engines.rs b/src/catalog/src/information_schema/engines.rs new file mode 100644 index 000000000000..e82359f331c9 --- /dev/null +++ b/src/catalog/src/information_schema/engines.rs @@ -0,0 +1,147 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use arrow_schema::SchemaRef as ArrowSchemaRef; +use common_catalog::consts::INFORMATION_SCHEMA_ENGINES_TABLE_ID; +use common_error::ext::BoxedError; +use common_query::physical_plan::TaskContext; +use common_recordbatch::adapter::RecordBatchStreamAdapter; +use common_recordbatch::{RecordBatch, SendableRecordBatchStream}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter; +use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream; +use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream; +use datatypes::prelude::{ConcreteDataType, VectorRef}; +use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; +use datatypes::vectors::StringVector; +use snafu::ResultExt; +use store_api::storage::TableId; + +use super::ENGINES; +use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result}; +use crate::information_schema::InformationTable; + +pub(super) struct InformationSchemaEngines { + schema: SchemaRef, +} + +impl InformationSchemaEngines { + pub(super) fn new() -> Self { + Self { + schema: Self::schema(), + } + } + + pub(crate) fn schema() -> SchemaRef { + Arc::new(Schema::new(vec![ + // The name of the storage engine. + ColumnSchema::new("engine", ConcreteDataType::string_datatype(), false), + // The level of support that the server has on the storage engine + ColumnSchema::new("support", ConcreteDataType::string_datatype(), false), + // The brief comment on the storage engine + ColumnSchema::new("comment", ConcreteDataType::string_datatype(), false), + // Whether the storage engine supports transactions. + ColumnSchema::new("transactions", ConcreteDataType::string_datatype(), false), + // Whether the storage engine supports XA transactions. + ColumnSchema::new("xa", ConcreteDataType::string_datatype(), true), + // Whether the storage engine supports `savepoints`. + ColumnSchema::new("savepoints", ConcreteDataType::string_datatype(), true), + ])) + } + + fn builder(&self) -> InformationSchemaEnginesBuilder { + InformationSchemaEnginesBuilder::new(self.schema.clone()) + } +} + +impl InformationTable for InformationSchemaEngines { + fn table_id(&self) -> TableId { + INFORMATION_SCHEMA_ENGINES_TABLE_ID + } + + fn table_name(&self) -> &'static str { + ENGINES + } + + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + fn to_stream(&self) -> Result { + let schema = self.schema.arrow_schema().clone(); + let mut builder = self.builder(); + let stream = Box::pin(DfRecordBatchStreamAdapter::new( + schema, + futures::stream::once(async move { + builder + .make_engines() + .await + .map(|x| x.into_df_record_batch()) + .map_err(Into::into) + }), + )); + Ok(Box::pin( + RecordBatchStreamAdapter::try_new(stream) + .map_err(BoxedError::new) + .context(InternalSnafu)?, + )) + } +} + +struct InformationSchemaEnginesBuilder { + schema: SchemaRef, +} + +impl InformationSchemaEnginesBuilder { + fn new(schema: SchemaRef) -> Self { + Self { schema } + } + + /// Construct the `information_schema.engines` virtual table + async fn make_engines(&mut self) -> Result { + let columns: Vec = vec![ + Arc::new(StringVector::from(vec!["Mito"])), + Arc::new(StringVector::from(vec!["DEFAULT"])), + Arc::new(StringVector::from(vec![ + "Storage engine for time-series data", + ])), + Arc::new(StringVector::from(vec!["NO"])), + Arc::new(StringVector::from(vec!["NO"])), + Arc::new(StringVector::from(vec!["NO"])), + ]; + RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu) + } +} + +impl DfPartitionStream for InformationSchemaEngines { + fn schema(&self) -> &ArrowSchemaRef { + self.schema.arrow_schema() + } + + fn execute(&self, _: Arc) -> DfSendableRecordBatchStream { + let schema = self.schema.arrow_schema().clone(); + let mut builder = self.builder(); + Box::pin(DfRecordBatchStreamAdapter::new( + schema, + futures::stream::once(async move { + builder + .make_engines() + .await + .map(|x| x.into_df_record_batch()) + .map_err(Into::into) + }), + )) + } +} diff --git a/src/common/catalog/src/consts.rs b/src/common/catalog/src/consts.rs index 111ffb8a4d45..53226dc94695 100644 --- a/src/common/catalog/src/consts.rs +++ b/src/common/catalog/src/consts.rs @@ -33,6 +33,8 @@ pub const NUMBERS_TABLE_ID: u32 = 2; pub const INFORMATION_SCHEMA_TABLES_TABLE_ID: u32 = 3; /// id for information_schema.columns pub const INFORMATION_SCHEMA_COLUMNS_TABLE_ID: u32 = 4; +/// id for information_schema.engines +pub const INFORMATION_SCHEMA_ENGINES_TABLE_ID: u32 = 5; pub const MITO_ENGINE: &str = "mito"; pub const MITO2_ENGINE: &str = "mito2"; diff --git a/tests/cases/standalone/common/system/information_schema.result b/tests/cases/standalone/common/system/information_schema.result index 6a33a00baff3..df70cb2a67c8 100644 --- a/tests/cases/standalone/common/system/information_schema.result +++ b/tests/cases/standalone/common/system/information_schema.result @@ -93,3 +93,20 @@ drop schema my_db; Error: 1001(Unsupported), SQL statement is not supported: drop schema my_db;, keyword: schema +use information_schema; + +Affected Rows: 0 + +-- test engines +select * from engines; + ++--------+---------+-------------------------------------+--------------+----+------------+ +| engine | support | comment | transactions | xa | savepoints | ++--------+---------+-------------------------------------+--------------+----+------------+ +| Mito | DEFAULT | Storage engine for time-series data | NO | NO | NO | ++--------+---------+-------------------------------------+--------------+----+------------+ + +use public; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/system/information_schema.sql b/tests/cases/standalone/common/system/information_schema.sql index 8c44edb4c885..321de80fb4f5 100644 --- a/tests/cases/standalone/common/system/information_schema.sql +++ b/tests/cases/standalone/common/system/information_schema.sql @@ -38,3 +38,10 @@ order by table_schema, table_name; use public; drop schema my_db; + +use information_schema; + +-- test engines +select * from engines; + +use public; From d22d8565f9e4dfffc2c6a539406e815ba15e1d55 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Thu, 14 Dec 2023 19:56:19 +0800 Subject: [PATCH 2/9] feat: adds COLUMN_PRIVILEGES and COLUMN_STATISTICS --- src/catalog/src/information_schema.rs | 32 ++++- .../src/information_schema/empty_table.rs | 120 ++++++++++++++++++ .../information_schema/empty_table_schemas.rs | 65 ++++++++++ src/catalog/src/information_schema/engines.rs | 4 +- .../src/information_schema/table_names.rs | 21 +++ src/common/catalog/src/consts.rs | 7 + src/common/recordbatch/src/recordbatch.rs | 10 ++ .../common/system/information_schema.result | 42 +++++- .../common/system/information_schema.sql | 9 ++ 9 files changed, 302 insertions(+), 8 deletions(-) create mode 100644 src/catalog/src/information_schema/empty_table.rs create mode 100644 src/catalog/src/information_schema/empty_table_schemas.rs create mode 100644 src/catalog/src/information_schema/table_names.rs diff --git a/src/catalog/src/information_schema.rs b/src/catalog/src/information_schema.rs index 5c236943a63c..3763340bcd8a 100644 --- a/src/catalog/src/information_schema.rs +++ b/src/catalog/src/information_schema.rs @@ -13,16 +13,20 @@ // limitations under the License. mod columns; +mod empty_table; +mod empty_table_schemas; mod engines; +mod table_names; mod tables; use std::collections::HashMap; use std::sync::{Arc, Weak}; -use common_catalog::consts::INFORMATION_SCHEMA_NAME; +use common_catalog::consts::{self, INFORMATION_SCHEMA_NAME}; use common_error::ext::BoxedError; use common_recordbatch::{RecordBatchStreamWrapper, SendableRecordBatchStream}; use datatypes::schema::SchemaRef; +use empty_table_schemas::get_schema; use futures_util::StreamExt; use snafu::ResultExt; use store_api::data_source::DataSource; @@ -33,17 +37,15 @@ use table::metadata::{ }; use table::thin_table::{ThinTable, ThinTableAdapter}; use table::TableRef; +pub use table_names::*; use self::columns::InformationSchemaColumns; use crate::error::Result; +use crate::information_schema::empty_table::EmptyTable; use crate::information_schema::engines::InformationSchemaEngines; use crate::information_schema::tables::InformationSchemaTables; use crate::CatalogManager; -pub const TABLES: &str = "tables"; -pub const COLUMNS: &str = "columns"; -pub const ENGINES: &str = "engines"; - pub struct InformationSchemaProvider { catalog_name: String, catalog_manager: Weak, @@ -69,6 +71,15 @@ impl InformationSchemaProvider { schema.insert(TABLES.to_owned(), provider.table(TABLES).unwrap()); schema.insert(COLUMNS.to_owned(), provider.table(COLUMNS).unwrap()); schema.insert(ENGINES.to_owned(), provider.table(ENGINES).unwrap()); + // Tables not implemented + schema.insert( + COLUMN_PRIVILEGES.to_owned(), + provider.table(COLUMN_PRIVILEGES).unwrap(), + ); + schema.insert( + COLUMN_STATISTICS.to_owned(), + provider.table(COLUMN_STATISTICS).unwrap(), + ); schema } @@ -94,6 +105,17 @@ impl InformationSchemaProvider { self.catalog_manager.clone(), )) as _), ENGINES => Some(Arc::new(InformationSchemaEngines::new()) as _), + // Table not implemented + COLUMN_PRIVILEGES => Some(Arc::new(EmptyTable::new( + consts::INFORMATION_SCHEMA_COLUMN_PRIVILEGES_TABLE_ID, + COLUMN_PRIVILEGES, + get_schema(COLUMN_PRIVILEGES), + ))), + COLUMN_STATISTICS => Some(Arc::new(EmptyTable::new( + consts::INFORMATION_SCHEMA_COLUMN_STATISTICS_TABLE_ID, + COLUMN_STATISTICS, + get_schema(COLUMN_STATISTICS), + ))), _ => None, } } diff --git a/src/catalog/src/information_schema/empty_table.rs b/src/catalog/src/information_schema/empty_table.rs new file mode 100644 index 000000000000..6c63cca5def5 --- /dev/null +++ b/src/catalog/src/information_schema/empty_table.rs @@ -0,0 +1,120 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use arrow_schema::SchemaRef as ArrowSchemaRef; +use common_error::ext::BoxedError; +use common_query::physical_plan::TaskContext; +use common_recordbatch::adapter::RecordBatchStreamAdapter; +use common_recordbatch::{RecordBatch, SendableRecordBatchStream}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter; +use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream; +use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream; +use datatypes::schema::SchemaRef; +use snafu::ResultExt; +use store_api::storage::TableId; + +use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result}; +use crate::information_schema::InformationTable; + +/// An empty table with specific schema. +pub(super) struct EmptyTable { + schema: SchemaRef, + table_id: TableId, + table_name: &'static str, +} + +impl EmptyTable { + pub(super) fn new(table_id: TableId, table_name: &'static str, schema: SchemaRef) -> Self { + Self { + table_id, + table_name, + schema, + } + } + fn builder(&self) -> EmptyTableBuilder { + EmptyTableBuilder::new(self.schema.clone()) + } +} + +impl InformationTable for EmptyTable { + fn table_id(&self) -> TableId { + self.table_id + } + + fn table_name(&self) -> &'static str { + self.table_name + } + + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + fn to_stream(&self) -> Result { + let schema = self.schema.arrow_schema().clone(); + let mut builder = self.builder(); + let stream = Box::pin(DfRecordBatchStreamAdapter::new( + schema, + futures::stream::once(async move { + builder + .empty_records() + .await + .map(|x| x.into_df_record_batch()) + .map_err(Into::into) + }), + )); + Ok(Box::pin( + RecordBatchStreamAdapter::try_new(stream) + .map_err(BoxedError::new) + .context(InternalSnafu)?, + )) + } +} + +struct EmptyTableBuilder { + schema: SchemaRef, +} + +impl EmptyTableBuilder { + fn new(schema: SchemaRef) -> Self { + Self { schema } + } + + /// Construct the `information_schema.{table_name}` virtual table + async fn empty_records(&mut self) -> Result { + RecordBatch::new_empty(self.schema.clone()).context(CreateRecordBatchSnafu) + } +} + +impl DfPartitionStream for EmptyTable { + fn schema(&self) -> &ArrowSchemaRef { + self.schema.arrow_schema() + } + + fn execute(&self, _: Arc) -> DfSendableRecordBatchStream { + let schema = self.schema.arrow_schema().clone(); + let mut builder = self.builder(); + Box::pin(DfRecordBatchStreamAdapter::new( + schema, + futures::stream::once(async move { + builder + .empty_records() + .await + .map(|x| x.into_df_record_batch()) + .map_err(Into::into) + }), + )) + } +} diff --git a/src/catalog/src/information_schema/empty_table_schemas.rs b/src/catalog/src/information_schema/empty_table_schemas.rs new file mode 100644 index 000000000000..eee5b1c6b209 --- /dev/null +++ b/src/catalog/src/information_schema/empty_table_schemas.rs @@ -0,0 +1,65 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +/// Return the schemas of tables which are not implemented. +use datatypes::prelude::ConcreteDataType; +use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; + +use crate::information_schema::table_names::*; + +/// Find the schema by the table_name. +/// Safety: the user MUST ensure the table schema exists, panic otherwise. +pub fn get_schema(table_name: &'static str) -> SchemaRef { + let column_schemas = match table_name { + // https://dev.mysql.com/doc/refman/8.0/en/information-schema-column-privileges-table.html + COLUMN_PRIVILEGES => string_columns(&[ + "GRANTEE", + "TABLE_CATALOG", + "TABLE_SCHEMA", + "TABLE_NAME", + "COLUMN_NAME", + "PRIVILEGE_TYPE", + "IS_GRANTABLE", + ]), + // https://dev.mysql.com/doc/refman/8.0/en/information-schema-column-statistics-table.html + COLUMN_STATISTICS => string_columns(&[ + "SCHEMA_NAME", + "TABLE_NAME", + "COLUMN_NAME", + // TODO(dennis): It must be a JSON type, but we don't support it yet + "HISTOGRAM", + ]), + + _ => unreachable!("Unknown table in information_schema: {}", table_name), + }; + + Arc::new(Schema::new(column_schemas)) +} + +fn string_columns(names: &[&'static str]) -> Vec { + names.iter().map(|name| string_column(name)).collect() +} + +fn string_column(name: &str) -> ColumnSchema { + ColumnSchema::new( + str::to_lowercase(name), + ConcreteDataType::string_datatype(), + false, + ) +} + +#[cfg(test)] +mod tests {} diff --git a/src/catalog/src/information_schema/engines.rs b/src/catalog/src/information_schema/engines.rs index e82359f331c9..045ce06754f7 100644 --- a/src/catalog/src/information_schema/engines.rs +++ b/src/catalog/src/information_schema/engines.rs @@ -15,7 +15,7 @@ use std::sync::Arc; use arrow_schema::SchemaRef as ArrowSchemaRef; -use common_catalog::consts::INFORMATION_SCHEMA_ENGINES_TABLE_ID; +use common_catalog::consts::{INFORMATION_SCHEMA_ENGINES_TABLE_ID, MITO_ENGINE}; use common_error::ext::BoxedError; use common_query::physical_plan::TaskContext; use common_recordbatch::adapter::RecordBatchStreamAdapter; @@ -112,7 +112,7 @@ impl InformationSchemaEnginesBuilder { /// Construct the `information_schema.engines` virtual table async fn make_engines(&mut self) -> Result { let columns: Vec = vec![ - Arc::new(StringVector::from(vec!["Mito"])), + Arc::new(StringVector::from(vec![MITO_ENGINE])), Arc::new(StringVector::from(vec!["DEFAULT"])), Arc::new(StringVector::from(vec![ "Storage engine for time-series data", diff --git a/src/catalog/src/information_schema/table_names.rs b/src/catalog/src/information_schema/table_names.rs new file mode 100644 index 000000000000..0b3404f472a2 --- /dev/null +++ b/src/catalog/src/information_schema/table_names.rs @@ -0,0 +1,21 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// All table names in `information_schema`. + +pub const TABLES: &str = "tables"; +pub const COLUMNS: &str = "columns"; +pub const ENGINES: &str = "engines"; +pub const COLUMN_PRIVILEGES: &str = "column_privileges"; +pub const COLUMN_STATISTICS: &str = "column_statistics"; diff --git a/src/common/catalog/src/consts.rs b/src/common/catalog/src/consts.rs index 53226dc94695..3788f03adb22 100644 --- a/src/common/catalog/src/consts.rs +++ b/src/common/catalog/src/consts.rs @@ -29,12 +29,19 @@ pub const SYSTEM_CATALOG_TABLE_ID: u32 = 0; pub const SCRIPTS_TABLE_ID: u32 = 1; /// numbers table id pub const NUMBERS_TABLE_ID: u32 = 2; + +/// ----- Begin of information_schema tables ---- /// id for information_schema.tables pub const INFORMATION_SCHEMA_TABLES_TABLE_ID: u32 = 3; /// id for information_schema.columns pub const INFORMATION_SCHEMA_COLUMNS_TABLE_ID: u32 = 4; /// id for information_schema.engines pub const INFORMATION_SCHEMA_ENGINES_TABLE_ID: u32 = 5; +/// id for information_schema.column_privileges +pub const INFORMATION_SCHEMA_COLUMN_PRIVILEGES_TABLE_ID: u32 = 6; +/// id for information_schema.column_statistics +pub const INFORMATION_SCHEMA_COLUMN_STATISTICS_TABLE_ID: u32 = 7; +/// ----- End of information_schema tables ---- pub const MITO_ENGINE: &str = "mito"; pub const MITO2_ENGINE: &str = "mito2"; diff --git a/src/common/recordbatch/src/recordbatch.rs b/src/common/recordbatch/src/recordbatch.rs index 5de969dbbdb7..420901902b64 100644 --- a/src/common/recordbatch/src/recordbatch.rs +++ b/src/common/recordbatch/src/recordbatch.rs @@ -57,6 +57,16 @@ impl RecordBatch { }) } + /// Create an empty [`RecordBatch`] from `schema`. + pub fn new_empty(schema: SchemaRef) -> Result { + let df_record_batch = DfRecordBatch::new_empty(schema.arrow_schema().clone()); + Ok(RecordBatch { + schema, + columns: vec![], + df_record_batch, + }) + } + pub fn try_project(&self, indices: &[usize]) -> Result { let schema = Arc::new(self.schema.try_project(indices).context(DataTypesSnafu)?); let mut columns = Vec::with_capacity(indices.len()); diff --git a/tests/cases/standalone/common/system/information_schema.result b/tests/cases/standalone/common/system/information_schema.result index df70cb2a67c8..be60fdd701e5 100644 --- a/tests/cases/standalone/common/system/information_schema.result +++ b/tests/cases/standalone/common/system/information_schema.result @@ -103,9 +103,49 @@ select * from engines; +--------+---------+-------------------------------------+--------------+----+------------+ | engine | support | comment | transactions | xa | savepoints | +--------+---------+-------------------------------------+--------------+----+------------+ -| Mito | DEFAULT | Storage engine for time-series data | NO | NO | NO | +| mito | DEFAULT | Storage engine for time-series data | NO | NO | NO | +--------+---------+-------------------------------------+--------------+----+------------+ +-- tables not implemented +desc table COLUMN_PRIVILEGES; + ++----------------+--------+-----+------+---------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++----------------+--------+-----+------+---------+---------------+ +| grantee | String | | NO | | FIELD | +| table_catalog | String | | NO | | FIELD | +| table_schema | String | | NO | | FIELD | +| table_name | String | | NO | | FIELD | +| column_name | String | | NO | | FIELD | +| privilege_type | String | | NO | | FIELD | +| is_grantable | String | | NO | | FIELD | ++----------------+--------+-----+------+---------+---------------+ + +select * from COLUMN_PRIVILEGES; + ++---------+---------------+--------------+------------+-------------+----------------+--------------+ +| grantee | table_catalog | table_schema | table_name | column_name | privilege_type | is_grantable | ++---------+---------------+--------------+------------+-------------+----------------+--------------+ ++---------+---------------+--------------+------------+-------------+----------------+--------------+ + +desc table COLUMN_STATISTICS; + ++-------------+--------+-----+------+---------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++-------------+--------+-----+------+---------+---------------+ +| schema_name | String | | NO | | FIELD | +| table_name | String | | NO | | FIELD | +| column_name | String | | NO | | FIELD | +| histogram | String | | NO | | FIELD | ++-------------+--------+-----+------+---------+---------------+ + +select * from COLUMN_STATISTICS; + ++-------------+------------+-------------+-----------+ +| schema_name | table_name | column_name | histogram | ++-------------+------------+-------------+-----------+ ++-------------+------------+-------------+-----------+ + use public; Affected Rows: 0 diff --git a/tests/cases/standalone/common/system/information_schema.sql b/tests/cases/standalone/common/system/information_schema.sql index 321de80fb4f5..f4ebc63303e7 100644 --- a/tests/cases/standalone/common/system/information_schema.sql +++ b/tests/cases/standalone/common/system/information_schema.sql @@ -44,4 +44,13 @@ use information_schema; -- test engines select * from engines; +-- tables not implemented +desc table COLUMN_PRIVILEGES; + +select * from COLUMN_PRIVILEGES; + +desc table COLUMN_STATISTICS; + +select * from COLUMN_STATISTICS; + use public; From abf4821d612eaf51d2f72973f873bef66d08fd04 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Fri, 15 Dec 2023 11:22:14 +0800 Subject: [PATCH 3/9] feat: refactor memory tables --- Cargo.lock | 1 + src/catalog/Cargo.toml | 1 + src/catalog/src/information_schema.rs | 71 +++++---- .../information_schema/empty_table_schemas.rs | 65 -------- src/catalog/src/information_schema/engines.rs | 147 ------------------ .../{empty_table.rs => memory_table.rs} | 50 ++++-- .../src/information_schema/memory_tables.rs | 93 +++++++++++ 7 files changed, 170 insertions(+), 258 deletions(-) delete mode 100644 src/catalog/src/information_schema/empty_table_schemas.rs delete mode 100644 src/catalog/src/information_schema/engines.rs rename src/catalog/src/information_schema/{empty_table.rs => memory_table.rs} (71%) create mode 100644 src/catalog/src/information_schema/memory_tables.rs diff --git a/Cargo.lock b/Cargo.lock index c0bbae37fd71..051626bd3670 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1192,6 +1192,7 @@ dependencies = [ "object-store", "parking_lot 0.12.1", "partition", + "paste", "prometheus", "regex", "serde", diff --git a/src/catalog/Cargo.toml b/src/catalog/Cargo.toml index fb41bf15d94d..93f5b8b91c52 100644 --- a/src/catalog/Cargo.toml +++ b/src/catalog/Cargo.toml @@ -33,6 +33,7 @@ meta-client.workspace = true moka = { workspace = true, features = ["future"] } parking_lot = "0.12" partition.workspace = true +paste = "1.0" prometheus.workspace = true regex.workspace = true serde.workspace = true diff --git a/src/catalog/src/information_schema.rs b/src/catalog/src/information_schema.rs index 3763340bcd8a..d8d8426ddabb 100644 --- a/src/catalog/src/information_schema.rs +++ b/src/catalog/src/information_schema.rs @@ -13,9 +13,8 @@ // limitations under the License. mod columns; -mod empty_table; -mod empty_table_schemas; -mod engines; +mod memory_table; +mod memory_tables; mod table_names; mod tables; @@ -26,8 +25,10 @@ use common_catalog::consts::{self, INFORMATION_SCHEMA_NAME}; use common_error::ext::BoxedError; use common_recordbatch::{RecordBatchStreamWrapper, SendableRecordBatchStream}; use datatypes::schema::SchemaRef; -use empty_table_schemas::get_schema; use futures_util::StreamExt; +use lazy_static::lazy_static; +use memory_tables::get_schema_columns; +use paste::paste; use snafu::ResultExt; use store_api::data_source::DataSource; use store_api::storage::{ScanRequest, TableId}; @@ -41,11 +42,35 @@ pub use table_names::*; use self::columns::InformationSchemaColumns; use crate::error::Result; -use crate::information_schema::empty_table::EmptyTable; -use crate::information_schema::engines::InformationSchemaEngines; +use crate::information_schema::memory_table::MemoryTable; use crate::information_schema::tables::InformationSchemaTables; use crate::CatalogManager; +lazy_static! { + // Memory tables in `information_schema`. + static ref MEMORY_TABLES: Vec<&'static str> = vec![ + ENGINES, + COLUMN_PRIVILEGES, + COLUMN_STATISTICS + ]; +} + +macro_rules! setup_memory_table { + ($name: expr) => { + paste! { + { + let (schema, columns) = get_schema_columns($name); + Some(Arc::new(MemoryTable::new( + consts::[], + $name, + schema, + columns + )) as _) + } + } + }; +} + pub struct InformationSchemaProvider { catalog_name: String, catalog_manager: Weak, @@ -70,16 +95,12 @@ impl InformationSchemaProvider { let mut schema = HashMap::new(); schema.insert(TABLES.to_owned(), provider.table(TABLES).unwrap()); schema.insert(COLUMNS.to_owned(), provider.table(COLUMNS).unwrap()); - schema.insert(ENGINES.to_owned(), provider.table(ENGINES).unwrap()); - // Tables not implemented - schema.insert( - COLUMN_PRIVILEGES.to_owned(), - provider.table(COLUMN_PRIVILEGES).unwrap(), - ); - schema.insert( - COLUMN_STATISTICS.to_owned(), - provider.table(COLUMN_STATISTICS).unwrap(), - ); + + // Add memory tables + for name in MEMORY_TABLES.iter() { + schema.insert((*name).to_owned(), provider.table(name).unwrap()); + } + schema } @@ -95,7 +116,8 @@ impl InformationSchemaProvider { } fn information_table(&self, name: &str) -> Option { - match name.to_ascii_lowercase().as_str() { + let name = name.to_ascii_lowercase(); + match name.clone().as_str() { TABLES => Some(Arc::new(InformationSchemaTables::new( self.catalog_name.clone(), self.catalog_manager.clone(), @@ -104,18 +126,9 @@ impl InformationSchemaProvider { self.catalog_name.clone(), self.catalog_manager.clone(), )) as _), - ENGINES => Some(Arc::new(InformationSchemaEngines::new()) as _), - // Table not implemented - COLUMN_PRIVILEGES => Some(Arc::new(EmptyTable::new( - consts::INFORMATION_SCHEMA_COLUMN_PRIVILEGES_TABLE_ID, - COLUMN_PRIVILEGES, - get_schema(COLUMN_PRIVILEGES), - ))), - COLUMN_STATISTICS => Some(Arc::new(EmptyTable::new( - consts::INFORMATION_SCHEMA_COLUMN_STATISTICS_TABLE_ID, - COLUMN_STATISTICS, - get_schema(COLUMN_STATISTICS), - ))), + ENGINES => setup_memory_table!(ENGINES), + COLUMN_PRIVILEGES => setup_memory_table!(COLUMN_PRIVILEGES), + COLUMN_STATISTICS => setup_memory_table!(COLUMN_STATISTICS), _ => None, } } diff --git a/src/catalog/src/information_schema/empty_table_schemas.rs b/src/catalog/src/information_schema/empty_table_schemas.rs deleted file mode 100644 index eee5b1c6b209..000000000000 --- a/src/catalog/src/information_schema/empty_table_schemas.rs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -/// Return the schemas of tables which are not implemented. -use datatypes::prelude::ConcreteDataType; -use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; - -use crate::information_schema::table_names::*; - -/// Find the schema by the table_name. -/// Safety: the user MUST ensure the table schema exists, panic otherwise. -pub fn get_schema(table_name: &'static str) -> SchemaRef { - let column_schemas = match table_name { - // https://dev.mysql.com/doc/refman/8.0/en/information-schema-column-privileges-table.html - COLUMN_PRIVILEGES => string_columns(&[ - "GRANTEE", - "TABLE_CATALOG", - "TABLE_SCHEMA", - "TABLE_NAME", - "COLUMN_NAME", - "PRIVILEGE_TYPE", - "IS_GRANTABLE", - ]), - // https://dev.mysql.com/doc/refman/8.0/en/information-schema-column-statistics-table.html - COLUMN_STATISTICS => string_columns(&[ - "SCHEMA_NAME", - "TABLE_NAME", - "COLUMN_NAME", - // TODO(dennis): It must be a JSON type, but we don't support it yet - "HISTOGRAM", - ]), - - _ => unreachable!("Unknown table in information_schema: {}", table_name), - }; - - Arc::new(Schema::new(column_schemas)) -} - -fn string_columns(names: &[&'static str]) -> Vec { - names.iter().map(|name| string_column(name)).collect() -} - -fn string_column(name: &str) -> ColumnSchema { - ColumnSchema::new( - str::to_lowercase(name), - ConcreteDataType::string_datatype(), - false, - ) -} - -#[cfg(test)] -mod tests {} diff --git a/src/catalog/src/information_schema/engines.rs b/src/catalog/src/information_schema/engines.rs deleted file mode 100644 index 045ce06754f7..000000000000 --- a/src/catalog/src/information_schema/engines.rs +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use arrow_schema::SchemaRef as ArrowSchemaRef; -use common_catalog::consts::{INFORMATION_SCHEMA_ENGINES_TABLE_ID, MITO_ENGINE}; -use common_error::ext::BoxedError; -use common_query::physical_plan::TaskContext; -use common_recordbatch::adapter::RecordBatchStreamAdapter; -use common_recordbatch::{RecordBatch, SendableRecordBatchStream}; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter; -use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream; -use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream; -use datatypes::prelude::{ConcreteDataType, VectorRef}; -use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; -use datatypes::vectors::StringVector; -use snafu::ResultExt; -use store_api::storage::TableId; - -use super::ENGINES; -use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result}; -use crate::information_schema::InformationTable; - -pub(super) struct InformationSchemaEngines { - schema: SchemaRef, -} - -impl InformationSchemaEngines { - pub(super) fn new() -> Self { - Self { - schema: Self::schema(), - } - } - - pub(crate) fn schema() -> SchemaRef { - Arc::new(Schema::new(vec![ - // The name of the storage engine. - ColumnSchema::new("engine", ConcreteDataType::string_datatype(), false), - // The level of support that the server has on the storage engine - ColumnSchema::new("support", ConcreteDataType::string_datatype(), false), - // The brief comment on the storage engine - ColumnSchema::new("comment", ConcreteDataType::string_datatype(), false), - // Whether the storage engine supports transactions. - ColumnSchema::new("transactions", ConcreteDataType::string_datatype(), false), - // Whether the storage engine supports XA transactions. - ColumnSchema::new("xa", ConcreteDataType::string_datatype(), true), - // Whether the storage engine supports `savepoints`. - ColumnSchema::new("savepoints", ConcreteDataType::string_datatype(), true), - ])) - } - - fn builder(&self) -> InformationSchemaEnginesBuilder { - InformationSchemaEnginesBuilder::new(self.schema.clone()) - } -} - -impl InformationTable for InformationSchemaEngines { - fn table_id(&self) -> TableId { - INFORMATION_SCHEMA_ENGINES_TABLE_ID - } - - fn table_name(&self) -> &'static str { - ENGINES - } - - fn schema(&self) -> SchemaRef { - self.schema.clone() - } - - fn to_stream(&self) -> Result { - let schema = self.schema.arrow_schema().clone(); - let mut builder = self.builder(); - let stream = Box::pin(DfRecordBatchStreamAdapter::new( - schema, - futures::stream::once(async move { - builder - .make_engines() - .await - .map(|x| x.into_df_record_batch()) - .map_err(Into::into) - }), - )); - Ok(Box::pin( - RecordBatchStreamAdapter::try_new(stream) - .map_err(BoxedError::new) - .context(InternalSnafu)?, - )) - } -} - -struct InformationSchemaEnginesBuilder { - schema: SchemaRef, -} - -impl InformationSchemaEnginesBuilder { - fn new(schema: SchemaRef) -> Self { - Self { schema } - } - - /// Construct the `information_schema.engines` virtual table - async fn make_engines(&mut self) -> Result { - let columns: Vec = vec![ - Arc::new(StringVector::from(vec![MITO_ENGINE])), - Arc::new(StringVector::from(vec!["DEFAULT"])), - Arc::new(StringVector::from(vec![ - "Storage engine for time-series data", - ])), - Arc::new(StringVector::from(vec!["NO"])), - Arc::new(StringVector::from(vec!["NO"])), - Arc::new(StringVector::from(vec!["NO"])), - ]; - RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu) - } -} - -impl DfPartitionStream for InformationSchemaEngines { - fn schema(&self) -> &ArrowSchemaRef { - self.schema.arrow_schema() - } - - fn execute(&self, _: Arc) -> DfSendableRecordBatchStream { - let schema = self.schema.arrow_schema().clone(); - let mut builder = self.builder(); - Box::pin(DfRecordBatchStreamAdapter::new( - schema, - futures::stream::once(async move { - builder - .make_engines() - .await - .map(|x| x.into_df_record_batch()) - .map_err(Into::into) - }), - )) - } -} diff --git a/src/catalog/src/information_schema/empty_table.rs b/src/catalog/src/information_schema/memory_table.rs similarity index 71% rename from src/catalog/src/information_schema/empty_table.rs rename to src/catalog/src/information_schema/memory_table.rs index 6c63cca5def5..49e8d3d3fb10 100644 --- a/src/catalog/src/information_schema/empty_table.rs +++ b/src/catalog/src/information_schema/memory_table.rs @@ -23,33 +23,43 @@ use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatch use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream; use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream; use datatypes::schema::SchemaRef; +use datatypes::vectors::VectorRef; use snafu::ResultExt; use store_api::storage::TableId; use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result}; use crate::information_schema::InformationTable; -/// An empty table with specific schema. -pub(super) struct EmptyTable { - schema: SchemaRef, +/// A memory table with specific schema and columns. +pub(super) struct MemoryTable { table_id: TableId, table_name: &'static str, + schema: SchemaRef, + columns: Vec, } -impl EmptyTable { - pub(super) fn new(table_id: TableId, table_name: &'static str, schema: SchemaRef) -> Self { +impl MemoryTable { + /// Creates a memory table with table id, name, schema and columns. + pub(super) fn new( + table_id: TableId, + table_name: &'static str, + schema: SchemaRef, + columns: Vec, + ) -> Self { Self { table_id, table_name, schema, + columns, } } - fn builder(&self) -> EmptyTableBuilder { - EmptyTableBuilder::new(self.schema.clone()) + + fn builder(&self) -> MemoryTableBuilder { + MemoryTableBuilder::new(self.schema.clone(), self.columns.clone()) } } -impl InformationTable for EmptyTable { +impl InformationTable for MemoryTable { fn table_id(&self) -> TableId { self.table_id } @@ -69,7 +79,7 @@ impl InformationTable for EmptyTable { schema, futures::stream::once(async move { builder - .empty_records() + .memory_records() .await .map(|x| x.into_df_record_batch()) .map_err(Into::into) @@ -83,22 +93,28 @@ impl InformationTable for EmptyTable { } } -struct EmptyTableBuilder { +struct MemoryTableBuilder { schema: SchemaRef, + columns: Vec, } -impl EmptyTableBuilder { - fn new(schema: SchemaRef) -> Self { - Self { schema } +impl MemoryTableBuilder { + fn new(schema: SchemaRef, columns: Vec) -> Self { + Self { schema, columns } } /// Construct the `information_schema.{table_name}` virtual table - async fn empty_records(&mut self) -> Result { - RecordBatch::new_empty(self.schema.clone()).context(CreateRecordBatchSnafu) + async fn memory_records(&mut self) -> Result { + if self.columns.is_empty() { + RecordBatch::new_empty(self.schema.clone()).context(CreateRecordBatchSnafu) + } else { + RecordBatch::new(self.schema.clone(), std::mem::take(&mut self.columns)) + .context(CreateRecordBatchSnafu) + } } } -impl DfPartitionStream for EmptyTable { +impl DfPartitionStream for MemoryTable { fn schema(&self) -> &ArrowSchemaRef { self.schema.arrow_schema() } @@ -110,7 +126,7 @@ impl DfPartitionStream for EmptyTable { schema, futures::stream::once(async move { builder - .empty_records() + .memory_records() .await .map(|x| x.into_df_record_batch()) .map_err(Into::into) diff --git a/src/catalog/src/information_schema/memory_tables.rs b/src/catalog/src/information_schema/memory_tables.rs new file mode 100644 index 000000000000..e25c796911f5 --- /dev/null +++ b/src/catalog/src/information_schema/memory_tables.rs @@ -0,0 +1,93 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use common_catalog::consts::MITO_ENGINE; +/// Return the schemas of tables which are not implemented. +use datatypes::prelude::{ConcreteDataType, VectorRef}; +use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; +use datatypes::vectors::StringVector; + +use crate::information_schema::table_names::*; + +/// Find the schema and columns by the table_name, only valid for memory tables. +/// Safety: the user MUST ensure the table schema exists, panic otherwise. +pub fn get_schema_columns(table_name: &str) -> (SchemaRef, Vec) { + let (column_schemas, columns): (_, Vec) = match table_name { + COLUMN_PRIVILEGES => ( + string_columns(&[ + "GRANTEE", + "TABLE_CATALOG", + "TABLE_SCHEMA", + "TABLE_NAME", + "COLUMN_NAME", + "PRIVILEGE_TYPE", + "IS_GRANTABLE", + ]), + vec![], + ), + + COLUMN_STATISTICS => ( + string_columns(&[ + "SCHEMA_NAME", + "TABLE_NAME", + "COLUMN_NAME", + // TODO(dennis): It must be a JSON type, but we don't support it yet + "HISTOGRAM", + ]), + vec![], + ), + + ENGINES => ( + string_columns(&[ + "ENGINE", + "SUPPORT", + "COMMENT", + "TRANSACTIONS", + "XA", + "SAVEPOINTS", + ]), + vec![ + Arc::new(StringVector::from(vec![MITO_ENGINE])), + Arc::new(StringVector::from(vec!["DEFAULT"])), + Arc::new(StringVector::from(vec![ + "Storage engine for time-series data", + ])), + Arc::new(StringVector::from(vec!["NO"])), + Arc::new(StringVector::from(vec!["NO"])), + Arc::new(StringVector::from(vec!["NO"])), + ], + ), + + _ => unreachable!("Unknown table in information_schema: {}", table_name), + }; + + (Arc::new(Schema::new(column_schemas)), columns) +} + +fn string_columns(names: &[&'static str]) -> Vec { + names.iter().map(|name| string_column(name)).collect() +} + +fn string_column(name: &str) -> ColumnSchema { + ColumnSchema::new( + str::to_lowercase(name), + ConcreteDataType::string_datatype(), + false, + ) +} + +#[cfg(test)] +mod tests {} From f5d5456839dbbbd7bc8a0b4bb85c04926d9d5c4c Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Fri, 15 Dec 2023 11:35:19 +0800 Subject: [PATCH 4/9] chore: rename memory_tables --- src/catalog/src/information_schema.rs | 4 +--- src/catalog/src/information_schema/memory_table.rs | 2 ++ .../{memory_tables.rs => memory_table/tables.rs} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename src/catalog/src/information_schema/{memory_tables.rs => memory_table/tables.rs} (100%) diff --git a/src/catalog/src/information_schema.rs b/src/catalog/src/information_schema.rs index d8d8426ddabb..d4b155be4d90 100644 --- a/src/catalog/src/information_schema.rs +++ b/src/catalog/src/information_schema.rs @@ -14,7 +14,6 @@ mod columns; mod memory_table; -mod memory_tables; mod table_names; mod tables; @@ -27,7 +26,6 @@ use common_recordbatch::{RecordBatchStreamWrapper, SendableRecordBatchStream}; use datatypes::schema::SchemaRef; use futures_util::StreamExt; use lazy_static::lazy_static; -use memory_tables::get_schema_columns; use paste::paste; use snafu::ResultExt; use store_api::data_source::DataSource; @@ -42,7 +40,7 @@ pub use table_names::*; use self::columns::InformationSchemaColumns; use crate::error::Result; -use crate::information_schema::memory_table::MemoryTable; +use crate::information_schema::memory_table::{get_schema_columns, MemoryTable}; use crate::information_schema::tables::InformationSchemaTables; use crate::CatalogManager; diff --git a/src/catalog/src/information_schema/memory_table.rs b/src/catalog/src/information_schema/memory_table.rs index 49e8d3d3fb10..05c345e79bea 100644 --- a/src/catalog/src/information_schema/memory_table.rs +++ b/src/catalog/src/information_schema/memory_table.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod tables; use std::sync::Arc; use arrow_schema::SchemaRef as ArrowSchemaRef; @@ -26,6 +27,7 @@ use datatypes::schema::SchemaRef; use datatypes::vectors::VectorRef; use snafu::ResultExt; use store_api::storage::TableId; +pub use tables::get_schema_columns; use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result}; use crate::information_schema::InformationTable; diff --git a/src/catalog/src/information_schema/memory_tables.rs b/src/catalog/src/information_schema/memory_table/tables.rs similarity index 100% rename from src/catalog/src/information_schema/memory_tables.rs rename to src/catalog/src/information_schema/memory_table/tables.rs From b851cb2bf14240d52474122b325a3d60be26c0ea Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Fri, 15 Dec 2023 14:44:20 +0800 Subject: [PATCH 5/9] test: adds unit tests --- .../src/information_schema/memory_table.rs | 78 ++++++++++++++++++- .../information_schema/memory_table/tables.rs | 19 ++++- 2 files changed, 94 insertions(+), 3 deletions(-) diff --git a/src/catalog/src/information_schema/memory_table.rs b/src/catalog/src/information_schema/memory_table.rs index 05c345e79bea..cce53c88a73c 100644 --- a/src/catalog/src/information_schema/memory_table.rs +++ b/src/catalog/src/information_schema/memory_table.rs @@ -32,7 +32,7 @@ pub use tables::get_schema_columns; use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result}; use crate::information_schema::InformationTable; -/// A memory table with specific schema and columns. +/// A memory table with specified schema and columns. pub(super) struct MemoryTable { table_id: TableId, table_name: &'static str, @@ -136,3 +136,79 @@ impl DfPartitionStream for MemoryTable { )) } } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use common_recordbatch::RecordBatches; + use datatypes::prelude::ConcreteDataType; + use datatypes::schema::{ColumnSchema, Schema}; + use datatypes::vectors::StringVector; + + use super::*; + + #[tokio::test] + async fn test_memory_table() { + let schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("a", ConcreteDataType::string_datatype(), false), + ColumnSchema::new("b", ConcreteDataType::string_datatype(), false), + ])); + + let table = MemoryTable::new( + 42, + "test", + schema.clone(), + vec![ + Arc::new(StringVector::from(vec!["a1", "a2"])), + Arc::new(StringVector::from(vec!["b1", "b2"])), + ], + ); + + assert_eq!(42, table.table_id()); + assert_eq!("test", table.table_name()); + assert_eq!(schema, InformationTable::schema(&table)); + + let stream = table.to_stream().unwrap(); + + let batches = RecordBatches::try_collect(stream).await.unwrap(); + + assert_eq!( + "\ ++----+----+ +| a | b | ++----+----+ +| a1 | b1 | +| a2 | b2 | ++----+----+", + batches.pretty_print().unwrap() + ); + } + + #[tokio::test] + async fn test_empty_memory_table() { + let schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("a", ConcreteDataType::string_datatype(), false), + ColumnSchema::new("b", ConcreteDataType::string_datatype(), false), + ])); + + let table = MemoryTable::new(42, "test", schema.clone(), vec![]); + + assert_eq!(42, table.table_id()); + assert_eq!("test", table.table_name()); + assert_eq!(schema, InformationTable::schema(&table)); + + let stream = table.to_stream().unwrap(); + + let batches = RecordBatches::try_collect(stream).await.unwrap(); + + assert_eq!( + "\ ++---+---+ +| a | b | ++---+---+ ++---+---+", + batches.pretty_print().unwrap() + ); + } +} diff --git a/src/catalog/src/information_schema/memory_table/tables.rs b/src/catalog/src/information_schema/memory_table/tables.rs index e25c796911f5..b17e1d997643 100644 --- a/src/catalog/src/information_schema/memory_table/tables.rs +++ b/src/catalog/src/information_schema/memory_table/tables.rs @@ -15,7 +15,6 @@ use std::sync::Arc; use common_catalog::consts::MITO_ENGINE; -/// Return the schemas of tables which are not implemented. use datatypes::prelude::{ConcreteDataType, VectorRef}; use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; use datatypes::vectors::StringVector; @@ -90,4 +89,20 @@ fn string_column(name: &str) -> ColumnSchema { } #[cfg(test)] -mod tests {} +mod tests { + use super::*; + + #[test] + fn test_string_columns() { + let columns = ["a", "b", "c"]; + let column_schemas = string_columns(&columns); + + assert_eq!(3, column_schemas.len()); + for (i, name) in columns.iter().enumerate() { + let cs = column_schemas.get(i).unwrap(); + + assert_eq!(*name, cs.name); + assert_eq!(ConcreteDataType::string_datatype(), cs.data_type); + } + } +} From 718d3ab1b61e68c9a1b1bf21ed022ba433e46078 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Fri, 15 Dec 2023 14:48:34 +0800 Subject: [PATCH 6/9] chore: format --- src/common/catalog/src/consts.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/catalog/src/consts.rs b/src/common/catalog/src/consts.rs index 3788f03adb22..1de1e263fae4 100644 --- a/src/common/catalog/src/consts.rs +++ b/src/common/catalog/src/consts.rs @@ -30,7 +30,7 @@ pub const SCRIPTS_TABLE_ID: u32 = 1; /// numbers table id pub const NUMBERS_TABLE_ID: u32 = 2; -/// ----- Begin of information_schema tables ---- +/// ----- Begin of information_schema tables ----- /// id for information_schema.tables pub const INFORMATION_SCHEMA_TABLES_TABLE_ID: u32 = 3; /// id for information_schema.columns @@ -41,7 +41,7 @@ pub const INFORMATION_SCHEMA_ENGINES_TABLE_ID: u32 = 5; pub const INFORMATION_SCHEMA_COLUMN_PRIVILEGES_TABLE_ID: u32 = 6; /// id for information_schema.column_statistics pub const INFORMATION_SCHEMA_COLUMN_STATISTICS_TABLE_ID: u32 = 7; -/// ----- End of information_schema tables ---- +/// ----- End of information_schema tables ----- pub const MITO_ENGINE: &str = "mito"; pub const MITO2_ENGINE: &str = "mito2"; From 40c78e3329a7c33ded0a1b9da04da513aa5600c5 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Fri, 15 Dec 2023 14:54:16 +0800 Subject: [PATCH 7/9] chore: style --- src/catalog/src/information_schema.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/catalog/src/information_schema.rs b/src/catalog/src/information_schema.rs index d4b155be4d90..6b1002565b16 100644 --- a/src/catalog/src/information_schema.rs +++ b/src/catalog/src/information_schema.rs @@ -91,12 +91,12 @@ impl InformationSchemaProvider { let provider = Self::new(catalog_name, catalog_manager); let mut schema = HashMap::new(); - schema.insert(TABLES.to_owned(), provider.table(TABLES).unwrap()); - schema.insert(COLUMNS.to_owned(), provider.table(COLUMNS).unwrap()); + schema.insert(TABLES.to_string(), provider.table(TABLES).unwrap()); + schema.insert(COLUMNS.to_string(), provider.table(COLUMNS).unwrap()); // Add memory tables for name in MEMORY_TABLES.iter() { - schema.insert((*name).to_owned(), provider.table(name).unwrap()); + schema.insert((*name).to_string(), provider.table(name).unwrap()); } schema @@ -140,9 +140,9 @@ impl InformationSchemaProvider { .unwrap(); let table_info = TableInfoBuilder::default() .table_id(table.table_id()) - .name(table.table_name().to_owned()) + .name(table.table_name().to_string()) .catalog_name(catalog_name) - .schema_name(INFORMATION_SCHEMA_NAME.to_owned()) + .schema_name(INFORMATION_SCHEMA_NAME.to_string()) .meta(table_meta) .table_type(table.table_type()) .build() From 241f8c8a0bbd129d567cebf54d8502d09cdd0ab0 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Sat, 16 Dec 2023 02:21:56 +0800 Subject: [PATCH 8/9] fix: by cr comments --- src/catalog/src/information_schema.rs | 64 ++++++++++++----- src/catalog/src/information_schema/columns.rs | 68 ++++++++----------- src/catalog/src/information_schema/tables.rs | 32 ++------- src/catalog/src/kvbackend/manager.rs | 15 ++-- src/catalog/src/memory/manager.rs | 3 +- tests-integration/src/tests/instance_test.rs | 5 +- .../common/show/show_databases_tables.result | 15 ++-- .../common/system/information_schema.result | 68 ++++++++++++------- 8 files changed, 147 insertions(+), 123 deletions(-) diff --git a/src/catalog/src/information_schema.rs b/src/catalog/src/information_schema.rs index 6b1002565b16..e3d49f041a07 100644 --- a/src/catalog/src/information_schema.rs +++ b/src/catalog/src/information_schema.rs @@ -46,7 +46,7 @@ use crate::CatalogManager; lazy_static! { // Memory tables in `information_schema`. - static ref MEMORY_TABLES: Vec<&'static str> = vec![ + static ref MEMORY_TABLES: &'static [&'static str] = &[ ENGINES, COLUMN_PRIVILEGES, COLUMN_STATISTICS @@ -72,37 +72,65 @@ macro_rules! setup_memory_table { pub struct InformationSchemaProvider { catalog_name: String, catalog_manager: Weak, + tables: Option>, } impl InformationSchemaProvider { pub fn new(catalog_name: String, catalog_manager: Weak) -> Self { - Self { + let mut provider = Self { catalog_name, catalog_manager, - } + tables: None, + }; + + provider.build_tables(); + + provider } - /// Build a map of [TableRef] in information schema. - /// Including `tables` and `columns`. - pub fn build( - catalog_name: String, - catalog_manager: Weak, - ) -> HashMap { - let provider = Self::new(catalog_name, catalog_manager); + /// Returns table names in the order of table id. + pub fn table_names(&self) -> Vec { + let mut tables = self.tables().into_values().collect::>(); + tables.sort_by(|t1, t2| { + t1.table_info() + .table_id() + .partial_cmp(&t2.table_info().table_id()) + .unwrap() + }); + + tables + .into_iter() + .map(|t| t.table_info().name.clone()) + .collect() + } - let mut schema = HashMap::new(); - schema.insert(TABLES.to_string(), provider.table(TABLES).unwrap()); - schema.insert(COLUMNS.to_string(), provider.table(COLUMNS).unwrap()); + /// Returns a map of [TableRef] in information schema. + pub fn tables(&self) -> HashMap { + // Safety: already built in `new`. + self.tables.clone().unwrap() + } + + /// Returns the [TableRef] by table name. + pub fn table(&self, name: &str) -> Option { + self.tables().get(name).cloned() + } + + fn build_tables(&mut self) -> HashMap { + let mut tables = HashMap::new(); + tables.insert(TABLES.to_string(), self.build_table(TABLES).unwrap()); + tables.insert(COLUMNS.to_string(), self.build_table(COLUMNS).unwrap()); // Add memory tables for name in MEMORY_TABLES.iter() { - schema.insert((*name).to_string(), provider.table(name).unwrap()); + tables.insert((*name).to_string(), self.build_table(name).unwrap()); } - schema + self.tables = Some(tables.clone()); + + tables } - pub fn table(&self, name: &str) -> Option { + fn build_table(&self, name: &str) -> Option { self.information_table(name).map(|table| { let table_info = Self::table_info(self.catalog_name.clone(), &table); let filter_pushdown = FilterPushDownType::Unsupported; @@ -114,8 +142,7 @@ impl InformationSchemaProvider { } fn information_table(&self, name: &str) -> Option { - let name = name.to_ascii_lowercase(); - match name.clone().as_str() { + match name.to_ascii_lowercase().as_str() { TABLES => Some(Arc::new(InformationSchemaTables::new( self.catalog_name.clone(), self.catalog_manager.clone(), @@ -214,6 +241,7 @@ impl DataSource for InformationTableDataSource { stream: Box::pin(stream), output_ordering: None, }; + Ok(Box::pin(stream)) } } diff --git a/src/catalog/src/information_schema/columns.rs b/src/catalog/src/information_schema/columns.rs index 34e9c7ef66c5..53f338783ad3 100644 --- a/src/catalog/src/information_schema/columns.rs +++ b/src/catalog/src/information_schema/columns.rs @@ -16,8 +16,8 @@ use std::sync::{Arc, Weak}; use arrow_schema::SchemaRef as ArrowSchemaRef; use common_catalog::consts::{ - INFORMATION_SCHEMA_COLUMNS_TABLE_ID, INFORMATION_SCHEMA_NAME, SEMANTIC_TYPE_FIELD, - SEMANTIC_TYPE_PRIMARY_KEY, SEMANTIC_TYPE_TIME_INDEX, + INFORMATION_SCHEMA_COLUMNS_TABLE_ID, SEMANTIC_TYPE_FIELD, SEMANTIC_TYPE_PRIMARY_KEY, + SEMANTIC_TYPE_TIME_INDEX, }; use common_error::ext::BoxedError; use common_query::physical_plan::TaskContext; @@ -33,8 +33,7 @@ use datatypes::vectors::{StringVectorBuilder, VectorRef}; use snafu::{OptionExt, ResultExt}; use store_api::storage::TableId; -use super::tables::InformationSchemaTables; -use super::{InformationTable, COLUMNS, TABLES}; +use super::{InformationTable, COLUMNS}; use crate::error::{ CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu, }; @@ -102,7 +101,7 @@ impl InformationTable for InformationSchemaColumns { schema, futures::stream::once(async move { builder - .make_tables() + .make_columns() .await .map(|x| x.into_df_record_batch()) .map_err(Into::into) @@ -148,8 +147,8 @@ impl InformationSchemaColumnsBuilder { } } - /// Construct the `information_schema.tables` virtual table - async fn make_tables(&mut self) -> Result { + /// Construct the `information_schema.columns` virtual table + async fn make_columns(&mut self) -> Result { let catalog_name = self.catalog_name.clone(); let catalog_manager = self .catalog_manager @@ -163,48 +162,38 @@ impl InformationSchemaColumnsBuilder { { continue; } + for table_name in catalog_manager .table_names(&catalog_name, &schema_name) .await? { - let (keys, schema) = if let Some(table) = catalog_manager + if let Some(table) = catalog_manager .table(&catalog_name, &schema_name, &table_name) .await? { let keys = &table.table_info().meta.primary_key_indices; let schema = table.schema(); - (keys.clone(), schema) - } else { - // TODO: this specific branch is only a workaround for FrontendCatalogManager. - if schema_name == INFORMATION_SCHEMA_NAME { - if table_name == COLUMNS { - (vec![], InformationSchemaColumns::schema()) - } else if table_name == TABLES { - (vec![], InformationSchemaTables::schema()) + + for (idx, column) in schema.column_schemas().iter().enumerate() { + let semantic_type = if column.is_time_index() { + SEMANTIC_TYPE_TIME_INDEX + } else if keys.contains(&idx) { + SEMANTIC_TYPE_PRIMARY_KEY } else { - continue; - } - } else { - continue; + SEMANTIC_TYPE_FIELD + }; + + self.add_column( + &catalog_name, + &schema_name, + &table_name, + &column.name, + &column.data_type.name(), + semantic_type, + ); } - }; - - for (idx, column) in schema.column_schemas().iter().enumerate() { - let semantic_type = if column.is_time_index() { - SEMANTIC_TYPE_TIME_INDEX - } else if keys.contains(&idx) { - SEMANTIC_TYPE_PRIMARY_KEY - } else { - SEMANTIC_TYPE_FIELD - }; - self.add_column( - &catalog_name, - &schema_name, - &table_name, - &column.name, - &column.data_type.name(), - semantic_type, - ); + } else { + unreachable!(); } } } @@ -238,6 +227,7 @@ impl InformationSchemaColumnsBuilder { Arc::new(self.data_types.finish()), Arc::new(self.semantic_types.finish()), ]; + RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu) } } @@ -254,7 +244,7 @@ impl DfPartitionStream for InformationSchemaColumns { schema, futures::stream::once(async move { builder - .make_tables() + .make_columns() .await .map(|x| x.into_df_record_batch()) .map_err(Into::into) diff --git a/src/catalog/src/information_schema/tables.rs b/src/catalog/src/information_schema/tables.rs index a626dbfdd31a..d258dd490130 100644 --- a/src/catalog/src/information_schema/tables.rs +++ b/src/catalog/src/information_schema/tables.rs @@ -15,10 +15,7 @@ use std::sync::{Arc, Weak}; use arrow_schema::SchemaRef as ArrowSchemaRef; -use common_catalog::consts::{ - INFORMATION_SCHEMA_COLUMNS_TABLE_ID, INFORMATION_SCHEMA_NAME, - INFORMATION_SCHEMA_TABLES_TABLE_ID, -}; +use common_catalog::consts::INFORMATION_SCHEMA_TABLES_TABLE_ID; use common_error::ext::BoxedError; use common_query::physical_plan::TaskContext; use common_recordbatch::adapter::RecordBatchStreamAdapter; @@ -33,7 +30,7 @@ use snafu::{OptionExt, ResultExt}; use store_api::storage::TableId; use table::metadata::TableType; -use super::{COLUMNS, TABLES}; +use super::TABLES; use crate::error::{ CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu, }; @@ -178,29 +175,8 @@ impl InformationSchemaTablesBuilder { Some(&table_info.meta.engine), ); } else { - // TODO: this specific branch is only a workaround for FrontendCatalogManager. - if schema_name == INFORMATION_SCHEMA_NAME { - if table_name == COLUMNS { - self.add_table( - &catalog_name, - &schema_name, - &table_name, - TableType::Temporary, - Some(INFORMATION_SCHEMA_COLUMNS_TABLE_ID), - None, - ); - } else if table_name == TABLES { - self.add_table( - &catalog_name, - &schema_name, - &table_name, - TableType::Temporary, - Some(INFORMATION_SCHEMA_TABLES_TABLE_ID), - None, - ); - } - } - }; + unreachable!(); + } } } diff --git a/src/catalog/src/kvbackend/manager.rs b/src/catalog/src/kvbackend/manager.rs index 2c5028b40908..93536ab73f41 100644 --- a/src/catalog/src/kvbackend/manager.rs +++ b/src/catalog/src/kvbackend/manager.rs @@ -38,7 +38,7 @@ use crate::error::{ self as catalog_err, ListCatalogsSnafu, ListSchemasSnafu, Result as CatalogResult, TableMetadataManagerSnafu, }; -use crate::information_schema::{InformationSchemaProvider, COLUMNS, TABLES}; +use crate::information_schema::InformationSchemaProvider; use crate::CatalogManager; /// Access all existing catalog, schema and tables. @@ -81,6 +81,11 @@ impl KvBackendCatalogManager { cache_invalidator, system_catalog: SystemCatalog { catalog_manager: me.clone(), + information_schema_provider: Arc::new(InformationSchemaProvider::new( + // The catalog name is not used in system_catalog, so let it empty + "".to_string(), + me.clone(), + )), }, }) } @@ -231,11 +236,11 @@ impl CatalogManager for KvBackendCatalogManager { // a new catalog is created. /// Existing system tables: /// - public.numbers -/// - information_schema.tables -/// - information_schema.columns +/// - information_schema.{tables} #[derive(Clone)] struct SystemCatalog { catalog_manager: Weak, + information_schema_provider: Arc, } impl SystemCatalog { @@ -245,7 +250,7 @@ impl SystemCatalog { fn table_names(&self, schema: &str) -> Vec { if schema == INFORMATION_SCHEMA_NAME { - vec![TABLES.to_string(), COLUMNS.to_string()] + self.information_schema_provider.table_names() } else if schema == DEFAULT_SCHEMA_NAME { vec![NUMBERS_TABLE_NAME.to_string()] } else { @@ -259,7 +264,7 @@ impl SystemCatalog { fn table_exist(&self, schema: &str, table: &str) -> bool { if schema == INFORMATION_SCHEMA_NAME { - table == TABLES || table == COLUMNS + self.information_schema_provider.table(table).is_some() } else if schema == DEFAULT_SCHEMA_NAME { table == NUMBERS_TABLE_NAME } else { diff --git a/src/catalog/src/memory/manager.rs b/src/catalog/src/memory/manager.rs index 5d08c1162600..d35d734f40fc 100644 --- a/src/catalog/src/memory/manager.rs +++ b/src/catalog/src/memory/manager.rs @@ -243,10 +243,11 @@ impl MemoryCatalogManager { } fn create_catalog_entry(self: &Arc, catalog: String) -> SchemaEntries { - let information_schema = InformationSchemaProvider::build( + let information_schema_provider = InformationSchemaProvider::new( catalog, Arc::downgrade(self) as Weak, ); + let information_schema = information_schema_provider.tables(); let mut catalog = HashMap::new(); catalog.insert(INFORMATION_SCHEMA_NAME.to_string(), information_schema); catalog diff --git a/tests-integration/src/tests/instance_test.rs b/tests-integration/src/tests/instance_test.rs index 6bf5fc0d4784..5054182b56d4 100644 --- a/tests-integration/src/tests/instance_test.rs +++ b/tests-integration/src/tests/instance_test.rs @@ -1732,7 +1732,7 @@ async fn test_information_schema_dot_tables(instance: Arc) { // User can only see information schema under current catalog. // A necessary requirement to GreptimeCloud. - let sql = "select table_catalog, table_schema, table_name, table_type, table_id, engine from information_schema.tables where table_type != 'SYSTEM VIEW' order by table_name"; + let sql = "select table_catalog, table_schema, table_name, table_type, table_id, engine from information_schema.tables where table_type != 'SYSTEM VIEW' and table_name in ('columns', 'numbers', 'tables', 'another_table') order by table_name"; let output = execute_sql(&instance, sql).await; let expected = "\ @@ -1760,6 +1760,7 @@ async fn test_information_schema_dot_tables(instance: Arc) { #[apply(both_instances_cases)] async fn test_information_schema_dot_columns(instance: Arc) { + logging::init_default_ut_logging(); let instance = instance.frontend(); let sql = "create table another_table(i timestamp time index)"; @@ -1769,7 +1770,7 @@ async fn test_information_schema_dot_columns(instance: Arc) { // User can only see information schema under current catalog. // A necessary requirement to GreptimeCloud. - let sql = "select table_catalog, table_schema, table_name, column_name, data_type, semantic_type from information_schema.columns order by table_name"; + let sql = "select table_catalog, table_schema, table_name, column_name, data_type, semantic_type from information_schema.columns where table_name in ('columns', 'numbers', 'tables', 'another_table') order by table_name"; let output = execute_sql(&instance, sql).await; let expected = "\ diff --git a/tests/cases/standalone/common/show/show_databases_tables.result b/tests/cases/standalone/common/show/show_databases_tables.result index c4e4f402603e..d68f1d991612 100644 --- a/tests/cases/standalone/common/show/show_databases_tables.result +++ b/tests/cases/standalone/common/show/show_databases_tables.result @@ -16,10 +16,13 @@ Affected Rows: 0 show tables; -+---------+ -| Tables | -+---------+ -| columns | -| tables | -+---------+ ++-------------------+ +| Tables | ++-------------------+ +| column_privileges | +| column_statistics | +| columns | +| engines | +| tables | ++-------------------+ diff --git a/tests/cases/standalone/common/system/information_schema.result b/tests/cases/standalone/common/system/information_schema.result index be60fdd701e5..e9bb4535e201 100644 --- a/tests/cases/standalone/common/system/information_schema.result +++ b/tests/cases/standalone/common/system/information_schema.result @@ -4,33 +4,53 @@ from information_schema.tables where table_name != 'scripts' order by table_schema, table_name; -+---------------+--------------------+------------+-----------------+----------+-------------+ -| table_catalog | table_schema | table_name | table_type | table_id | engine | -+---------------+--------------------+------------+-----------------+----------+-------------+ -| greptime | information_schema | columns | LOCAL TEMPORARY | 4 | | -| greptime | information_schema | tables | LOCAL TEMPORARY | 3 | | -| greptime | public | numbers | LOCAL TEMPORARY | 2 | test_engine | -+---------------+--------------------+------------+-----------------+----------+-------------+ ++---------------+--------------------+-------------------+-----------------+----------+-------------+ +| table_catalog | table_schema | table_name | table_type | table_id | engine | ++---------------+--------------------+-------------------+-----------------+----------+-------------+ +| greptime | information_schema | column_privileges | LOCAL TEMPORARY | 6 | | +| greptime | information_schema | column_statistics | LOCAL TEMPORARY | 7 | | +| greptime | information_schema | columns | LOCAL TEMPORARY | 4 | | +| greptime | information_schema | engines | LOCAL TEMPORARY | 5 | | +| greptime | information_schema | tables | LOCAL TEMPORARY | 3 | | +| greptime | public | numbers | LOCAL TEMPORARY | 2 | test_engine | ++---------------+--------------------+-------------------+-----------------+----------+-------------+ select * from information_schema.columns order by table_schema, table_name; -+---------------+--------------------+------------+---------------+-----------+---------------+ -| table_catalog | table_schema | table_name | column_name | data_type | semantic_type | -+---------------+--------------------+------------+---------------+-----------+---------------+ -| greptime | information_schema | columns | table_catalog | String | FIELD | -| greptime | information_schema | columns | table_schema | String | FIELD | -| greptime | information_schema | columns | table_name | String | FIELD | -| greptime | information_schema | columns | column_name | String | FIELD | -| greptime | information_schema | columns | data_type | String | FIELD | -| greptime | information_schema | columns | semantic_type | String | FIELD | -| greptime | information_schema | tables | table_catalog | String | FIELD | -| greptime | information_schema | tables | table_schema | String | FIELD | -| greptime | information_schema | tables | table_name | String | FIELD | -| greptime | information_schema | tables | table_type | String | FIELD | -| greptime | information_schema | tables | table_id | UInt32 | FIELD | -| greptime | information_schema | tables | engine | String | FIELD | -| greptime | public | numbers | number | UInt32 | TAG | -+---------------+--------------------+------------+---------------+-----------+---------------+ ++---------------+--------------------+-------------------+----------------+-----------+---------------+ +| table_catalog | table_schema | table_name | column_name | data_type | semantic_type | ++---------------+--------------------+-------------------+----------------+-----------+---------------+ +| greptime | information_schema | column_privileges | grantee | String | FIELD | +| greptime | information_schema | column_privileges | is_grantable | String | FIELD | +| greptime | information_schema | column_privileges | privilege_type | String | FIELD | +| greptime | information_schema | column_privileges | column_name | String | FIELD | +| greptime | information_schema | column_privileges | table_name | String | FIELD | +| greptime | information_schema | column_privileges | table_schema | String | FIELD | +| greptime | information_schema | column_privileges | table_catalog | String | FIELD | +| greptime | information_schema | column_statistics | histogram | String | FIELD | +| greptime | information_schema | column_statistics | column_name | String | FIELD | +| greptime | information_schema | column_statistics | table_name | String | FIELD | +| greptime | information_schema | column_statistics | schema_name | String | FIELD | +| greptime | information_schema | columns | table_catalog | String | FIELD | +| greptime | information_schema | columns | semantic_type | String | FIELD | +| greptime | information_schema | columns | data_type | String | FIELD | +| greptime | information_schema | columns | column_name | String | FIELD | +| greptime | information_schema | columns | table_name | String | FIELD | +| greptime | information_schema | columns | table_schema | String | FIELD | +| greptime | information_schema | engines | engine | String | FIELD | +| greptime | information_schema | engines | support | String | FIELD | +| greptime | information_schema | engines | comment | String | FIELD | +| greptime | information_schema | engines | transactions | String | FIELD | +| greptime | information_schema | engines | xa | String | FIELD | +| greptime | information_schema | engines | savepoints | String | FIELD | +| greptime | information_schema | tables | table_catalog | String | FIELD | +| greptime | information_schema | tables | engine | String | FIELD | +| greptime | information_schema | tables | table_id | UInt32 | FIELD | +| greptime | information_schema | tables | table_type | String | FIELD | +| greptime | information_schema | tables | table_name | String | FIELD | +| greptime | information_schema | tables | table_schema | String | FIELD | +| greptime | public | numbers | number | UInt32 | TAG | ++---------------+--------------------+-------------------+----------------+-----------+---------------+ create database my_db; From a9a31757ab3873ea01bc84766f4245539f6453fb Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Sat, 16 Dec 2023 09:57:44 +0800 Subject: [PATCH 9/9] refactor: tables --- src/catalog/src/information_schema.rs | 24 ++++++++++++------------ src/catalog/src/memory/manager.rs | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/catalog/src/information_schema.rs b/src/catalog/src/information_schema.rs index e3d49f041a07..839f83035010 100644 --- a/src/catalog/src/information_schema.rs +++ b/src/catalog/src/information_schema.rs @@ -69,10 +69,11 @@ macro_rules! setup_memory_table { }; } +/// The `information_schema` tables info provider. pub struct InformationSchemaProvider { catalog_name: String, catalog_manager: Weak, - tables: Option>, + tables: HashMap, } impl InformationSchemaProvider { @@ -80,7 +81,7 @@ impl InformationSchemaProvider { let mut provider = Self { catalog_name, catalog_manager, - tables: None, + tables: HashMap::new(), }; provider.build_tables(); @@ -90,14 +91,14 @@ impl InformationSchemaProvider { /// Returns table names in the order of table id. pub fn table_names(&self) -> Vec { - let mut tables = self.tables().into_values().collect::>(); + let mut tables = self.tables.values().clone().collect::>(); + tables.sort_by(|t1, t2| { t1.table_info() .table_id() .partial_cmp(&t2.table_info().table_id()) .unwrap() }); - tables .into_iter() .map(|t| t.table_info().name.clone()) @@ -105,17 +106,18 @@ impl InformationSchemaProvider { } /// Returns a map of [TableRef] in information schema. - pub fn tables(&self) -> HashMap { - // Safety: already built in `new`. - self.tables.clone().unwrap() + pub fn tables(&self) -> &HashMap { + assert!(!self.tables.is_empty()); + + &self.tables } /// Returns the [TableRef] by table name. pub fn table(&self, name: &str) -> Option { - self.tables().get(name).cloned() + self.tables.get(name).cloned() } - fn build_tables(&mut self) -> HashMap { + fn build_tables(&mut self) { let mut tables = HashMap::new(); tables.insert(TABLES.to_string(), self.build_table(TABLES).unwrap()); tables.insert(COLUMNS.to_string(), self.build_table(COLUMNS).unwrap()); @@ -125,9 +127,7 @@ impl InformationSchemaProvider { tables.insert((*name).to_string(), self.build_table(name).unwrap()); } - self.tables = Some(tables.clone()); - - tables + self.tables = tables; } fn build_table(&self, name: &str) -> Option { diff --git a/src/catalog/src/memory/manager.rs b/src/catalog/src/memory/manager.rs index d35d734f40fc..745256beeaa8 100644 --- a/src/catalog/src/memory/manager.rs +++ b/src/catalog/src/memory/manager.rs @@ -247,7 +247,7 @@ impl MemoryCatalogManager { catalog, Arc::downgrade(self) as Weak, ); - let information_schema = information_schema_provider.tables(); + let information_schema = information_schema_provider.tables().clone(); let mut catalog = HashMap::new(); catalog.insert(INFORMATION_SCHEMA_NAME.to_string(), information_schema); catalog