From 6535b5d38d6d692c9a86d8f1d051bb0c19f0a935 Mon Sep 17 00:00:00 2001 From: "a.gacenko" Date: Thu, 20 Jun 2024 16:05:17 +0700 Subject: [PATCH] feat: Add terraform modules Managed-clickhouse [Yandex Cloud] --- README.md | 1 + YandexCloud/Managed-clickhouse/README.md | 11 + .../managed-clickhouse/README.md | 284 +++++++++++++ .../managed-clickhouse/cluster.tf | 378 ++++++++++++++++++ .../docs/example/example.tf | 155 +++++++ .../docs/example/versions.tf | 10 + .../managed-clickhouse/password.tf | 12 + .../managed-clickhouse/variables.tf | 5 + .../managed-clickhouse/versions.tf | 10 + 9 files changed, 866 insertions(+) create mode 100644 YandexCloud/Managed-clickhouse/README.md create mode 100644 YandexCloud/Managed-clickhouse/managed-clickhouse/README.md create mode 100644 YandexCloud/Managed-clickhouse/managed-clickhouse/cluster.tf create mode 100644 YandexCloud/Managed-clickhouse/managed-clickhouse/docs/example/example.tf create mode 100644 YandexCloud/Managed-clickhouse/managed-clickhouse/docs/example/versions.tf create mode 100644 YandexCloud/Managed-clickhouse/managed-clickhouse/password.tf create mode 100644 YandexCloud/Managed-clickhouse/managed-clickhouse/variables.tf create mode 100644 YandexCloud/Managed-clickhouse/managed-clickhouse/versions.tf diff --git a/README.md b/README.md index 21244d0..eaa4db6 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ The available variables are listed in each modules's README file, along with the 7. [Storage](https://github.com/nixys/nxs-marketplace-terraform/blob/main/YandexCloud/Storage/README.md) - create resources in Object Storage 8. [Managed-opensearch](https://github.com/nixys/nxs-marketplace-terraform/blob/main/YandexCloud/Managed-opensearch/README.md) - create resources in Managed Service for OpenSearch 9. [Managed-redis](https://github.com/nixys/nxs-marketplace-terraform/blob/main/YandexCloud/Managed-redis/README.md) - create resources in Managed Service for Redis +10. [Managed-clickhouse](https://github.com/nixys/nxs-marketplace-terraform/blob/main/YandexCloud/Managed-clickhouse/README.md) - create resources in Managed Service for ClickHouse diff --git a/YandexCloud/Managed-clickhouse/README.md b/YandexCloud/Managed-clickhouse/README.md new file mode 100644 index 0000000..825cebc --- /dev/null +++ b/YandexCloud/Managed-clickhouse/README.md @@ -0,0 +1,11 @@ +# Managed-clickhouse + +## Introduction + +This is a set of terraform modules for the Yandex Cloud provider for building a Managed clickhouse and creating any different clickhouse resources + +## Modules + +| Modules | Settings | Resources | Description | +| --- | --- | --- | --- | +| `managed-clickhouse` |[managed-clickhouse](managed-clickhouse/README.md)| - yandex_mdb_clickhouse_cluster
- random_password | Managed-clickhouse settings | diff --git a/YandexCloud/Managed-clickhouse/managed-clickhouse/README.md b/YandexCloud/Managed-clickhouse/managed-clickhouse/README.md new file mode 100644 index 0000000..9cc3139 --- /dev/null +++ b/YandexCloud/Managed-clickhouse/managed-clickhouse/README.md @@ -0,0 +1,284 @@ +# Managed-clickhouse + +## Introduction + +This is a set of terraform modules for the Yandex Cloud provider for building a Managed-clickhouse and creating any different clickhouse resources + +## Features + +- Supported clickhouse-cluster + +## Settings + +| Option | Type | Required | Default value |Description | +| --- | --- | --- | --- | --- | +| `clusters.network_id` | String | Yes | - | ID of the network, to which the Clickhouse cluster belongs. | +| `clusters.description` | String | No | "Created by Terraform" | Description of the Clickhouse cluster. | +| `clusters.environment` | String | Yes | "PRODUCTION" | Deployment environment of the Clickhouse cluster. | +| `clusters.version` | Int | No | null | Version of the ClickHouse server software. | +| `clusters.deletion_protection` | Bool | No | false | Inhibits deletion of the cluster. Can be either true or false. | +| `clusters.resources_preset_id` | String | Yes | - | The ID of the preset for computational resources available to a host (CPU, memory etc.). | +| `clusters.disk_size` | String | Yes | "network-hdd" | Volume of the storage available to a host, in gigabytes. | +| `clusters.disk_type_id` | Int | Yes | 10 | Type of the storage of hosts. | +| `clusters.config_log_level` | String | No | "TRACE" | Clickhouse settings. | +| `clusters.config_max_connections` | Int | No | 100 | Clickhouse settings. | +| `clusters.config_max_concurrent_queries` | Int | No | 50 | Clickhouse settings. | +| `clusters.config_keep_alive_timeout` | Int | No | 3000 | Clickhouse settings. | +| `clusters.config_uncompressed_cache_size` | Int | No | 8589934592 | Clickhouse settings. | +| `clusters.config_mark_cache_size` | Int | No | 5368709120 | Clickhouse settings. | +| `clusters.config_max_table_size_to_drop` | Int | No | 53687091200 | Clickhouse settings. | +| `clusters.config_max_partition_size_to_drop` | Int | No | 53687091200 | Clickhouse settings. | +| `clusters.config_timezone` | String | No | "UTC" | Clickhouse settings. | +| `clusters.config_geobase_uri` | String | No | null | Clickhouse settings. | +| `clusters.config_query_log_retention_size` | Int | No | 1073741824 | Clickhouse settings. | +| `clusters.config_query_log_retention_time` | Int | No | 259200000 | Clickhouse settings. | +| `clusters.config_query_thread_log_enabled` | String | No | true | Clickhouse settings. | +| `clusters.config_query_thread_log_retention_size` | Int | No | 536870912 | Clickhouse settings. | +| `clusters.config_query_thread_log_retention_time` | Int | No | 259200000 | Clickhouse settings. | +| `clusters.config_part_log_retention_size` | Int | No | 536870912 | Clickhouse settings. | +| `clusters.config_part_log_retention_time` | Int | No | 259200000 | Clickhouse settings. | +| `clusters.config_metric_log_enabled` | Bool | No | true | Clickhouse settings. | +| `clusters.config_metric_log_retention_size` | Int | No | 536870912 | Clickhouse settings. | +| `clusters.config_metric_log_retention_time` | Int | No | 259200000 | Clickhouse settings. | +| `clusters.config_trace_log_enabled` | Bool | No | true | Clickhouse settings. | +| `clusters.config_trace_log_retention_size` | Int | No | 536870912 | Clickhouse settings. | +| `clusters.config_trace_log_retention_time` | Int | No | 259200000 | Clickhouse settings. | +| `clusters.config_text_log_enabled` | Bool | No | true | Clickhouse settings. | +| `clusters.config_text_log_retention_size` | Int | No | 536870912 | Clickhouse settings. | +| `clusters.config_text_log_retention_time` | Int | No | 259200000 | Clickhouse settings. | +| `clusters.config_text_log_level` | String | No | "TRACE" | Clickhouse settings. | +| `clusters.config_background_pool_size` | Int | No | 16 | Clickhouse settings. | +| `clusters.config_background_schedule_pool_size` | Int | No | 16 | Clickhouse settings. | +| `clusters.merge_tree` | List | No | [] | MergeTree engine configuration. | +| `clusters.merge_tree.replicated_deduplication_window` | Int | No | null | Replicated deduplication window: Number of recent hash blocks that ZooKeeper will store (the old ones will be deleted). | +| `clusters.merge_tree.replicated_deduplication_window_seconds` | Int | No | null | Replicated deduplication window seconds: Time during which ZooKeeper stores the hash blocks (the old ones wil be deleted). | +| `clusters.merge_tree.parts_to_delay_insert` | Int | No | null | Parts to delay insert: Number of active data parts in a table, on exceeding which ClickHouse starts artificially reduce the rate of inserting data into the table. | +| `clusters.merge_tree.parts_to_throw_insert` | Int | No | null | Parts to throw insert: Threshold value of active data parts in a table, on exceeding which ClickHouse throws the 'Too many parts …' exception. | +| `clusters.merge_tree.max_replicated_merges_in_queue` | Int | No | null | Max replicated merges in queue: Maximum number of merge tasks that can be in the ReplicatedMergeTree queue at the same time. | +| `clusters.merge_tree.number_of_free_entries_in_pool_to_lower_max_size_of_merge` | Int | No | null | Number of free entries in pool to lower max size of merge: Threshold value of free entries in the pool. If the number of entries in the pool falls below this value, ClickHouse reduces the maximum size of a data part to merge. This helps handle small merges faster, rather than filling the pool with lengthy merges. | +| `clusters.merge_tree.max_bytes_to_merge_at_min_space_in_pool` | Int | No | null | Max bytes to merge at min space in pool: Maximum total size of a data part to merge when the number of free threads in the background pool is minimum. | +| `clusters.merge_tree.min_bytes_for_wide_part` | Int | No | null | Minimum number of bytes in a data part that can be stored in Wide format. You can set one, both or none of these settings. | +| `clusters.merge_tree.min_rows_for_wide_part` | Int | No | null | Minimum number of rows in a data part that can be stored in Wide format. You can set one, both or none of these settings. | +| `clusters.merge_tree.ttl_only_drop_parts` | String | No | null | Enables or disables complete dropping of data parts where all rows are expired in MergeTree tables. | +| `clusters.merge_tree.merge_with_ttl_timeout` | Int | No | null | Minimum delay in seconds before repeating a merge with delete TTL. Default value: 14400 seconds (4 hours). | +| `clusters.merge_tree.merge_with_recompression_ttl_timeout` | Int | No | null | Minimum delay in seconds before repeating a merge with recompression TTL. Default value: 14400 seconds (4 hours). | +| `clusters.merge_tree.max_parts_in_total` | Int | No | null | Maximum number of parts in all partitions. | +| `clusters.merge_tree.max_number_of_merges_with_ttl_in_pool` | Int | No | null | When there is more than specified number of merges with TTL entries in pool, do not assign new merge with TTL. | +| `clusters.merge_tree.cleanup_delay_period` | Int | No | null | Minimum period to clean old queue logs, blocks hashes and parts. | +| `clusters.merge_tree.max_avg_part_size_for_too_many_parts` | Int | No | null | The too many parts check according to parts_to_delay_insert and parts_to_throw_insert will be active only if the average part size (in the relevant partition) is not larger than the specified threshold. If it is larger than the specified threshold, the INSERTs will be neither delayed or rejected. This allows to have hundreds of terabytes in a single table on a single server if the parts are successfully merged to larger parts. This does not affect the thresholds on inactive parts or total parts. | +| `clusters.merge_tree.min_age_to_force_merge_seconds` | Int | No | null | Merge parts if every part in the range is older than the value of min_age_to_force_merge_seconds. | +| `clusters.merge_tree.min_age_to_force_merge_on_partition_only` | Int | No | null | Whether min_age_to_force_merge_seconds should be applied only on the entire partition and not on subset. | +| `clusters.merge_tree.merge_selecting_sleep_ms` | Int | No | null | Sleep time for merge selecting when no part is selected. A lower setting triggers selecting tasks in background_schedule_pool frequently, which results in a large number of requests to ClickHouse Keeper in large-scale clusters. | +| `clusters.kafka` | List | No | [] | Kafka engine configuration. | +| `clusters.kafka.security_protocol` | String | No | null | Security protocol used to connect to kafka server. | +| `clusters.kafka.sasl_mechanism` | String | No | null | SASL mechanism used in kafka authentication. | +| `clusters.kafka.sasl_username` | String | No | null | Username on kafka server. | +| `clusters.kafka.sasl_password` | String | No | null | User password on kafka server. | +| `clusters.kafka.enable_ssl_certificate_verification` | String | No | null | enable verification of SSL certificates. | +| `clusters.kafka.max_poll_interval_ms` | String | No | null | Maximum allowed time between calls to consume messages (e.g., rd_kafka_consumer_poll()) for high-level consumers. If this interval is exceeded the consumer is considered failed and the group will rebalance in order to reassign the partitions to another consumer group member. | +| `clusters.kafka.session_timeout_ms` | Int | No | null | Client group session and failure detection timeout. The consumer sends periodic heartbeats (heartbeat.interval.ms) to indicate its liveness to the broker. If no hearts are received by the broker for a group member within the session timeout, the broker will remove the consumer from the group and trigger a rebalance. | +| `clusters.kafka_topic` | List | No | [] | MKafka topic connection configuration. | +| `clusters.kafka_topic.name` | String | Yes | - | Kafka topic name. | +| `clusters.kafka_topic.security_protocol` | String | No | null | Security protocol used to connect to kafka server. | +| `clusters.kafka_topic.sasl_mechanism` | String | No | null | SASL mechanism used in kafka authentication. | +| `clusters.kafka_topic.sasl_username` | String | No | null | Username on kafka server. | +| `clusters.kafka_topic.sasl_password` | String | No | null | User password on kafka server. | +| `clusters.kafka_topic.enable_ssl_certificate_verification` | String | No | null | enable verification of SSL certificates. | +| `clusters.kafka_topic.max_poll_interval_ms` | String | No | null | Maximum allowed time between calls to consume messages (e.g., rd_kafka_consumer_poll()) for high-level consumers. If this interval is exceeded the consumer is considered failed and the group will rebalance in order to reassign the partitions to another consumer group member. | +| `clusters.kafka_topic.session_timeout_ms` | Int | No | null | Client group session and failure detection timeout. The consumer sends periodic heartbeats (heartbeat.interval.ms) to indicate its liveness to the broker. If no hearts are received by the broker for a group member within the session timeout, the broker will remove the consumer from the group and trigger a rebalance. | +| `clusters.compression` | List | No | [] | Data compression configuration. | +| `clusters.compression.method` | String | No | null | Method: Compression method. Two methods are available: LZ4 and zstd. | +| `clusters.compression.min_part_size` | String | No | null | Min part size: Minimum size (in bytes) of a data part in a table. ClickHouse only applies the rule to tables with data parts greater than or equal to the Min part size value. | +| `clusters.compression.min_part_size_ratio` | String | No | null | Min part size ratio: Minimum table part size to total table size ratio. ClickHouse only applies the rule to tables in which this ratio is greater than or equal to the Min part size ratio value. | +| `clusters.compression.level` | String | No | null | Compression level for ZSTD method. | +| `clusters.rabbitmq` | List | No | [] | RabbitMQ connection configuration. | +| `clusters.rabbitmq.username` | String | No | null | RabbitMQ username. | +| `clusters.rabbitmq.password` | String | No | null | RabbitMQ user password. | +| `clusters.rabbitmq.vhost` | String | No | null | RabbitMQ vhost. Default: '\'. | +| `clusters.graphite_rollup` | List | No | [] | Graphite rollup configuration. | +| `clusters.graphite_rollup.name` | String | Yes | - | Graphite rollup configuration name. | +| `clusters.graphite_rollup.pattern_function` | String | Yes| - | Aggregation function name. | +| `clusters.graphite_rollup.pattern_regexp` | String | No | null | Regular expression that the metric name must match. | +| `clusters.graphite_rollup.pattern_retention_age` | String | Yes | - | Minimum data age in seconds. | +| `clusters.graphite_rollup.pattern_retention_precision` | String | Yes | - | Accuracy of determining the age of the data in seconds. | +| `clusters.shard` | List | No | [] | Shard configurations. | +| `clusters.shard.name` | String | Yes| - | The name of shard. | +| `clusters.shard.weight` | Int | No | - | The weight of shard. | +| `clusters.shard.resources` | List | No | - | Resources allocated to host of the shard. The resources specified for the shard takes precedence over the resources specified for the cluster. The structure is documented below. | +| `clusters.shard.resources.resources_preset_id` | String | No | null | The ID of the preset for computational resources available to a host (CPU, memory etc.). | +| `clusters.shard.resources.disk_size` | String | No | null | Volume of the storage available to a host, in gigabytes. | +| `clusters.shard.resources.disk_type_id` | Int | No | null | Type of the storage of hosts. | +| `clusters.shard_group` | List | No | [] | A group of clickhouse shards. | +| `clusters.shard_group.name` | String | Yes | - | The name of the shard group, used as cluster name in Distributed tables. | +| `clusters.shard_group.description` | String | No | "Created by Terraform" | Description of the shard group. | +| `clusters.shard_group.shard_names` | String | Yes | - | List of shards names that belong to the shard group. | +| `clusters.admin_password` | String | No | - | A password used to authorize as user admin when sql_user_management enabled. | +| `clusters.sql_user_management` | String | No | - | Enables admin user with user management permission. | +| `clusters.sql_database_management` | String | No | - | Grants admin user database management permission. | +| `clusters.embedded_keeper` | String | No | - | Whether to use ClickHouse Keeper as a coordination system and place it on the same hosts with ClickHouse. If not, it's used ZooKeeper with placement on separate hosts. | +| `clusters.security_group_ids` | List | No | [] | A set of ids of security groups assigned to hosts of the cluster. | +| `clusters.zookeeper` | List | No | [] | Configuration of the ZooKeeper subcluster. | +| `clusters.zookeeper.resources.resources_preset_id` | String | No | null | The ID of the preset for computational resources available to a host (CPU, memory etc.). | +| `clusters.zookeeper.resources.disk_size` | String | No | null | Volume of the storage available to a host, in gigabytes. | +| `clusters.zookeeper.resources.disk_type_id` | Int | No | null | Type of the storage of hosts. | +| `clusters.database` | List | No | [] | A database of the ClickHouse cluster. | +| `clusters.database.name` | String | Yes | - | A database name of the ClickHouse cluster. | +| `clusters.user` | List | No | [] | A user of the ClickHouse cluster. | +| `clusters.user.name` | String | Yes | - | The name of the user. | +| `clusters.user.permission` | List | No | [] | Set of permissions granted to the user. The structure is documented below. | +| `clusters.user.permission.database_name` | String | Yes | - | The name of the database that the permission grants access to. | +| `clusters.user.settings` | List | No | [] | Custom settings for user. The list is documented below. | +| `clusters.user.settings.readonly` | String | No | null | Restricts permissions for reading data, write data and change settings queries. | +| `clusters.user.settings.allow_ddl` | Int | No | null | Allows or denies DDL queries. | +| `clusters.user.settings.insert_quorum` | Int | No | null | Enables the quorum writes. | +| `clusters.user.settings.connect_timeout` | Int | No | null | Connect timeout in milliseconds on the socket used for communicating with the client. | +| `clusters.user.settings.receive_timeout` | Int | No | null | Receive timeout in milliseconds on the socket used for communicating with the client. | +| `clusters.user.settings.send_timeout` | Int | No | null | Send timeout in milliseconds on the socket used for communicating with the client. | +| `clusters.user.settings.insert_quorum_timeout` | Int | No | null | Write to a quorum timeout in milliseconds. | +| `clusters.user.settings.select_sequential_consistency` | Int | No | null | Enables or disables sequential consistency for SELECT queries. | +| `clusters.user.settings.max_replica_delay_for_distributed_queries` | Int | No | null | Disables lagging replicas for distributed queries. | +| `clusters.user.settings.fallback_to_stale_replicas_for_distributed_queries` | Int | No | null | Forces a query to an out-of-date replica if updated data is not available. | +| `clusters.user.settings.replication_alter_partitions_sync` | Int | No | null | For ALTER … ATTACH|DETACH|DROP queries, you can use the replication_alter_partitions_sync setting to set up waiting. | +| `clusters.user.settings.distributed_product_mode` | Int | No | null | Changes the behaviour of distributed subqueries. | +| `clusters.user.settings.distributed_aggregation_memory_efficient` | Int | No | null | Determine the behavior of distributed subqueries. | +| `clusters.user.settings.distributed_ddl_task_timeout` | Int | No | null | Timeout for DDL queries, in milliseconds. | +| `clusters.user.settings.skip_unavailable_shards` | Int | No | null | Enables or disables silently skipping of unavailable shards. | +| `clusters.user.settings.compile` | Int | No | null | Enable compilation of queries. | +| `clusters.user.settings.min_count_to_compile` | Int | No | null | How many times to potentially use a compiled chunk of code before running compilation. | +| `clusters.user.settings.compile_expressions` | Int | No | null | Turn on expression compilation. | +| `clusters.user.settings.min_count_to_compile_expression` | Int | No | null | A query waits for expression compilation process to complete prior to continuing execution. | +| `clusters.user.settings.max_block_size` | Int | No | null | A recommendation for what size of the block (in a count of rows) to load from tables. | +| `clusters.user.settings.min_insert_block_size_rows` | Int | No | null | Sets the minimum number of rows in the block which can be inserted into a table by an INSERT query. | +| `clusters.user.settings.min_insert_block_size_bytes` | Int | No | null | Sets the minimum number of bytes in the block which can be inserted into a table by an INSERT query. | +| `clusters.user.settings.max_insert_block_size` | Int | No | null | The size of blocks (in a count of rows) to form for insertion into a table. | +| `clusters.user.settings.min_bytes_to_use_direct_io` | Int | No | null | The minimum data volume required for using direct I/O access to the storage disk. | +| `clusters.user.settings.use_uncompressed_cache` | Int | No | null | Whether to use a cache of uncompressed blocks. | +| `clusters.user.settings.merge_tree_max_rows_to_use_cache` | Int | No | null | If ClickHouse should read more than merge_tree_max_rows_to_use_cache rows in one query, it doesn’t use the cache of uncompressed blocks. | +| `clusters.user.settings.merge_tree_max_bytes_to_use_cache` | Int | No | null | If ClickHouse should read more than merge_tree_max_bytes_to_use_cache bytes in one query, it doesn’t use the cache of uncompressed blocks. | +| `clusters.user.settings.merge_tree_min_rows_for_concurrent_read` | Int | No | null | If the number of rows to be read from a file of a MergeTree table exceeds merge_tree_min_rows_for_concurrent_read then ClickHouse tries to perform a concurrent reading from this file on several threads. | +| `clusters.user.settings.merge_tree_min_bytes_for_concurrent_read` | Int | No | null | If the number of bytes to read from one file of a MergeTree-engine table exceeds merge_tree_min_bytes_for_concurrent_read, then ClickHouse tries to concurrently read from this file in several threads. | +| `clusters.user.settings.max_bytes_before_external_group_by` | Int | No | null | Limit in bytes for using memoru for GROUP BY before using swap on disk. | +| `clusters.user.settings.max_bytes_before_external_sort` | Int | No | null | This setting is equivalent of the max_bytes_before_external_group_by setting, except for it is for sort operation (ORDER BY), not aggregation. | +| `clusters.user.settings.group_by_two_level_threshold` | Int | No | null | Sets the threshold of the number of keys, after that the two-level aggregation should be used. | +| `clusters.user.settings.group_by_two_level_threshold_bytes` | Int | No | null | Sets the threshold of the number of bytes, after that the two-level aggregation should be used. | +| `clusters.user.settings.priority` | Int | No | null | Query priority. | +| `clusters.user.settings.max_threads` | Int | No | null | The maximum number of query processing threads, excluding threads for retrieving data from remote servers. | +| `clusters.user.settings.max_memory_usage` | Int | No | null | Limits the maximum memory usage (in bytes) for processing queries on a single server. | +| `clusters.user.settings.max_memory_usage_for_user` | Int | No | null | Limits the maximum memory usage (in bytes) for processing of user's queries on a single server. | +| `clusters.user.settings.max_network_bandwidth` | Int | No | null | Limits the speed of the data exchange over the network in bytes per second. | +| `clusters.user.settings.max_network_bandwidth_for_user` | Int | No | null | Limits the speed of the data exchange over the network in bytes per second. | +| `clusters.user.settings.force_index_by_date` | Int | No | null | Disables query execution if the index can’t be used by date. | +| `clusters.user.settings.force_primary_key` | Int | No | null | Disables query execution if indexing by the primary key is not possible. | +| `clusters.user.settings.max_rows_to_read` | Int | No | null | Limits the maximum number of rows that can be read from a table when running a query. | +| `clusters.user.settings.max_bytes_to_read` | Int | No | null | Limits the maximum number of bytes (uncompressed data) that can be read from a table when running a query. | +| `clusters.user.settings.read_overflow_mode` | Int | No | null | Sets behaviour on overflow while read. Possible values: | +| `clusters.user.settings.max_rows_to_group_by` | Int | No | null | Limits the maximum number of unique keys received from aggregation function. | +| `clusters.user.settings.group_by_overflow_mode` | Int | No | null | Sets behaviour on overflow while GROUP BY operation. Possible values: | +| `clusters.user.settings.max_rows_to_sort` | Int | No | null | Limits the maximum number of rows that can be read from a table for sorting. | +| `clusters.user.settings.max_bytes_to_sort` | Int | No | null | Limits the maximum number of bytes (uncompressed data) that can be read from a table for sorting. | +| `clusters.user.settings.sort_overflow_mode` | Int | No | null | Sets behaviour on overflow while sort. Possible values: | +| `clusters.user.settings.max_result_rows` | Int | No | null | Limits the number of rows in the result. | +| `clusters.user.settings.max_result_bytes` | Int | No | null | Limits the number of bytes in the result. | +| `clusters.user.settings.result_overflow_mode` | Int | No | null | Sets behaviour on overflow in result. Possible values: | +| `clusters.user.settings.max_rows_in_distinct` | Int | No | null | Limits the maximum number of different rows when using DISTINCT. | +| `clusters.user.settings.max_bytes_in_distinct` | Int | No | null | Limits the maximum size of a hash table in bytes (uncompressed data) when using DISTINCT. | +| `clusters.user.settings.distinct_overflow_mode` | Int | No | null | Sets behaviour on overflow when using DISTINCT. Possible values: | +| `clusters.user.settings.max_rows_to_transfer` | Int | No | null | Limits the maximum number of rows that can be passed to a remote server or saved in a temporary table when using GLOBAL IN. | +| `clusters.user.settings.max_bytes_to_transfer` | Int | No | null | Limits the maximum number of bytes (uncompressed data) that can be passed to a remote server or saved in a temporary table when using GLOBAL IN. | +| `clusters.user.settings.transfer_overflow_mode` | Int | No | null | Sets behaviour on overflow. Possible values: | +| `clusters.user.settings.max_execution_time` | Int | No | null | Limits the maximum query execution time in milliseconds. | +| `clusters.user.settings.timeout_overflow_mode` | Int | No | null | Sets behaviour on overflow. Possible values: | +| `clusters.user.settings.max_rows_in_set` | Int | No | null | Limit on the number of rows in the set resulting from the execution of the IN section. | +| `clusters.user.settings.max_bytes_in_set` | Int | No | null | Limit on the number of bytes in the set resulting from the execution of the IN section. | +| `clusters.user.settings.set_overflow_mode` | Int | No | null | Sets behaviour on overflow in the set resulting. Possible values: | +| `clusters.user.settings.max_rows_in_join` | Int | No | null | Limit on maximum size of the hash table for JOIN, in rows. | +| `clusters.user.settings.max_bytes_in_join` | Int | No | null | Limit on maximum size of the hash table for JOIN, in bytes. | +| `clusters.user.settings.join_overflow_mode` | Int | No | null | Sets behaviour on overflow in JOIN. Possible values: | +| `clusters.user.settings.max_columns_to_read` | Int | No | null | Limits the maximum number of columns that can be read from a table in a single query. | +| `clusters.user.settings.max_temporary_columns` | Int | No | null | Limits the maximum number of temporary columns that must be kept in RAM at the same time when running a query, including constant columns. | +| `clusters.user.settings.max_temporary_non_const_columns` | Int | No | null | Limits the maximum number of temporary columns that must be kept in RAM at the same time when running a query, excluding constant columns. | +| `clusters.user.settings.max_query_size` | Int | No | null | The maximum part of a query that can be taken to RAM for parsing with the SQL parser. | +| `clusters.user.settings.max_ast_depth` | Int | No | null | Maximum abstract syntax tree depth. | +| `clusters.user.settings.max_ast_elements` | Int | No | null | Maximum abstract syntax tree elements. | +| `clusters.user.settings.max_expanded_ast_elements` | Int | No | null | Maximum abstract syntax tree depth after after expansion of aliases. | +| `clusters.user.settings.min_execution_speed` | Int | No | null | Minimal execution speed in rows per second. | +| `clusters.user.settings.min_execution_speed_bytes` | Int | No | null | Minimal execution speed in bytes per second. | +| `clusters.user.settings.count_distinct_implementation` | Int | No | null | Specifies which of the uniq* functions should be used to perform the COUNT(DISTINCT …) construction. | +| `clusters.user.settings.input_format_values_interpret_expressions` | Int | No | null | Enables or disables the full SQL parser if the fast stream parser can’t parse the data. | +| `clusters.user.settings.input_format_defaults_for_omitted_fields` | Int | No | null | When performing INSERT queries, replace omitted input column values with default values of the respective columns. | +| `clusters.user.settings.output_format_json_quote_64bit_integers` | Int | No | null | If the value is true, integers appear in quotes when using JSON* Int64 and UInt64 formats (for compatibility with most JavaScript implementations); otherwise, integers are output without the quotes. | +| `clusters.user.settings.output_format_json_quote_denormals` | Int | No | null | Enables +nan, -nan, +inf, -inf outputs in JSON output format. | +| `clusters.user.settings.low_cardinality_allow_in_native_format` | Int | No | null | Allows or restricts using the LowCardinality data type with the Native format. | +| `clusters.user.settings.empty_result_for_aggregation_by_empty_set` | Int | No | null | Allows to retunr empty result. | +| `clusters.user.settings.joined_subquery_requires_alias` | Int | No | null | Require aliases for subselects and table functions in FROM that more than one table is present. | +| `clusters.user.settings.join_use_nulls` | Int | No | null | Sets the type of JOIN behaviour. When merging tables, empty cells may appear. ClickHouse fills them differently based on this setting. | +| `clusters.user.settings.transform_null_in` | Int | No | null | Enables equality of NULL values for IN operator. | +| `clusters.user.settings.http_connection_timeout` | Int | No | null | Timeout for HTTP connection in milliseconds. | +| `clusters.user.settings.http_receive_timeout` | Int | No | null | Timeout for HTTP connection in milliseconds. | +| `clusters.user.settings.http_send_timeout` | Int | No | null | Timeout for HTTP connection in milliseconds. | +| `clusters.user.settings.enable_http_compression` | Int | No | null | Enables or disables data compression in the response to an HTTP request. | +| `clusters.user.settings.send_progress_in_http_headers` | Int | No | null | Enables or disables X-ClickHouse-Progress HTTP response headers in clickhouse-server responses. | +| `clusters.user.settings.http_headers_progress_interval` | Int | No | null | Sets minimal interval between notifications about request process in HTTP header X-ClickHouse-Progress. | +| `clusters.user.settings.add_http_cors_header` | Int | No | null | Include CORS headers in HTTP responces. | +| `clusters.user.settings.quota_mode` | Int | No | null | Quota accounting mode. | +| `clusters.user.settings.max_concurrent_queries_for_user` | Int | No | null | The maximum number of concurrent requests per user. Default value: 0 (no limit). | +| `clusters.user.settings.memory_profiler_step` | Int | No | null | Memory profiler step (in bytes). If the next query step requires more memory than this parameter specifies, the memory profiler collects the allocating stack trace. Values lower than a few megabytes slow down query processing. Default value: 4194304 (4 MB). Zero means disabled memory profiler. | +| `clusters.user.settings.memory_profiler_sample_probability` | Int | No | null | Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation. Possible values: from 0 to 1. Default: 0. | +| `clusters.user.settings.insert_null_as_default` | Int | No | null | Enables the insertion of default values instead of NULL into columns with not nullable data type. Default value: true. | +| `clusters.user.settings.allow_suspicious_low_cardinality_types` | Int | No | null | Allows specifying LowCardinality modifier for types of small fixed size (8 or less) in CREATE TABLE statements. Enabling this may increase merge times and memory consumption. | +| `clusters.user.settings.connect_timeout_with_failover` | Int | No | null | The timeout in milliseconds for connecting to a remote server for a Distributed table engine, if the ‘shard’ and ‘replica’ sections are used in the cluster definition. If unsuccessful, several attempts are made to connect to various replicas. Default value: 50. | +| `clusters.user.settings.allow_introspection_functions` | Int | No | null | Enables introspections functions for query profiling. | +| `clusters.user.settings.async_insert` | Int | No | null | Enables asynchronous inserts. Disabled by default. | +| `clusters.user.settings.async_insert_threads` | Int | No | null | The maximum number of threads for background data parsing and insertion. If the parameter is set to 0, asynchronous insertions are disabled. Default value: 16. | +| `clusters.user.settings.wait_for_async_insert` | Int | No | null | Enables waiting for processing of asynchronous insertion. If enabled, server returns OK only after the data is inserted. | +| `clusters.user.settings.wait_for_async_insert_timeout` | Int | No | null | The timeout (in seconds) for waiting for processing of asynchronous insertion. Value must be at least 1000 (1 second). | +| `clusters.user.settings.async_insert_max_data_size` | Int | No | null | The maximum size of the unparsed data in bytes collected per query before being inserted. If the parameter is set to 0, asynchronous insertions are disabled. Default value: 100000. | +| `clusters.user.settings.async_insert_busy_timeout` | Int | No | null | The maximum timeout in milliseconds since the first INSERT query before inserting collected data. If the parameter is set to 0, the timeout is disabled. Default value: 200. | +| `clusters.user.settings.async_insert_stale_timeout` | Int | No | null | The maximum timeout in milliseconds since the last INSERT query before dumping collected data. If enabled, the settings prolongs the async_insert_busy_timeout with every INSERT query as long as async_insert_max_data_size is not exceeded. | +| `clusters.user.settings.timeout_before_checking_execution_speed` | Int | No | null | Timeout (in seconds) between checks of execution speed. It is checked that execution speed is not less that specified in min_execution_speed parameter. Must be at least 1000. | +| `clusters.user.settings.cancel_http_readonly_queries_on_client_close` | Int | No | null | Cancels HTTP read-only queries (e.g. SELECT) when a client closes the connection without waiting for the response. Default value: false. | +| `clusters.user.settings.flatten_nested` | Int | No | null | Sets the data format of a nested columns. | +| `clusters.user.settings.max_http_get_redirects` | Int | No | null | Limits the maximum number of HTTP GET redirect hops for URL-engine tables. | +| `clusters.user.settings.input_format_import_nested_json` | Int | No | null | Enables or disables the insertion of JSON data with nested objects. | +| `clusters.user.settings.input_format_parallel_parsing` | Int | No | null | Enables or disables order-preserving parallel parsing of data formats. Supported only for TSV, TKSV, CSV and JSONEachRow formats. | +| `clusters.user.settings.max_read_buffer_size` | Int | No | null | The maximum size of the buffer to read from the filesystem. | +| `clusters.user.settings.max_final_threads` | Int | No | null | Sets the maximum number of parallel threads for the SELECT query data read phase with the FINAL modifier. | +| `clusters.user.settings.local_filesystem_read_method` | Int | No | null | Method of reading data from local filesystem. Possible values: | +| `clusters.user.settings.remote_filesystem_read_method` | Int | No | null | Method of reading data from remote filesystem, one of: read, threadpool. | +| `clusters.user.settings.max_read_buffer_size` | Int | No | null | The maximum size of the buffer to read from the filesystem. | +| `clusters.user.settings.insert_keeper_max_retries` | Int | No | null | The setting sets the maximum number of retries for ClickHouse Keeper (or ZooKeeper) requests during insert into replicated MergeTree. Only Keeper requests which failed due to network error, Keeper session timeout, or request timeout are considered for retries. | +| `clusters.user.settings.max_temporary_data_on_disk_size_for_user` | Int | No | null | The maximum amount of data consumed by temporary files on disk in bytes for all concurrently running user queries. Zero means unlimited. | +| `clusters.user.settings.max_temporary_data_on_disk_size_for_query` | Int | No | null | The maximum amount of data consumed by temporary files on disk in bytes for all concurrently running queries. Zero means unlimited. | +| `clusters.user.settings.max_parser_depth` | Int | No | null | Limits maximum recursion depth in the recursive descent parser. Allows controlling the stack size. Zero means unlimited. | +| `clusters.user.settings.memory_overcommit_ratio_denominator` | Int | No | null | It represents soft memory limit in case when hard limit is reached on user level. This value is used to compute overcommit ratio for the query. Zero means skip the query. | +| `clusters.user.settings.memory_overcommit_ratio_denominator_for_user` | Int | No | null | It represents soft memory limit in case when hard limit is reached on global level. This value is used to compute overcommit ratio for the query. Zero means skip the query. | +| `clusters.user.settings.memory_usage_overcommit_max_wait_microseconds` | Int | No | null | Maximum time thread will wait for memory to be freed in the case of memory overcommit on a user level. If the timeout is reached and memory is not freed, an exception is thrown. | +| `clusters.user.quota` | List | No | [] | Set of user quotas. The structure is documented below. | +| `clusters.user.quota.interval_duration` | Int | Yes | null | Duration of interval for quota in milliseconds. | +| `clusters.user.quota.queries` | Int | No | null | The total number of queries. | +| `clusters.user.quota.errors` | Int | No | null | The number of queries that threw exception. | +| `clusters.user.quota.result_rows` | Int | No | null | The total number of rows given as the result. | +| `clusters.user.quota.read_rows` | Int | No | null | The total number of source rows read from tables for running the query, on all remote servers. | +| `clusters.user.quota.execution_time` | Int | No | null | The total query execution time, in milliseconds (wall time). | +| `clusters.user.clickhouse_hosts` | List | No | [] | Set of hosts quotas. The structure is documented below. | +| `clusters.user.clickhouse_hosts.fqdn` | String | No | null | The fully qualified domain name of the host. | +| `clusters.user.clickhouse_hosts.type` | String | Yes | null | The type of the host to be deployed. Can be either CLICKHOUSE or ZOOKEEPER. | +| `clusters.user.clickhouse_hosts.zone` | String | Yes | null | The availability zone where the ClickHouse host will be created. For more information see the official documentation. | +| `clusters.user.clickhouse_hosts.subnet_id` | String | No | null | The ID of the subnet, to which the host belongs. The subnet must be a part of the network to which the cluster belongs. | +| `clusters.user.clickhouse_hosts.shard_name` | String | No | null | The name of the shard to which the host belongs. | +| `clusters.user.clickhouse_hosts.assign_public_ip` | Bool | No | null | Sets whether the host should get a public IP address on creation. Can be either true or false. | +| `clusters.user.clickhouse_hosts.web_sql` | Bool | No | null | Allow access for Web SQL. Can be either true or false. | +| `clusters.user.clickhouse_hosts.data_lens` | Bool | No | null | Allow access for DataLens. Can be either true or false. | +| `clusters.user.clickhouse_hosts.metrika` | Bool | No | null | Allow access for Yandex.Metrika. Can be either true or false. | +| `clusters.user.clickhouse_hosts.serverless` | Bool | No | null | Allow access for Serverless. Can be either true or false. | +| `clusters.user.clickhouse_hosts.yandex_query` | Bool | No | null | Allow access for YandexQuery. Can be either true or false. | +| `clusters.user.clickhouse_hosts.data_transfer` | Bool | No | null | Allow access for DataTransfer. Can be either true or false. | +| `clusters.service_account_id` | String | No | null | ID of the service account used for access to Yandex Object Storage. | +| `clusters.cloud_storage_enable` | Bool | No | false | Whether to use Yandex Object Storage for storing ClickHouse data. Can be either true or false. | +| `clusters.maintenance_window_type` | String | No | "ANYTIME" | Type of maintenance window. Can be either ANYTIME or WEEKLY. A day and hour of window need to be specified with weekly window. | +| `clusters.maintenance_window_hour` | Int | No | null | Hour of day in UTC time zone (1-24) for maintenance window if window type is weekly. | +| `clusters.maintenance_window_day` | Int | No | null | Day of week for maintenance window if window type is weekly. Possible values: MON, TUE, WED, THU, FRI, SAT, SUN. | + +## Example + +Usage example located in this [directory](docs/example). diff --git a/YandexCloud/Managed-clickhouse/managed-clickhouse/cluster.tf b/YandexCloud/Managed-clickhouse/managed-clickhouse/cluster.tf new file mode 100644 index 0000000..e54be01 --- /dev/null +++ b/YandexCloud/Managed-clickhouse/managed-clickhouse/cluster.tf @@ -0,0 +1,378 @@ +## Create Clickhouse cluster + +resource "yandex_mdb_clickhouse_cluster" "cluster" { + for_each = var.clusters + + name = each.key + environment = try(each.value.environment, "PRODUCTION") + description = try(each.value.description, "Created by Terraform") + network_id = each.value.network_id + version = try(each.value.clickhouse_version, null) + deletion_protection = try(each.value.deletion_protection, false) + + clickhouse { + resources { + resource_preset_id = each.value.resource_preset_id + disk_type_id = try(each.value.disk_type_id, "network-hdd") + disk_size = try(each.value.disk_size, 10) + } + + config { + log_level = try(each.value.config_log_level , "TRACE") + max_connections = try(each.value.config_max_connections , 100) + max_concurrent_queries = try(each.value.config_max_concurrent_queries , 50) + keep_alive_timeout = try(each.value.config_keep_alive_timeout , 3000) + uncompressed_cache_size = try(each.value.config_uncompressed_cache_size , 8589934592) + mark_cache_size = try(each.value.config_mark_cache_size , 5368709120) + max_table_size_to_drop = try(each.value.config_max_table_size_to_drop , 53687091200) + max_partition_size_to_drop = try(each.value.config_max_partition_size_to_drop , 53687091200) + timezone = try(each.value.config_timezone , "UTC") + geobase_uri = try(each.value.config_geobase_uri , "") + query_log_retention_size = try(each.value.config_query_log_retention_size , 1073741824) + query_log_retention_time = try(each.value.config_query_log_retention_time , 259200000) + query_thread_log_enabled = try(each.value.config_query_thread_log_enabled , true) + query_thread_log_retention_size = try(each.value.config_query_thread_log_retention_size , 536870912) + query_thread_log_retention_time = try(each.value.config_query_thread_log_retention_time , 259200000) + part_log_retention_size = try(each.value.config_part_log_retention_size , 536870912) + part_log_retention_time = try(each.value.config_part_log_retention_time , 259200000) + metric_log_enabled = try(each.value.config_metric_log_enabled , true) + metric_log_retention_size = try(each.value.config_metric_log_retention_size , 536870912) + metric_log_retention_time = try(each.value.config_metric_log_retention_time , 259200000) + trace_log_enabled = try(each.value.config_trace_log_enabled , true) + trace_log_retention_size = try(each.value.config_trace_log_retention_size , 536870912) + trace_log_retention_time = try(each.value.config_trace_log_retention_time , 259200000) + text_log_enabled = try(each.value.config_text_log_enabled , true) + text_log_retention_size = try(each.value.config_text_log_retention_size , 536870912) + text_log_retention_time = try(each.value.config_text_log_retention_time , 259200000) + text_log_level = try(each.value.config_text_log_level , "TRACE") + background_pool_size = try(each.value.config_background_pool_size , 16) + background_schedule_pool_size = try(each.value.config_background_schedule_pool_size , 16) + + dynamic "merge_tree" { + for_each = try(each.value.merge_tree, []) + content { + replicated_deduplication_window = try(merge_tree.value.replicated_deduplication_window, null) + replicated_deduplication_window_seconds = try(merge_tree.value.replicated_deduplication_window_seconds, null) + parts_to_delay_insert = try(merge_tree.value.parts_to_delay_insert, null) + parts_to_throw_insert = try(merge_tree.value.parts_to_throw_insert, null) + max_replicated_merges_in_queue = try(merge_tree.value.max_replicated_merges_in_queue, null) + number_of_free_entries_in_pool_to_lower_max_size_of_merge = try(merge_tree.value.number_of_free_entries_in_pool_to_lower_max_size_of_merge, null) + max_bytes_to_merge_at_min_space_in_pool = try(merge_tree.value.max_bytes_to_merge_at_min_space_in_pool, null) + min_bytes_for_wide_part = try(merge_tree.value.min_bytes_for_wide_part, null) + min_rows_for_wide_part = try(merge_tree.value.min_rows_for_wide_part, null) + ttl_only_drop_parts = try(merge_tree.value.ttl_only_drop_parts, null) + merge_with_ttl_timeout = try(merge_tree.value.merge_with_ttl_timeout, null) + merge_with_recompression_ttl_timeout = try(merge_tree.value.merge_with_recompression_ttl_timeout, null) + max_parts_in_total = try(merge_tree.value.max_parts_in_total, null) + max_number_of_merges_with_ttl_in_pool = try(merge_tree.value.max_number_of_merges_with_ttl_in_pool, null) + cleanup_delay_period = try(merge_tree.value.cleanup_delay_period, null) + max_avg_part_size_for_too_many_parts = try(merge_tree.value.max_avg_part_size_for_too_many_parts, null) + min_age_to_force_merge_seconds = try(merge_tree.value.min_age_to_force_merge_seconds, null) + min_age_to_force_merge_on_partition_only = try(merge_tree.value.min_age_to_force_merge_on_partition_only, null) + merge_selecting_sleep_ms = try(merge_tree.value.merge_selecting_sleep_ms, null) + } + } + + dynamic "kafka" { + for_each = try(each.value.kafka, []) + content { + security_protocol = try(kafka.value.security_protocol, null) + sasl_mechanism = try(kafka.value.sasl_mechanism, null) + sasl_username = try(kafka.value.sasl_username, null) + sasl_password = try(kafka.value.sasl_password, null) + enable_ssl_certificate_verification = try(kafka.value.enable_ssl_certificate_verification, null) + max_poll_interval_ms = try(kafka.value.max_poll_interval_ms, null) + session_timeout_ms = try(kafka.value.session_timeout_ms, null) + } + } + + dynamic "kafka_topic" { + for_each = try(each.value.kafka_topic, []) + content { + name = kafka_topic.value.name + settings { + security_protocol = try(kafka_topic.value.security_protocol, null) + sasl_mechanism = try(kafka_topic.value.sasl_mechanism, null) + sasl_username = try(kafka_topic.value.sasl_username, null) + sasl_password = try(kafka_topic.value.sasl_password, null) + enable_ssl_certificate_verification = try(kafka_topic.value.enable_ssl_certificate_verification, null) + max_poll_interval_ms = try(kafka_topic.value.max_poll_interval_ms, null) + session_timeout_ms = try(kafka_topic.value.session_timeout_ms, null) + } + } + } + + dynamic "rabbitmq" { + for_each = try(each.value.rabbitmq, []) + content { + username = try(rabbitmq.value.username, null) + password = try(rabbitmq.value.password, null) + vhost = try(rabbitmq.value.vhost, null) + } + } + + dynamic "compression" { + for_each = try(each.value.compression, []) + content { + method = try(compression.value.method, null) + min_part_size = try(compression.value.min_part_size, null) + min_part_size_ratio = try(compression.value.min_part_size_ratio, null) + level = try(compression.value.level, null) + } + } + + dynamic "graphite_rollup" { + for_each = try(each.value.graphite_rollup, []) + content { + name = graphite_rollup.value.name + pattern { + regexp = graphite_rollup.value.pattern_regexp + function = graphite_rollup.value.pattern_function + retention { + age = graphite_rollup.value.pattern_retention_age + precision = graphite_rollup.value.pattern_retention_precision + } + } + } + } + + } + } + + dynamic "shard" { + for_each = try(each.value.shard, []) + content { + name = shard.value.name + weight = try(shard.value.weight, null) + dynamic "resources" { + for_each = try(shard.value.resources, []) + content { + resource_preset_id = try(resources.value.resource_preset_id, null) + disk_type_id = try(resources.value.disk_type, null) + disk_size = try(resources.value.disk_size, null) + } + } + } + } + + dynamic "shard_group" { + for_each = try(each.value.shard_group, []) + content { + name = shard_group.value.name + description = try(shard_group.value.description, "Created by Terraform") + shard_names = shard_group.value.shard_names + } + } + + admin_password = try(each.value.admin_password, random_password.admin_pass.result, null) + sql_user_management = try(each.value.sql_user_management, null) + sql_database_management = try(each.value.sql_database_management, null) + security_group_ids = try(each.value.security_group_ids, []) + embedded_keeper = try(each.value.clickhouse_keeper_enable, null) + + dynamic "zookeeper" { + for_each = try(each.value.zookeeper, []) + content { + resources { + resource_preset_id = try(zookeeper.value.resource_preset_id, null) + disk_type_id = try(zookeeper.value.disk_type_id, null) + disk_size = try(zookeeper.value.disk_size, null) + } + } + } + + dynamic "database" { + for_each = try(each.value.database, []) + content { + name = database.value.name + } + } + + dynamic "user" { + for_each = try(each.value.user, []) + content { + name = user.value.name + password = random_password.user_pass[user.value.name].result + + dynamic "permission" { + for_each = try(user.value.permission, []) + content { + database_name = permission.value.database_name + } + } + dynamic "settings" { + for_each = try(user.value.settings, []) + content { + readonly = try(settings.value.readonly, null) + allow_ddl = try(settings.value.allow_ddl, null) + insert_quorum = try(settings.value.insert_quorum, null) + connect_timeout = try(settings.value.connect_timeout, null) + receive_timeout = try(settings.value.receive_timeout, null) + send_timeout = try(settings.value.send_timeout, null) + insert_quorum_timeout = try(settings.value.insert_quorum_timeout, null) + select_sequential_consistency = try(settings.value.select_sequential_consistency, null) + max_replica_delay_for_distributed_queries = try(settings.value.max_replica_delay_for_distributed_queries, null) + fallback_to_stale_replicas_for_distributed_queries = try(settings.value.fallback_to_stale_replicas_for_distributed_queries, null) + replication_alter_partitions_sync = try(settings.value.replication_alter_partitions_sync, null) + distributed_product_mode = try(settings.value.distributed_product_mode, null) + distributed_aggregation_memory_efficient = try(settings.value.distributed_aggregation_memory_efficient, null) + distributed_ddl_task_timeout = try(settings.value.distributed_ddl_task_timeout, null) + skip_unavailable_shards = try(settings.value.skip_unavailable_shards, null) + compile = try(settings.value.compile, null) + min_count_to_compile = try(settings.value.min_count_to_compile, null) + compile_expressions = try(settings.value.compile_expressions, null) + min_count_to_compile_expression = try(settings.value.min_count_to_compile_expression, null) + max_block_size = try(settings.value.max_block_size, null) + min_insert_block_size_rows = try(settings.value.min_insert_block_size_rows, null) + min_insert_block_size_bytes = try(settings.value.min_insert_block_size_bytes, null) + max_insert_block_size = try(settings.value.max_insert_block_size, null) + min_bytes_to_use_direct_io = try(settings.value.min_bytes_to_use_direct_io, null) + use_uncompressed_cache = try(settings.value.use_uncompressed_cache, null) + merge_tree_max_rows_to_use_cache = try(settings.value.merge_tree_max_rows_to_use_cache, null) + merge_tree_max_bytes_to_use_cache = try(settings.value.merge_tree_max_bytes_to_use_cache, null) + merge_tree_min_rows_for_concurrent_read = try(settings.value.merge_tree_min_rows_for_concurrent_read, null) + merge_tree_min_bytes_for_concurrent_read = try(settings.value.merge_tree_min_bytes_for_concurrent_read, null) + max_bytes_before_external_group_by = try(settings.value.max_bytes_before_external_group_by, null) + max_bytes_before_external_sort = try(settings.value.max_bytes_before_external_sort, null) + group_by_two_level_threshold = try(settings.value.group_by_two_level_threshold, null) + group_by_two_level_threshold_bytes = try(settings.value.group_by_two_level_threshold_bytes, null) + priority = try(settings.value.priority, null) + max_threads = try(settings.value.max_threads, null) + max_memory_usage = try(settings.value.max_memory_usage, null) + max_memory_usage_for_user = try(settings.value.max_memory_usage_for_user, null) + max_network_bandwidth = try(settings.value.max_network_bandwidth, null) + max_network_bandwidth_for_user = try(settings.value.max_network_bandwidth_for_user, null) + force_index_by_date = try(settings.value.force_index_by_date, null) + force_primary_key = try(settings.value.force_primary_key, null) + max_rows_to_read = try(settings.value.max_rows_to_read, null) + max_bytes_to_read = try(settings.value.max_bytes_to_read, null) + read_overflow_mode = try(settings.value.read_overflow_mode, null) + max_rows_to_group_by = try(settings.value.max_rows_to_group_by, null) + group_by_overflow_mode = try(settings.value.group_by_overflow_mode, null) + max_rows_to_sort = try(settings.value.max_rows_to_sort, null) + max_bytes_to_sort = try(settings.value.max_bytes_to_sort, null) + sort_overflow_mode = try(settings.value.sort_overflow_mode, null) + max_result_rows = try(settings.value.max_result_rows, null) + max_result_bytes = try(settings.value.max_result_bytes, null) + result_overflow_mode = try(settings.value.result_overflow_mode, null) + max_rows_in_distinct = try(settings.value.max_rows_in_distinct, null) + max_bytes_in_distinct = try(settings.value.max_bytes_in_distinct, null) + distinct_overflow_mode = try(settings.value.distinct_overflow_mode, null) + max_rows_to_transfer = try(settings.value.max_rows_to_transfer, null) + max_bytes_to_transfer = try(settings.value.max_bytes_to_transfer, null) + transfer_overflow_mode = try(settings.value.transfer_overflow_mode, null) + max_execution_time = try(settings.value.max_execution_time, null) + timeout_overflow_mode = try(settings.value.timeout_overflow_mode, null) + max_rows_in_set = try(settings.value.max_rows_in_set, null) + max_bytes_in_set = try(settings.value.max_bytes_in_set, null) + set_overflow_mode = try(settings.value.set_overflow_mode, null) + max_rows_in_join = try(settings.value.max_rows_in_join, null) + max_bytes_in_join = try(settings.value.max_bytes_in_join, null) + join_overflow_mode = try(settings.value.join_overflow_mode, null) + max_columns_to_read = try(settings.value.max_columns_to_read, null) + max_temporary_columns = try(settings.value.max_temporary_columns, null) + max_temporary_non_const_columns = try(settings.value.max_temporary_non_const_columns, null) + max_query_size = try(settings.value.max_query_size, null) + max_ast_depth = try(settings.value.max_ast_depth, null) + max_ast_elements = try(settings.value.max_ast_elements, null) + max_expanded_ast_elements = try(settings.value.max_expanded_ast_elements, null) + min_execution_speed = try(settings.value.min_execution_speed, null) + min_execution_speed_bytes = try(settings.value.min_execution_speed_bytes, null) + count_distinct_implementation = try(settings.value.count_distinct_implementation, null) + input_format_values_interpret_expressions = try(settings.value.input_format_values_interpret_expressions, null) + input_format_defaults_for_omitted_fields = try(settings.value.input_format_defaults_for_omitted_fields, null) + output_format_json_quote_64bit_integers = try(settings.value.output_format_json_quote_64bit_integers, null) + output_format_json_quote_denormals = try(settings.value.output_format_json_quote_denormals, null) + low_cardinality_allow_in_native_format = try(settings.value.low_cardinality_allow_in_native_format, null) + empty_result_for_aggregation_by_empty_set = try(settings.value.empty_result_for_aggregation_by_empty_set, null) + joined_subquery_requires_alias = try(settings.value.joined_subquery_requires_alias, null) + join_use_nulls = try(settings.value.join_use_nulls, null) + transform_null_in = try(settings.value.transform_null_in, null) + http_connection_timeout = try(settings.value.http_connection_timeout, null) + http_receive_timeout = try(settings.value.http_receive_timeout, null) + http_send_timeout = try(settings.value.http_send_timeout, null) + enable_http_compression = try(settings.value.enable_http_compression, null) + send_progress_in_http_headers = try(settings.value.send_progress_in_http_headers, null) + http_headers_progress_interval = try(settings.value.http_headers_progress_interval, null) + add_http_cors_header = try(settings.value.add_http_cors_header, null) + quota_mode = try(settings.value.quota_mode, null) + max_concurrent_queries_for_user = try(settings.value.max_concurrent_queries_for_user, null) + memory_profiler_step = try(settings.value.memory_profiler_step, null) + memory_profiler_sample_probability = try(settings.value.memory_profiler_sample_probability, null) + insert_null_as_default = try(settings.value.insert_null_as_default, null) + allow_suspicious_low_cardinality_types = try(settings.value.allow_suspicious_low_cardinality_types, null) + connect_timeout_with_failover = try(settings.value.connect_timeout_with_failover, null) + allow_introspection_functions = try(settings.value.allow_introspection_functions, null) + async_insert = try(settings.value.async_insert, null) + async_insert_threads = try(settings.value.async_insert_threads, null) + wait_for_async_insert = try(settings.value.wait_for_async_insert, null) + wait_for_async_insert_timeout = try(settings.value.wait_for_async_insert_timeout, null) + async_insert_max_data_size = try(settings.value.async_insert_max_data_size, null) + async_insert_busy_timeout = try(settings.value.async_insert_busy_timeout, null) + async_insert_stale_timeout = try(settings.value.async_insert_stale_timeout, null) + timeout_before_checking_execution_speed = try(settings.value.timeout_before_checking_execution_speed, null) + cancel_http_readonly_queries_on_client_close = try(settings.value.cancel_http_readonly_queries_on_client_close, null) + flatten_nested = try(settings.value.flatten_nested, null) + max_http_get_redirects = try(settings.value.max_http_get_redirects, null) + input_format_import_nested_json = try(settings.value.input_format_import_nested_json, null) + input_format_parallel_parsing = try(settings.value.input_format_parallel_parsing, null) + max_final_threads = try(settings.value.max_final_threads, null) + local_filesystem_read_method = try(settings.value.local_filesystem_read_method, null) + remote_filesystem_read_method = try(settings.value.remote_filesystem_read_method, null) + max_read_buffer_size = try(settings.value.max_read_buffer_size, null) + insert_keeper_max_retries = try(settings.value.insert_keeper_max_retries, null) + max_temporary_data_on_disk_size_for_user = try(settings.value.max_temporary_data_on_disk_size_for_user, null) + max_temporary_data_on_disk_size_for_query = try(settings.value.max_temporary_data_on_disk_size_for_query, null) + max_parser_depth = try(settings.value.max_parser_depth, null) + memory_overcommit_ratio_denominator = try(settings.value.memory_overcommit_ratio_denominator, null) + memory_overcommit_ratio_denominator_for_user = try(settings.value.memory_overcommit_ratio_denominator_for_user, null) + memory_usage_overcommit_max_wait_microseconds = try(settings.value.memory_usage_overcommit_max_wait_microseconds, null) + } + } + dynamic "quota" { + for_each = try(user.value.quota, []) + content { + interval_duration = quota.value.interval_duration + queries = quota.value.queries + errors = quota.value.errors + result_rows = quota.value.result_rows + read_rows = quota.value.read_rows + execution_time = quota.value.execution_time + } + } + } + } + + dynamic "host" { + for_each = try(each.value.clickhouse_hosts, []) + content { + type = host.value.type + subnet_id = try(host.value.subnet_id, null) + zone = host.value.zone + fqdn = try(host.value.fqdn, null) + shard_name = try(host.value.shard_name, null) + assign_public_ip = try(host.value.assign_public_ip, null) + } + } + + access { + web_sql = try(each.value.acess_web_sql, false) + data_lens = try(each.value.acess_data_lens, false) + metrika = try(each.value.acess_metrika, false) + serverless = try(each.value.acess_serverless, false) + yandex_query = try(each.value.acess_yandex_query, false) + data_transfer = try(each.value.acess_data_transfer, false) + } + + service_account_id = try(each.value.service_account_id, null) + + cloud_storage { + enabled = try(each.value.cloud_storage_enable, false) + } + + maintenance_window { + type = try(each.value.maintenance_window_type, "ANYTIME") + day = try(each.value.maintenance_window_day, null) + hour = try(each.value.maintenance_window_hour, null) + } +} diff --git a/YandexCloud/Managed-clickhouse/managed-clickhouse/docs/example/example.tf b/YandexCloud/Managed-clickhouse/managed-clickhouse/docs/example/example.tf new file mode 100644 index 0000000..c82a098 --- /dev/null +++ b/YandexCloud/Managed-clickhouse/managed-clickhouse/docs/example/example.tf @@ -0,0 +1,155 @@ +provider "yandex" { + service_account_key_file = "key.json" + cloud_id = YOUR_CLOUD_ID + folder_id = YOUR_FOLDER_ID + zone = "ru-central1-a" +} + +terraform { + backend "s3" { + bucket = "name-your-bucket" + key = "managed-clickhouse.tfstate" + region = "ru-central1" + endpoint = "https://storage.yandexcloud.net" + shared_credentials_file = "storage-cred" + + skip_region_validation = true + skip_credentials_validation = true + skip_metadata_api_check = true + } +} + +data "terraform_remote_state" "vpc" { + backend = "s3" + config = { + bucket = "name-your-bucket" + key = "vpc.tfstate" + region = "ru-central1" + endpoint = "https://storage.yandexcloud.net" + shared_credentials_file = "storage-cred" + + skip_region_validation = true + skip_credentials_validation = true + skip_metadata_api_check = true + } +} + +module "managed-clickhouse" { + source = "github.com/nixys/nxs-marketplace-terraform/YandexCloud/Managed-clickhouse/managed-clickhouse" + + clusters = { + name-your-clickhouse-cluster-1 = { + network_id = data.terraform_remote_state.vpc.outputs.network_ids["name-your-network-1"] + description = "Created by Terraform" + deletion_protection = false + maintenance_window_type = "WEEKLY" + maintenance_window_day = "MON" + maintenance_window_hour = 21 + environment = "PRODUCTION" + clickhouse_version = 23.8 + resource_preset_id = "m2.micro" + disk_type_id = "network-ssd" + disk_size = "100" + admin_password = "" + service_account_id = "aje2dldasdasа5ubdd0k42" + acess_data_lens = true + cloud_storage_enable = true + +######### +### config +######### + config_background_pool_size = 0 + config_background_schedule_pool_size = 0 + config_keep_alive_timeout = 3 + config_log_level = "INFORMATION" + config_mark_cache_size = 4294967296 + config_max_concurrent_queries = 500 + config_max_connections = 4096 + config_metric_log_retention_time = 2592000000 + config_part_log_retention_time = 2592000000 + config_query_log_retention_time = 2592000000 + config_query_thread_log_retention_time = 2592000000 + config_text_log_enabled = false + config_text_log_retention_time = 2592000000 + config_timezone = "Europe/Moscow" + config_trace_log_retention_time = 2592000000 + config_uncompressed_cache_size = 0 +######### +### End Config +######### + + compression = [ + { + level = 0 + method = "LZ4" + min_part_size = 1024 + min_part_size_ratio = 0.5 + } + ] + + shard = [ + { + name = "shard1" + weight = 100 + resources = [{ + resource_preset_id = "m2.micro" + disk_type_id = "network-ssd" + disk_size = "100" + }] + }, + { + name = "shard3" + weight = 100 + resources = [{ + resource_preset_id = "m3-c2-m16" + disk_type_id = "network-ssd" + disk_size = "100" + }] + } + ] + + shard_group = [ + { + name = "shardGroup1" + description = "Created by Terraform" + shard_names = ["shard3", ] + } + ] + + clickhouse_hosts = [ + { + type = "CLICKHOUSE" + subnet_id = data.terraform_remote_state.vpc.outputs.subnet_ids["name-your-subnet-1"] + zone = "ru-central1-a" + shard_name = "shard1" + }, + { + type = "CLICKHOUSE" + subnet_id = data.terraform_remote_state.vpc.outputs.subnet_ids["name-your-subnet-2"] + zone = "ru-central1-b" + shard_name = "shard3" + } + ] + + database = [ + { + name = "name-your-database-1" + } + ] + + user = [ + { + name = "name-your-user-1" + permission = [{ + database_name = "name-your-database-1" + }] + } + ] + + labels = { + cluster = "name-your-clickhouse-cluster-1" + } + } + } + +} diff --git a/YandexCloud/Managed-clickhouse/managed-clickhouse/docs/example/versions.tf b/YandexCloud/Managed-clickhouse/managed-clickhouse/docs/example/versions.tf new file mode 100644 index 0000000..3a35109 --- /dev/null +++ b/YandexCloud/Managed-clickhouse/managed-clickhouse/docs/example/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= v1.0.0" + + required_providers { + yandex = { + source = "yandex-cloud/yandex" + version = ">= 0.107.0" + } + } +} \ No newline at end of file diff --git a/YandexCloud/Managed-clickhouse/managed-clickhouse/password.tf b/YandexCloud/Managed-clickhouse/managed-clickhouse/password.tf new file mode 100644 index 0000000..9b7311a --- /dev/null +++ b/YandexCloud/Managed-clickhouse/managed-clickhouse/password.tf @@ -0,0 +1,12 @@ +resource "random_password" "user_pass" { + for_each = toset(flatten([ for k in var.clusters : [ for user in k.user : user.name ] ])) + + length = 16 + special = true +} + +resource "random_password" "admin_pass" { + + length = 16 + special = true +} \ No newline at end of file diff --git a/YandexCloud/Managed-clickhouse/managed-clickhouse/variables.tf b/YandexCloud/Managed-clickhouse/managed-clickhouse/variables.tf new file mode 100644 index 0000000..2c9a8db --- /dev/null +++ b/YandexCloud/Managed-clickhouse/managed-clickhouse/variables.tf @@ -0,0 +1,5 @@ +variable "clusters" { + description = "Create Clickhouse cluster" + type = any + default = {} +} diff --git a/YandexCloud/Managed-clickhouse/managed-clickhouse/versions.tf b/YandexCloud/Managed-clickhouse/managed-clickhouse/versions.tf new file mode 100644 index 0000000..3a35109 --- /dev/null +++ b/YandexCloud/Managed-clickhouse/managed-clickhouse/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= v1.0.0" + + required_providers { + yandex = { + source = "yandex-cloud/yandex" + version = ">= 0.107.0" + } + } +} \ No newline at end of file