From 01a49495647ec69019a58423d13b6cb7164afbef Mon Sep 17 00:00:00 2001 From: Prykhodko Date: Tue, 7 Nov 2023 11:27:04 +0100 Subject: [PATCH 1/3] cudos v5 definition --- .../core/data/datasets/cudos/hourly_view.json | 178 +++++++++++++ .../data/datasets/cudos/resource_view.json | 242 ++++++++++++++++++ .../core/data/queries/cudos/hourly_view.sql | 25 ++ .../data/queries/cudos/hourly_view_ri.sql | 25 ++ .../data/queries/cudos/hourly_view_sp.sql | 25 ++ .../data/queries/cudos/hourly_view_sp_ri.sql | 25 ++ .../core/data/queries/cudos/resource_view.sql | 25 ++ .../data/queries/cudos/resource_view_ri.sql | 25 ++ .../data/queries/cudos/resource_view_sp.sql | 25 ++ .../queries/cudos/resource_view_sp_ri.sql | 25 ++ cid/builtin/core/data/resources.yaml | 50 +++- 11 files changed, 666 insertions(+), 4 deletions(-) create mode 100644 cid/builtin/core/data/datasets/cudos/hourly_view.json create mode 100644 cid/builtin/core/data/datasets/cudos/resource_view.json create mode 100644 cid/builtin/core/data/queries/cudos/hourly_view.sql create mode 100644 cid/builtin/core/data/queries/cudos/hourly_view_ri.sql create mode 100644 cid/builtin/core/data/queries/cudos/hourly_view_sp.sql create mode 100644 cid/builtin/core/data/queries/cudos/hourly_view_sp_ri.sql create mode 100644 cid/builtin/core/data/queries/cudos/resource_view.sql create mode 100644 cid/builtin/core/data/queries/cudos/resource_view_ri.sql create mode 100644 cid/builtin/core/data/queries/cudos/resource_view_sp.sql create mode 100644 cid/builtin/core/data/queries/cudos/resource_view_sp_ri.sql diff --git a/cid/builtin/core/data/datasets/cudos/hourly_view.json b/cid/builtin/core/data/datasets/cudos/hourly_view.json new file mode 100644 index 00000000..5f6f4438 --- /dev/null +++ b/cid/builtin/core/data/datasets/cudos/hourly_view.json @@ -0,0 +1,178 @@ +{ + "DataSetId": "78f4cc36-c46c-431b-91f0-c72e147d9bcb", + "Name": "hourly_view", + "PhysicalTableMap": { + "d456be68-afcd-4dc9-a024-6e0cfe596eb9": { + "RelationalTable": { + "DataSourceArn": "${athena_datasource_arn}", + "Schema": "${athena_database_name}", + "Name": "account_map", + "InputColumns": [ + { + "Name": "account_id", + "Type": "STRING" + }, + { + "Name": "account_name", + "Type": "STRING" + } + ] + } + }, + "e0c1051e-daef-4661-9554-42989b38effa": { + "RelationalTable": { + "DataSourceArn": "${athena_datasource_arn}", + "Schema": "${athena_database_name}", + "Name": "hourly_view", + "InputColumns": [ + { + "Name": "product_code", + "Type": "STRING" + }, + { + "Name": "service", + "Type": "STRING" + }, + { + "Name": "operation", + "Type": "STRING" + }, + { + "Name": "charge_type", + "Type": "STRING" + }, + { + "Name": "usage_type", + "Type": "STRING" + }, + { + "Name": "item_description", + "Type": "STRING" + }, + { + "Name": "pricing_unit", + "Type": "STRING" + }, + { + "Name": "region", + "Type": "STRING" + }, + { + "Name": "pricing_term", + "Type": "STRING" + }, + { + "Name": "billing_period", + "Type": "DATETIME" + }, + { + "Name": "usage_date", + "Type": "DATETIME" + }, + { + "Name": "payer_account_id", + "Type": "STRING" + }, + { + "Name": "linked_account_id", + "Type": "STRING" + }, + { + "Name": "savings_plan_a_r_n", + "Type": "STRING" + }, + { + "Name": "reservation_a_r_n", + "Type": "STRING" + }, + { + "Name": "unblended_cost", + "Type": "DECIMAL" + }, + { + "Name": "reservation_effective_cost", + "Type": "DECIMAL" + }, + { + "Name": "savings_plan_effective_cost", + "Type": "DECIMAL" + }, + { + "Name": "usage_quantity", + "Type": "DECIMAL" + } + ] + } + } + }, + "LogicalTableMap": { + "a34a38d8-7504-4aba-89e8-d038d5fb5706": { + "Alias": "Intermediate Table", + "DataTransforms": [ + { + "ProjectOperation": { + "ProjectedColumns": [ + "product_code", + "service", + "operation", + "charge_type", + "usage_type", + "item_description", + "pricing_unit", + "region", + "pricing_term", + "billing_period", + "usage_date", + "payer_account_id", + "linked_account_id", + "savings_plan_a_r_n", + "reservation_a_r_n", + "unblended_cost", + "reservation_effective_cost", + "savings_plan_effective_cost", + "usage_quantity", + "account_id", + "account_name" + ] + } + } + ], + "Source": { + "JoinInstruction": { + "LeftOperand": "c08f9029-26ae-4a4c-91bc-ffad52dc67e3", + "RightOperand": "dea38012-0b99-4729-bebb-fe071a8b5327", + "Type": "LEFT", + "OnClause": "{linked_account_id} = {account_id}" + } + } + }, + "c08f9029-26ae-4a4c-91bc-ffad52dc67e3": { + "Alias": "hourly_view", + "DataTransforms": [ + { + "CastColumnTypeOperation": { + "ColumnName": "reservation_effective_cost", + "NewColumnType": "DECIMAL" + } + }, + { + "CastColumnTypeOperation": { + "ColumnName": "usage_quantity", + "NewColumnType": "DECIMAL" + } + } + ], + "Source": { + "PhysicalTableId": "e0c1051e-daef-4661-9554-42989b38effa" + } + }, + "dea38012-0b99-4729-bebb-fe071a8b5327": { + "Alias": "account_map", + "Source": { + "PhysicalTableId": "d456be68-afcd-4dc9-a024-6e0cfe596eb9" + } + } + }, + "ImportMode": "SPICE", + "Permissions": [] +} diff --git a/cid/builtin/core/data/datasets/cudos/resource_view.json b/cid/builtin/core/data/datasets/cudos/resource_view.json new file mode 100644 index 00000000..b690cfaa --- /dev/null +++ b/cid/builtin/core/data/datasets/cudos/resource_view.json @@ -0,0 +1,242 @@ +{ + "DataSetId": "ce8e358c-3621-4983-9e0e-ba65b86346d6", + "Name": "resource_view", + "PhysicalTableMap": { + "122b690e-e762-40eb-9c6c-cc97ff3420fa": { + "RelationalTable": { + "DataSourceArn": "${athena_datasource_arn}", + "Schema": "${athena_database_name}", + "Name": "resource_view", + "InputColumns": [ + { + "Name": "usage_date", + "Type": "DATETIME" + }, + { + "Name": "payer_account_id", + "Type": "STRING" + }, + { + "Name": "linked_account_id", + "Type": "STRING" + }, + { + "Name": "billing_entity", + "Type": "STRING" + }, + { + "Name": "product_name", + "Type": "STRING" + }, + { + "Name": "resource_id", + "Type": "STRING" + }, + { + "Name": "product_code", + "Type": "STRING" + }, + { + "Name": "operation", + "Type": "STRING" + }, + { + "Name": "charge_type", + "Type": "STRING" + }, + { + "Name": "usage_type", + "Type": "STRING" + }, + { + "Name": "pricing_unit", + "Type": "STRING" + }, + { + "Name": "region", + "Type": "STRING" + }, + { + "Name": "item_description", + "Type": "STRING" + }, + { + "Name": "legal_entity", + "Type": "STRING" + }, + { + "Name": "pricing_term", + "Type": "STRING" + }, + { + "Name": "database_engine", + "Type": "STRING" + }, + { + "Name": "product_deployment_option", + "Type": "STRING" + }, + { + "Name": "product_from_location", + "Type": "STRING" + }, + { + "Name": "product_group", + "Type": "STRING" + }, + { + "Name": "instance_type", + "Type": "STRING" + }, + { + "Name": "instance_type_family", + "Type": "STRING" + }, + { + "Name": "platform", + "Type": "STRING" + }, + { + "Name": "product_family", + "Type": "STRING" + }, + { + "Name": "service", + "Type": "STRING" + }, + { + "Name": "product_storage", + "Type": "STRING" + }, + { + "Name": "product_to_location", + "Type": "STRING" + }, + { + "Name": "product_volume_api_name", + "Type": "STRING" + }, + { + "Name": "reservation_a_r_n", + "Type": "STRING" + }, + { + "Name": "savings_plan_a_r_n", + "Type": "STRING" + }, + { + "Name": "savings_plan_effective_cost", + "Type": "DECIMAL" + }, + { + "Name": "reservation_effective_cost", + "Type": "DECIMAL" + }, + { + "Name": "usage_quantity", + "Type": "DECIMAL" + }, + { + "Name": "unblended_cost", + "Type": "DECIMAL" + } + ] + } + }, + "bfc903ee-63a5-4e0b-bfb9-9ae033bb689b": { + "RelationalTable": { + "DataSourceArn": "${athena_datasource_arn}", + "Schema": "${athena_database_name}", + "Name": "account_map", + "InputColumns": [ + { + "Name": "account_id", + "Type": "STRING" + }, + { + "Name": "account_name", + "Type": "STRING" + } + ] + } + } + }, + "LogicalTableMap": { + "6216b450-cc86-4960-9db1-36557c62fdd0": { + "Alias": "account_map", + "Source": { + "PhysicalTableId": "bfc903ee-63a5-4e0b-bfb9-9ae033bb689b" + } + }, + "96dd49f3-b1fa-4b75-8052-887b4c64e1df": { + "Alias": "Intermediate Table", + "DataTransforms": [ + { + "ProjectOperation": { + "ProjectedColumns": [ + "usage_date", + "payer_account_id", + "linked_account_id", + "billing_entity", + "product_name", + "resource_id", + "product_code", + "operation", + "charge_type", + "usage_type", + "pricing_unit", + "region", + "item_description", + "legal_entity", + "pricing_term", + "database_engine", + "product_deployment_option", + "product_from_location", + "product_group", + "instance_type", + "instance_type_family", + "platform", + "product_family", + "service", + "product_storage", + "product_to_location", + "product_volume_api_name", + "reservation_a_r_n", + "savings_plan_a_r_n", + "savings_plan_effective_cost", + "reservation_effective_cost", + "usage_quantity", + "unblended_cost", + "account_id", + "account_name" + ] + } + } + ], + "Source": { + "JoinInstruction": { + "LeftOperand": "c3a83f91-b920-419d-a4fa-d5f023bd1aa6", + "RightOperand": "6216b450-cc86-4960-9db1-36557c62fdd0", + "Type": "LEFT", + "OnClause": "{linked_account_id} = {account_id}" + } + } + }, + "c3a83f91-b920-419d-a4fa-d5f023bd1aa6": { + "Alias": "resource_view", + "DataTransforms": [ + { + "CastColumnTypeOperation": { + "ColumnName": "reservation_effective_cost", + "NewColumnType": "DECIMAL" + } + } + ], + "Source": { + "PhysicalTableId": "122b690e-e762-40eb-9c6c-cc97ff3420fa" + } + } + }, + "ImportMode": "SPICE", + "Permissions": [] +} diff --git a/cid/builtin/core/data/queries/cudos/hourly_view.sql b/cid/builtin/core/data/queries/cudos/hourly_view.sql new file mode 100644 index 00000000..6f6af2da --- /dev/null +++ b/cid/builtin/core/data/queries/cudos/hourly_view.sql @@ -0,0 +1,25 @@ +CREATE OR REPLACE VIEW hourly_view AS + SELECT DISTINCT + "line_item_product_code" "product_code" + , "product_servicecode" "service" + , "line_item_operation" "operation" + , "line_item_line_item_type" "charge_type" + , "line_item_usage_type" "usage_type" + , "line_item_line_item_description" "item_description" + , "pricing_unit" "pricing_unit" + , "product_region" "region" + , "pricing_term" "pricing_term" + , "bill_billing_period_start_date" "billing_period" + , "line_item_usage_start_date" "usage_date" + , "bill_payer_account_id" "payer_account_id" + , "line_item_usage_account_id" "linked_account_id" + , CAST('' AS varchar) "savings_plan_a_r_n" + , CAST('' AS varchar) "reservation_a_r_n" + , "sum"("line_item_unblended_cost") "unblended_cost" + , CAST(0 AS double) "reservation_effective_cost" + , CAST(0 AS double) "savings_plan_effective_cost" + , "sum"("line_item_usage_amount") "usage_quantity" + FROM + "${cur_table_name}" + WHERE (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))) + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 \ No newline at end of file diff --git a/cid/builtin/core/data/queries/cudos/hourly_view_ri.sql b/cid/builtin/core/data/queries/cudos/hourly_view_ri.sql new file mode 100644 index 00000000..ab2acd1c --- /dev/null +++ b/cid/builtin/core/data/queries/cudos/hourly_view_ri.sql @@ -0,0 +1,25 @@ +CREATE OR REPLACE VIEW hourly_view AS + SELECT DISTINCT + "line_item_product_code" "product_code" + , "product_servicecode" "service" + , "line_item_operation" "operation" + , "line_item_line_item_type" "charge_type" + , "line_item_usage_type" "usage_type" + , "line_item_line_item_description" "item_description" + , "pricing_unit" "pricing_unit" + , "product_region" "region" + , "pricing_term" "pricing_term" + , "bill_billing_period_start_date" "billing_period" + , "line_item_usage_start_date" "usage_date" + , "bill_payer_account_id" "payer_account_id" + , "line_item_usage_account_id" "linked_account_id" + , CAST('' AS varchar) "savings_plan_a_r_n" + , "reservation_reservation_a_r_n" "reservation_a_r_n" + , "sum"("line_item_unblended_cost") "unblended_cost" + , "sum"("reservation_effective_cost") "reservation_effective_cost" + , CAST(0 AS double) "savings_plan_effective_cost" + , "sum"("line_item_usage_amount") "usage_quantity" + FROM + "${cur_table_name}" + WHERE (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))) + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 \ No newline at end of file diff --git a/cid/builtin/core/data/queries/cudos/hourly_view_sp.sql b/cid/builtin/core/data/queries/cudos/hourly_view_sp.sql new file mode 100644 index 00000000..4ded273a --- /dev/null +++ b/cid/builtin/core/data/queries/cudos/hourly_view_sp.sql @@ -0,0 +1,25 @@ +CREATE OR REPLACE VIEW hourly_view AS + SELECT DISTINCT + "line_item_product_code" "product_code" + , "product_servicecode" "service" + , "line_item_operation" "operation" + , "line_item_line_item_type" "charge_type" + , "line_item_usage_type" "usage_type" + , "line_item_line_item_description" "item_description" + , "pricing_unit" "pricing_unit" + , "product_region" "region" + , "pricing_term" "pricing_term" + , "bill_billing_period_start_date" "billing_period" + , "line_item_usage_start_date" "usage_date" + , "bill_payer_account_id" "payer_account_id" + , "line_item_usage_account_id" "linked_account_id" + , "savings_plan_savings_plan_a_r_n" "savings_plan_a_r_n" + , CAST('' AS varchar) "reservation_a_r_n" + , "sum"("line_item_unblended_cost") "unblended_cost" + , CAST(0 AS double) "reservation_effective_cost" + , "sum"("savings_plan_savings_plan_effective_cost") "savings_plan_effective_cost" + , "sum"("line_item_usage_amount") "usage_quantity" + FROM + "${cur_table_name}" + WHERE (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))) + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 \ No newline at end of file diff --git a/cid/builtin/core/data/queries/cudos/hourly_view_sp_ri.sql b/cid/builtin/core/data/queries/cudos/hourly_view_sp_ri.sql new file mode 100644 index 00000000..3bd35142 --- /dev/null +++ b/cid/builtin/core/data/queries/cudos/hourly_view_sp_ri.sql @@ -0,0 +1,25 @@ +CREATE OR REPLACE VIEW hourly_view AS + SELECT DISTINCT + "line_item_product_code" "product_code" + , "product_servicecode" "service" + , "line_item_operation" "operation" + , "line_item_line_item_type" "charge_type" + , "line_item_usage_type" "usage_type" + , "line_item_line_item_description" "item_description" + , "pricing_unit" "pricing_unit" + , "product_region" "region" + , "pricing_term" "pricing_term" + , "bill_billing_period_start_date" "billing_period" + , "line_item_usage_start_date" "usage_date" + , "bill_payer_account_id" "payer_account_id" + , "line_item_usage_account_id" "linked_account_id" + , CAST('' AS varchar) "savings_plan_a_r_n" + , "reservation_reservation_a_r_n" "reservation_a_r_n" + , "sum"("line_item_unblended_cost") "unblended_cost" + , "sum"("reservation_effective_cost") "reservation_effective_cost" + , CAST(0 AS double) "savings_plan_effective_cost" + , "sum"("line_item_usage_amount") "usage_quantity" + FROM + "${cur_table_name}" + WHERE (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))) + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 \ No newline at end of file diff --git a/cid/builtin/core/data/queries/cudos/resource_view.sql b/cid/builtin/core/data/queries/cudos/resource_view.sql new file mode 100644 index 00000000..3db9c5a8 --- /dev/null +++ b/cid/builtin/core/data/queries/cudos/resource_view.sql @@ -0,0 +1,25 @@ +CREATE OR REPLACE VIEW hourly_view AS + SELECT DISTINCT + "line_item_product_code" "product_code" + , "product_servicecode" "service" + , "line_item_operation" "operation" + , "line_item_line_item_type" "charge_type" + , "line_item_usage_type" "usage_type" + , "line_item_line_item_description" "item_description" + , "pricing_unit" "pricing_unit" + , "product_region" "region" + , "pricing_term" "pricing_term" + , "bill_billing_period_start_date" "billing_period" + , "line_item_usage_start_date" "usage_date" + , "bill_payer_account_id" "payer_account_id" + , "line_item_usage_account_id" "linked_account_id" + , CAST('' AS varchar) "savings_plan_a_r_n" + , CAST('' AS varchar) "reservation_a_r_n" + , "sum"("line_item_unblended_cost") "unblended_cost" + , CAST(0 AS double) "reservation_effective_cost" + , CAST(0 AS double) "savings_plan_effective_cost" + , "sum"("line_item_usage_amount") "usage_quantity" + FROM + "${cur_table_name}" + WHERE (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))) + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 \ No newline at end of file diff --git a/cid/builtin/core/data/queries/cudos/resource_view_ri.sql b/cid/builtin/core/data/queries/cudos/resource_view_ri.sql new file mode 100644 index 00000000..3bd35142 --- /dev/null +++ b/cid/builtin/core/data/queries/cudos/resource_view_ri.sql @@ -0,0 +1,25 @@ +CREATE OR REPLACE VIEW hourly_view AS + SELECT DISTINCT + "line_item_product_code" "product_code" + , "product_servicecode" "service" + , "line_item_operation" "operation" + , "line_item_line_item_type" "charge_type" + , "line_item_usage_type" "usage_type" + , "line_item_line_item_description" "item_description" + , "pricing_unit" "pricing_unit" + , "product_region" "region" + , "pricing_term" "pricing_term" + , "bill_billing_period_start_date" "billing_period" + , "line_item_usage_start_date" "usage_date" + , "bill_payer_account_id" "payer_account_id" + , "line_item_usage_account_id" "linked_account_id" + , CAST('' AS varchar) "savings_plan_a_r_n" + , "reservation_reservation_a_r_n" "reservation_a_r_n" + , "sum"("line_item_unblended_cost") "unblended_cost" + , "sum"("reservation_effective_cost") "reservation_effective_cost" + , CAST(0 AS double) "savings_plan_effective_cost" + , "sum"("line_item_usage_amount") "usage_quantity" + FROM + "${cur_table_name}" + WHERE (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))) + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 \ No newline at end of file diff --git a/cid/builtin/core/data/queries/cudos/resource_view_sp.sql b/cid/builtin/core/data/queries/cudos/resource_view_sp.sql new file mode 100644 index 00000000..186c3614 --- /dev/null +++ b/cid/builtin/core/data/queries/cudos/resource_view_sp.sql @@ -0,0 +1,25 @@ +CREATE OR REPLACE VIEW hourly_view AS + SELECT DISTINCT + "line_item_product_code" "product_code" + , "product_servicecode" "service" + , "line_item_operation" "operation" + , "line_item_line_item_type" "charge_type" + , "line_item_usage_type" "usage_type" + , "line_item_line_item_description" "item_description" + , "pricing_unit" "pricing_unit" + , "product_region" "region" + , "pricing_term" "pricing_term" + , "bill_billing_period_start_date" "billing_period" + , "line_item_usage_start_date" "usage_date" + , "bill_payer_account_id" "payer_account_id" + , "line_item_usage_account_id" "linked_account_id" + , "savings_plan_savings_plan_a_r_n" "savings_plan_a_r_n" + , CAST('' AS varchar) "reservation_a_r_n" + , "sum"("line_item_unblended_cost") "unblended_cost" + , CAST(0 AS double) "reservation_effective_cost" + , "sum"("savings_plan_savings_plan_effective_cost") "savings_plan_effective_cost" + , "sum"("line_item_usage_amount") "usage_quantity" + FROM + "${cur_table_name}" + WHERE (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))) + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 \ No newline at end of file diff --git a/cid/builtin/core/data/queries/cudos/resource_view_sp_ri.sql b/cid/builtin/core/data/queries/cudos/resource_view_sp_ri.sql new file mode 100644 index 00000000..d8dbc4e1 --- /dev/null +++ b/cid/builtin/core/data/queries/cudos/resource_view_sp_ri.sql @@ -0,0 +1,25 @@ +CREATE OR REPLACE VIEW hourly_view AS + SELECT DISTINCT + "line_item_product_code" "product_code" + , "product_servicecode" "service" + , "line_item_operation" "operation" + , "line_item_line_item_type" "charge_type" + , "line_item_usage_type" "usage_type" + , "line_item_line_item_description" "item_description" + , "pricing_unit" "pricing_unit" + , "product_region" "region" + , "pricing_term" "pricing_term" + , "bill_billing_period_start_date" "billing_period" + , "line_item_usage_start_date" "usage_date" + , "bill_payer_account_id" "payer_account_id" + , "line_item_usage_account_id" "linked_account_id" + , "savings_plan_savings_plan_a_r_n" "savings_plan_a_r_n" + , "reservation_reservation_a_r_n" "reservation_a_r_n" + , "sum"("line_item_unblended_cost") "unblended_cost" + , "sum"("reservation_effective_cost") "reservation_effective_cost" + , "sum"("savings_plan_savings_plan_effective_cost") "savings_plan_effective_cost" + , "sum"("line_item_usage_amount") "usage_quantity" + FROM + "${cur_table_name}" + WHERE (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))) + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 \ No newline at end of file diff --git a/cid/builtin/core/data/resources.yaml b/cid/builtin/core/data/resources.yaml index 1d2ce325..7ec6cd9e 100644 --- a/cid/builtin/core/data/resources.yaml +++ b/cid/builtin/core/data/resources.yaml @@ -16,6 +16,18 @@ dashboards: versions: minTemplateVersion: 189 minTemplateDescription: "v4.75.0" + CUDOSv5: + name: CUDOS Dashboard v5 + templateId: cudos_dashboard_v5 + dashboardId: cudos_v5 + dependsOn: + datasets: + - summary_view + - hourly_view + - resource_view + versions: + minTemplateVersion: 1 + minTemplateDescription: "v5.0.0" CID: name: Cost Intelligence Dashboard @@ -88,7 +100,7 @@ dashboards: # QuickSight DataSets definitions datasets: - # CID Shared DataSets + # Shared DataSet between CUDOS, CID and KPI summary_view: File: cid/summary_view.json dependsOn: @@ -97,6 +109,7 @@ datasets: - ri_sp_mapping schedules: - default + # CID dataset ec2_running_cost: File: cid/ec2_running_cost.json dependsOn: @@ -104,6 +117,7 @@ datasets: - ec2_running_cost schedules: - default + # CID dataset compute_savings_plan_eligible_spend: File: cid/compute.json dependsOn: @@ -111,6 +125,7 @@ datasets: - compute_savings_plan_eligible_spend schedules: - default + # CID dataset s3_view: File: cid/s3_view.json dependsOn: @@ -120,14 +135,28 @@ datasets: schedules: - default - # Shared DataSets + # CUDOS v4 dataset customer_all: File: shared/customer_all.json dependsOn: cur: true views: - ${cur_table_name} - + # CUDOS v5 datasets + hourly_view: + File: cudos/hourly_view.json + dependsOn: + views: + - hourly_view + schedules: + - default + resource_view: + File: cudos/resource_view.json + dependsOn: + views: + - resource_view + schedules: + - default # KPI DataSets kpi_ebs_snap: File: kpi/kpi_ebs_snap.json @@ -254,7 +283,20 @@ views: File: cid/ri_sp_mapping.sql dependsOn: cur: true - + hourly_view: + spriFile: cudos/hourly_view_sp_ri.sql + spFile: cudos/hourly_view_sp.sql + riFile: cudos/hourly_view_ri.sql + File: cudos/hourly_view.sql + dependsOn: + cur: true + resource_view: + spriFile: cudos/resource_view_sp_ri.sql + spFile: cudos/resource_view_sp.sql + riFile: cudos/resource_view_ri.sql + File: cudos/resource_view.sql + dependsOn: + cur: true # Trends daily_anomaly_detection: File: trends/daily_anomaly_detection.sql From 6a8ca14a740a51d36f765e6c74102be32f7ecf88 Mon Sep 17 00:00:00 2001 From: Prykhodko Date: Tue, 7 Nov 2023 12:03:54 +0100 Subject: [PATCH 2/3] cudos v5 resource view --- .../core/data/queries/cudos/resource_view.sql | 64 +++++++++++-------- .../data/queries/cudos/resource_view_ri.sql | 64 +++++++++++-------- .../data/queries/cudos/resource_view_sp.sql | 64 +++++++++++-------- .../queries/cudos/resource_view_sp_ri.sql | 64 +++++++++++-------- cid/builtin/core/data/resources.yaml | 2 +- 5 files changed, 157 insertions(+), 101 deletions(-) diff --git a/cid/builtin/core/data/queries/cudos/resource_view.sql b/cid/builtin/core/data/queries/cudos/resource_view.sql index 3db9c5a8..4c1f6beb 100644 --- a/cid/builtin/core/data/queries/cudos/resource_view.sql +++ b/cid/builtin/core/data/queries/cudos/resource_view.sql @@ -1,25 +1,39 @@ -CREATE OR REPLACE VIEW hourly_view AS - SELECT DISTINCT - "line_item_product_code" "product_code" - , "product_servicecode" "service" - , "line_item_operation" "operation" - , "line_item_line_item_type" "charge_type" - , "line_item_usage_type" "usage_type" - , "line_item_line_item_description" "item_description" - , "pricing_unit" "pricing_unit" - , "product_region" "region" - , "pricing_term" "pricing_term" - , "bill_billing_period_start_date" "billing_period" - , "line_item_usage_start_date" "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , CAST('' AS varchar) "savings_plan_a_r_n" - , CAST('' AS varchar) "reservation_a_r_n" - , "sum"("line_item_unblended_cost") "unblended_cost" - , CAST(0 AS double) "reservation_effective_cost" - , CAST(0 AS double) "savings_plan_effective_cost" - , "sum"("line_item_usage_amount") "usage_quantity" - FROM - "${cur_table_name}" - WHERE (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))) - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 \ No newline at end of file +CREATE OR REPLACE VIEW resource_view AS + SELECT DISTINCT + "date_trunc"('day', "line_item_usage_start_date") "usage_date" + , "bill_payer_account_id" "payer_account_id" + , "line_item_usage_account_id" "linked_account_id" + , "bill_billing_entity" "billing_entity" + , "product_product_name" "product_name" + , "line_item_resource_id" "resource_id" + , "line_item_product_code" "product_code" + , "line_item_operation" "operation" + , "line_item_line_item_type" "charge_type" + , "line_item_usage_type" "usage_type" + , "pricing_unit" "pricing_unit" + , "product_region" "region" + , "line_item_line_item_description" "item_description" + , "line_item_legal_entity" "legal_entity" + , "pricing_term" "pricing_term" + , "product_database_engine" "database_engine" + , "product_deployment_option" "product_deployment_option" + , "product_from_location" "product_from_location" + , "product_group" "product_group" + , "product_instance_type" "instance_type" + , "product_instance_type_family" "instance_type_family" + , "product_operating_system" "platform" + , "product_product_family" "product_family" + , "product_servicecode" "service" + , "product_storage" "product_storage" + , "product_to_location" "product_to_location" + , "product_volume_api_name" "product_volume_api_name" + , CAST('' AS varchar) "reservation_a_r_n" + , CAST('' AS varchar) "savings_plan_a_r_n" + , CAST(0 AS double) savings_plan_effective_cost + , CAST(0 AS double) reservation_effective_cost + , "sum"("line_item_usage_amount") "usage_quantity" + , "sum"("line_item_unblended_cost") unblended_cost + FROM + "${cur_table_name}" + WHERE (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) AND (line_item_resource_id <> '')) + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29 \ No newline at end of file diff --git a/cid/builtin/core/data/queries/cudos/resource_view_ri.sql b/cid/builtin/core/data/queries/cudos/resource_view_ri.sql index 3bd35142..af407d96 100644 --- a/cid/builtin/core/data/queries/cudos/resource_view_ri.sql +++ b/cid/builtin/core/data/queries/cudos/resource_view_ri.sql @@ -1,25 +1,39 @@ -CREATE OR REPLACE VIEW hourly_view AS - SELECT DISTINCT - "line_item_product_code" "product_code" - , "product_servicecode" "service" - , "line_item_operation" "operation" - , "line_item_line_item_type" "charge_type" - , "line_item_usage_type" "usage_type" - , "line_item_line_item_description" "item_description" - , "pricing_unit" "pricing_unit" - , "product_region" "region" - , "pricing_term" "pricing_term" - , "bill_billing_period_start_date" "billing_period" - , "line_item_usage_start_date" "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , CAST('' AS varchar) "savings_plan_a_r_n" - , "reservation_reservation_a_r_n" "reservation_a_r_n" - , "sum"("line_item_unblended_cost") "unblended_cost" - , "sum"("reservation_effective_cost") "reservation_effective_cost" - , CAST(0 AS double) "savings_plan_effective_cost" - , "sum"("line_item_usage_amount") "usage_quantity" - FROM - "${cur_table_name}" - WHERE (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))) - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 \ No newline at end of file +CREATE OR REPLACE VIEW resource_view AS + SELECT DISTINCT + "date_trunc"('day', "line_item_usage_start_date") "usage_date" + , "bill_payer_account_id" "payer_account_id" + , "line_item_usage_account_id" "linked_account_id" + , "bill_billing_entity" "billing_entity" + , "product_product_name" "product_name" + , "line_item_resource_id" "resource_id" + , "line_item_product_code" "product_code" + , "line_item_operation" "operation" + , "line_item_line_item_type" "charge_type" + , "line_item_usage_type" "usage_type" + , "pricing_unit" "pricing_unit" + , "product_region" "region" + , "line_item_line_item_description" "item_description" + , "line_item_legal_entity" "legal_entity" + , "pricing_term" "pricing_term" + , "product_database_engine" "database_engine" + , "product_deployment_option" "product_deployment_option" + , "product_from_location" "product_from_location" + , "product_group" "product_group" + , "product_instance_type" "instance_type" + , "product_instance_type_family" "instance_type_family" + , "product_operating_system" "platform" + , "product_product_family" "product_family" + , "product_servicecode" "service" + , "product_storage" "product_storage" + , "product_to_location" "product_to_location" + , "product_volume_api_name" "product_volume_api_name" + , "reservation_reservation_a_r_n" "reservation_a_r_n" + , CAST('' AS varchar) "savings_plan_a_r_n" + , CAST(0 AS double) savings_plan_effective_cost + , "sum"("reservation_effective_cost") reservation_effective_cost + , "sum"("line_item_usage_amount") "usage_quantity" + , "sum"("line_item_unblended_cost") unblended_cost + FROM + "${cur_table_name}" + WHERE (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) AND (line_item_resource_id <> '')) + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29 \ No newline at end of file diff --git a/cid/builtin/core/data/queries/cudos/resource_view_sp.sql b/cid/builtin/core/data/queries/cudos/resource_view_sp.sql index 186c3614..ac454a72 100644 --- a/cid/builtin/core/data/queries/cudos/resource_view_sp.sql +++ b/cid/builtin/core/data/queries/cudos/resource_view_sp.sql @@ -1,25 +1,39 @@ -CREATE OR REPLACE VIEW hourly_view AS - SELECT DISTINCT - "line_item_product_code" "product_code" - , "product_servicecode" "service" - , "line_item_operation" "operation" - , "line_item_line_item_type" "charge_type" - , "line_item_usage_type" "usage_type" - , "line_item_line_item_description" "item_description" - , "pricing_unit" "pricing_unit" - , "product_region" "region" - , "pricing_term" "pricing_term" - , "bill_billing_period_start_date" "billing_period" - , "line_item_usage_start_date" "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , "savings_plan_savings_plan_a_r_n" "savings_plan_a_r_n" - , CAST('' AS varchar) "reservation_a_r_n" - , "sum"("line_item_unblended_cost") "unblended_cost" - , CAST(0 AS double) "reservation_effective_cost" - , "sum"("savings_plan_savings_plan_effective_cost") "savings_plan_effective_cost" - , "sum"("line_item_usage_amount") "usage_quantity" - FROM - "${cur_table_name}" - WHERE (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))) - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 \ No newline at end of file +CREATE OR REPLACE VIEW resource_view AS + SELECT DISTINCT + "date_trunc"('day', "line_item_usage_start_date") "usage_date" + , "bill_payer_account_id" "payer_account_id" + , "line_item_usage_account_id" "linked_account_id" + , "bill_billing_entity" "billing_entity" + , "product_product_name" "product_name" + , "line_item_resource_id" "resource_id" + , "line_item_product_code" "product_code" + , "line_item_operation" "operation" + , "line_item_line_item_type" "charge_type" + , "line_item_usage_type" "usage_type" + , "pricing_unit" "pricing_unit" + , "product_region" "region" + , "line_item_line_item_description" "item_description" + , "line_item_legal_entity" "legal_entity" + , "pricing_term" "pricing_term" + , "product_database_engine" "database_engine" + , "product_deployment_option" "product_deployment_option" + , "product_from_location" "product_from_location" + , "product_group" "product_group" + , "product_instance_type" "instance_type" + , "product_instance_type_family" "instance_type_family" + , "product_operating_system" "platform" + , "product_product_family" "product_family" + , "product_servicecode" "service" + , "product_storage" "product_storage" + , "product_to_location" "product_to_location" + , "product_volume_api_name" "product_volume_api_name" + , CAST('' AS varchar) "reservation_a_r_n" + , "savings_plan_savings_plan_a_r_n" "savings_plan_a_r_n" + , "sum"("savings_plan_savings_plan_effective_cost") savings_plan_effective_cost + , CAST(0 AS double) reservation_effective_cost + , "sum"("line_item_usage_amount") "usage_quantity" + , "sum"("line_item_unblended_cost") unblended_cost + FROM + "${cur_table_name}" + WHERE (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) AND (line_item_resource_id <> '')) + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29 \ No newline at end of file diff --git a/cid/builtin/core/data/queries/cudos/resource_view_sp_ri.sql b/cid/builtin/core/data/queries/cudos/resource_view_sp_ri.sql index d8dbc4e1..61d7e4d0 100644 --- a/cid/builtin/core/data/queries/cudos/resource_view_sp_ri.sql +++ b/cid/builtin/core/data/queries/cudos/resource_view_sp_ri.sql @@ -1,25 +1,39 @@ -CREATE OR REPLACE VIEW hourly_view AS - SELECT DISTINCT - "line_item_product_code" "product_code" - , "product_servicecode" "service" - , "line_item_operation" "operation" - , "line_item_line_item_type" "charge_type" - , "line_item_usage_type" "usage_type" - , "line_item_line_item_description" "item_description" - , "pricing_unit" "pricing_unit" - , "product_region" "region" - , "pricing_term" "pricing_term" - , "bill_billing_period_start_date" "billing_period" - , "line_item_usage_start_date" "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , "savings_plan_savings_plan_a_r_n" "savings_plan_a_r_n" - , "reservation_reservation_a_r_n" "reservation_a_r_n" - , "sum"("line_item_unblended_cost") "unblended_cost" - , "sum"("reservation_effective_cost") "reservation_effective_cost" - , "sum"("savings_plan_savings_plan_effective_cost") "savings_plan_effective_cost" - , "sum"("line_item_usage_amount") "usage_quantity" - FROM - "${cur_table_name}" - WHERE (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))) - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 \ No newline at end of file +CREATE OR REPLACE VIEW resource_view AS + SELECT DISTINCT + "date_trunc"('day', "line_item_usage_start_date") "usage_date" + , "bill_payer_account_id" "payer_account_id" + , "line_item_usage_account_id" "linked_account_id" + , "bill_billing_entity" "billing_entity" + , "product_product_name" "product_name" + , "line_item_resource_id" "resource_id" + , "line_item_product_code" "product_code" + , "line_item_operation" "operation" + , "line_item_line_item_type" "charge_type" + , "line_item_usage_type" "usage_type" + , "pricing_unit" "pricing_unit" + , "product_region" "region" + , "line_item_line_item_description" "item_description" + , "line_item_legal_entity" "legal_entity" + , "pricing_term" "pricing_term" + , "product_database_engine" "database_engine" + , "product_deployment_option" "product_deployment_option" + , "product_from_location" "product_from_location" + , "product_group" "product_group" + , "product_instance_type" "instance_type" + , "product_instance_type_family" "instance_type_family" + , "product_operating_system" "platform" + , "product_product_family" "product_family" + , "product_servicecode" "service" + , "product_storage" "product_storage" + , "product_to_location" "product_to_location" + , "product_volume_api_name" "product_volume_api_name" + , "reservation_reservation_a_r_n" "reservation_a_r_n" + , "savings_plan_savings_plan_a_r_n" "savings_plan_a_r_n" + , "sum"("savings_plan_savings_plan_effective_cost") savings_plan_effective_cost + , "sum"("reservation_effective_cost") reservation_effective_cost + , "sum"("line_item_usage_amount") "usage_quantity" + , "sum"("line_item_unblended_cost") unblended_cost + FROM + "${cur_table_name}" + WHERE (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) AND (line_item_resource_id <> '')) + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29 \ No newline at end of file diff --git a/cid/builtin/core/data/resources.yaml b/cid/builtin/core/data/resources.yaml index 7ec6cd9e..e63019ae 100644 --- a/cid/builtin/core/data/resources.yaml +++ b/cid/builtin/core/data/resources.yaml @@ -18,7 +18,7 @@ dashboards: minTemplateDescription: "v4.75.0" CUDOSv5: name: CUDOS Dashboard v5 - templateId: cudos_dashboard_v5 + templateId: cudos_v5 dashboardId: cudos_v5 dependsOn: datasets: From e80d0d1b4e8e3e9229601798b96d8e007a0b696e Mon Sep 17 00:00:00 2001 From: Yuriy Prykhodko Date: Fri, 10 Nov 2023 15:09:09 +0100 Subject: [PATCH 3/3] Update resources.yaml --- cid/builtin/core/data/resources.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cid/builtin/core/data/resources.yaml b/cid/builtin/core/data/resources.yaml index e63019ae..eff1a33c 100644 --- a/cid/builtin/core/data/resources.yaml +++ b/cid/builtin/core/data/resources.yaml @@ -19,7 +19,7 @@ dashboards: CUDOSv5: name: CUDOS Dashboard v5 templateId: cudos_v5 - dashboardId: cudos_v5 + dashboardId: cudos-v5 dependsOn: datasets: - summary_view