Manage CUR fields (#710)

cid/builtin/core/data/resources.yaml

@@ -267,47 +267,186 @@ views:
     riFile: cid/summary_view_ri.sql
     File: cid/summary_view.sql
     dependsOn:
-      cur: true
+      cur:
+        - bill_billing_entity
+        - bill_billing_period_start_date
+        - bill_invoice_id
+        - bill_payer_account_id
+        - line_item_availability_zone
+        - line_item_legal_entity
+        - line_item_line_item_description
+        - line_item_line_item_type
+        - line_item_operation
+        - line_item_product_code
+        - line_item_resource_id
+        - line_item_unblended_cost
+        - line_item_usage_account_id
+        - line_item_usage_amount
+        - line_item_usage_start_date
+        - line_item_usage_type
+        - pricing_public_on_demand_cost
+        - pricing_unit
+        - product_current_generation
+        - product_database_engine
+        - product_from_location
+        - product_group
+        - product_operating_system
+        - product_physical_processor
+        - product_processor_features
+        - product_product_family
+        - product_product_name
+        - product_region
+        - product_servicecode
+        - product_tenancy
+        - product_to_location
+        - reservation_effective_cost
+        - reservation_reservation_a_r_n
+        - reservation_unused_amortized_upfront_fee_for_billing_period
+        - reservation_unused_recurring_fee
+        - savings_plan_amortized_upfront_commitment_for_billing_period
+        - savings_plan_savings_plan_a_r_n
+        - savings_plan_savings_plan_effective_cost
+        - savings_plan_total_commitment_to_date
+        - savings_plan_used_commitment
 
   ec2_running_cost:
     spriFile: cid/ec2_running_cost_sp_ri.sql
     spFile: cid/ec2_running_cost_sp.sql
     riFile: cid/ec2_running_cost_ri.sql
     File: cid/ec2_running_cost.sql
     dependsOn:
-      cur: true
+      cur:
+        - bill_billing_period_start_date
+        - bill_payer_account_id
+        - line_item_line_item_type
+        - line_item_unblended_cost
+        - line_item_usage_account_id
+        - line_item_usage_amount
+        - line_item_usage_start_date
+        - line_item_usage_type
+        - reservation_effective_cost
+        - reservation_reservation_a_r_n
+        - savings_plan_savings_plan_a_r_n
+        - savings_plan_savings_plan_effective_cost
 
   compute_savings_plan_eligible_spend:
     File: cid/compute_savings_plan_eligible_spend.sql
     dependsOn:
       cur: true
+        - bill_billing_period_start_date
+        - bill_payer_account_id
+        - line_item_line_item_type
+        - line_item_operation
+        - line_item_product_code
+        - line_item_unblended_cost
+        - line_item_usage_account_id
+        - line_item_usage_start_date
+        - line_item_usage_type
+        - product_servicecode
 
   s3_view:
     File: cid/s3.sql
     dependsOn:
-      cur: true
+      cur:
+        - bill_billing_period_start_date
+        - bill_payer_account_id
+        - line_item_line_item_type
+        - line_item_operation
+        - line_item_product_code
+        - line_item_resource_id
+        - line_item_unblended_cost
+        - line_item_usage_account_id
+        - line_item_usage_amount
+        - line_item_usage_start_date
+        - pricing_public_on_demand_cost
+        - pricing_unit
+        - product_region
 
   ri_sp_mapping:
     spriFile: cid/ri_sp_mapping_sp_ri.sql
     spFile: cid/ri_sp_mapping_sp.sql
     riFile: cid/ri_sp_mapping_ri.sql
     File: cid/ri_sp_mapping.sql
     dependsOn:
-      cur: true
+      cur:
+        - bill_billing_period_start_date
+        - bill_payer_account_id
+        - line_item_line_item_type
+        - pricing_lease_contract_length
+        - pricing_offering_class
+        - pricing_purchase_option
+        - reservation_reservation_a_r_n
+        - savings_plan_end_time
+        - savings_plan_savings_plan_a_r_n
+
   hourly_view:
     spriFile: cudos/hourly_view_sp_ri.sql
     spFile: cudos/hourly_view_sp.sql
     riFile: cudos/hourly_view_ri.sql
     File: cudos/hourly_view.sql
     dependsOn:
-      cur: true 
+      cur:
+        - bill_billing_period_start_date
+        - bill_payer_account_id
+        - line_item_line_item_description
+        - line_item_line_item_type
+        - line_item_operation
+        - line_item_product_code
+        - line_item_unblended_cost
+        - line_item_usage_account_id
+        - line_item_usage_amount
+        - line_item_usage_start_date
+        - line_item_usage_type
+        - pricing_term
+        - pricing_unit
+        - product_region
+        - product_servicecode
+        - reservation_effective_cost
+        - reservation_reservation_a_r_n
+        - savings_plan_savings_plan_a_r_n
+        - savings_plan_savings_plan_effective_cost
+
   resource_view:
     spriFile: cudos/resource_view_sp_ri.sql
     spFile: cudos/resource_view_sp.sql
     riFile: cudos/resource_view_ri.sql
     File: cudos/resource_view.sql
     dependsOn:
-      cur: true 
+      cur:
+        - bill_billing_entity
+        - bill_payer_account_id
+        - line_item_legal_entity
+        - line_item_line_item_description
+        - line_item_line_item_type
+        - line_item_operation
+        - line_item_product_code
+        - line_item_resource_id
+        - line_item_unblended_cost
+        - line_item_usage_account_id
+        - line_item_usage_amount
+        - line_item_usage_start_date
+        - line_item_usage_type
+        - pricing_term
+        - pricing_unit
+        - product_database_engine
+        - product_deployment_option
+        - product_from_location
+        - product_group
+        - product_instance_type
+        - product_instance_type_family
+        - product_operating_system
+        - product_product_family
+        - product_product_name
+        - product_region
+        - product_servicecode
+        - product_storage
+        - product_to_location
+        - product_volume_api_name
+        - reservation_effective_cost
+        - reservation_reservation_a_r_n
+        - savings_plan_savings_plan_a_r_n
+        - savings_plan_savings_plan_effective_cost
+
   # Trends
   daily_anomaly_detection:
     File: trends/daily_anomaly_detection.sql

cid/common.py

@@ -125,17 +125,18 @@ def cur(self) -> CUR:
         if not self._clients.get('cur'):
             _cur = CUR(self.base.session)
             _cur.athena = self.athena
+            _cur.glue = self.glue
             print('Checking if CUR is enabled and available...')
 
-            if not _cur.configured:
+            if not _cur.metadata:
                 raise CidCritical("Error: please ensure CUR is enabled, if yes allow it some time to propagate")
 
-            print(f'\tAthena table: {_cur.tableName}')
-            print(f"\tResource IDs: {'yes' if _cur.hasResourceIDs else 'no'}")
-            if not _cur.hasResourceIDs:
+            print(f'\tAthena table: {_cur.table_name}')
+            print(f"\tResource IDs: {'yes' if _cur.has_resource_ids else 'no'}")
+            if not _cur.has_resource_ids:
                 raise CidCritical("Error: CUR has to be created with Resource IDs")
-            print(f"\tSavingsPlans: {'yes' if _cur.hasSavingsPlans else 'no'}")
-            print(f"\tReserved Instances: {'yes' if _cur.hasReservations else 'no'}")
+            print(f"\tSavingsPlans: {'yes' if _cur.has_savings_plans else 'no'}")
+            print(f"\tReserved Instances: {'yes' if _cur.has_reservations else 'no'}")
             print('\n')
             self._clients.update({
                 'cur': _cur
@@ -1360,7 +1361,7 @@ def create_or_update_dataset(self, dataset_definition: dict, dataset_id: str=Non
 
         # Check for required views
         _views = dataset_definition.get('dependsOn', {}).get('views', [])
-        required_views = [(self.cur.tableName if cur_required and name =='${cur_table_name}' else name) for name in _views]
+        required_views = [(self.cur.table_name if cur_required and name =='${cur_table_name}' else name) for name in _views]
 
         self.athena.discover_views(required_views)
         found_views = utils.intersection(required_views, self.athena._metadata.keys())
@@ -1369,7 +1370,7 @@ def create_or_update_dataset(self, dataset_definition: dict, dataset_id: str=Non
         if recursive:
             print(f"Detected views: {', '.join(found_views)}")
             for view_name in found_views:
-                if cur_required and view_name == self.cur.tableName:
+                if cur_required and view_name == self.cur.table_name:
                     logger.debug(f'Dependancy view {view_name} is a CUR. Skip.')
                     continue
                 if view_name == 'account_map':
@@ -1388,7 +1389,7 @@ def create_or_update_dataset(self, dataset_definition: dict, dataset_id: str=Non
         columns_tpl = {
             'athena_datasource_arn': athena_datasource.arn,
             'athena_database_name': self.athena.DatabaseName,
-            'cur_table_name': self.cur.tableName if cur_required else None
+            'cur_table_name': self.cur.table_name if cur_required else None
         }
 
         logger.debug(f'dataset_id={dataset_id}')
@@ -1472,12 +1473,13 @@ def create_or_update_dataset(self, dataset_definition: dict, dataset_id: str=Non
 
 
     def create_or_update_view(self, view_name: str, recursive: bool=True, update: bool=False) -> None:
-        # For account mappings create a view using a special helper
-        if view_name in self._visited_views: # avoid checking a views multiple times in one cid session
+        # Avoid checking a views multiple times in one cid session
+        if view_name in self._visited_views:
             return
-        logger.info(f'Processing view: {view_name}')
         self._visited_views.append(view_name)
+        logger.info(f'Processing view: {view_name}')
 
+        # For account mappings create a view using a special helper
         if view_name in ['account_map', 'aws_accounts']:
             if view_name in self.athena._metadata.keys():
                 print(f'Account map {view_name} exists. Skipping.')
@@ -1495,10 +1497,20 @@ def create_or_update_view(self, view_name: str, recursive: bool=True, update: bo
             logger.info(f"Definition is unavailable {view_name}")
             return
         logger.debug(f'View definition: {view_definition}')
+        dependencies = view_definition.get('dependsOn', {})
+
+        # Process CUR columns
+        if isinstance(dependencies.get('cur'), list):
+            for column in dependencies.get('cur'):
+                self.cur.ensure_column(column)
+        elif isinstance(dependencies.get('cur'), dict):
+            for column, column_type in dependencies.get('cur').items():
+                self.cur.ensure_column(column, column_type)
 
         if recursive:
-            dependency_views = view_definition.get('dependsOn', dict()).get('views', list())
-            if 'cur' in dependency_views: dependency_views.remove('cur')
+            dependency_views = dependencies.get('views', [])
+            if 'cur' in dependency_views:
+                dependency_views.remove('cur')
             # Discover dependency views (may not be discovered earlier)
             self.athena.discover_views(dependency_views)
             logger.info(f"Dependency views: {', '.join(dependency_views)}" if dependency_views else 'No dependency views')
@@ -1588,11 +1600,11 @@ def get_view_query(self, view_name: str) -> str:
         # View path
         view_definition = self.get_definition("view", name=view_name)
         cur_required = view_definition.get('dependsOn', dict()).get('cur')
-        if cur_required and self.cur.hasSavingsPlans and self.cur.hasReservations and view_definition.get('spriFile'):
+        if cur_required and self.cur.has_savings_plans and self.cur.has_reservations and view_definition.get('spriFile'):
             view_definition['File'] = view_definition.get('spriFile')
-        elif cur_required and self.cur.hasSavingsPlans and view_definition.get('spFile'):
+        elif cur_required and self.cur.has_savings_plans and view_definition.get('spFile'):
             view_definition['File'] = view_definition.get('spFile')
-        elif cur_required and self.cur.hasReservations and view_definition.get('riFile'):
+        elif cur_required and self.cur.has_reservations and view_definition.get('riFile'):
             view_definition['File'] = view_definition.get('riFile')
         elif view_definition.get('File') or view_definition.get('Data') or view_definition.get('data'):
             pass
@@ -1609,7 +1621,7 @@ def get_view_query(self, view_name: str) -> str:
 
         # Prepare template parameters
         columns_tpl = {
-            'cur_table_name': self.cur.tableName if cur_required else None,
+            'cur_table_name': self.cur.table_name if cur_required else None,
             'athenaTableName': view_name,
             'athena_database_name': self.athena.DatabaseName,
         }

cid/helpers/__init__.py

@@ -1,6 +1,6 @@
 from cid.helpers.athena import Athena
-from cid.helpers.cur import CUR
 from cid.helpers.glue import Glue
+from cid.helpers.cur import CUR
 from cid.helpers.diff import diff
 from cid.helpers.quicksight import QuickSight, Dashboard, Dataset, Datasource, Template
 from cid.helpers.csv2view import csv2view

cid/helpers/account_map.py

@@ -159,7 +159,7 @@ def create(self, name) -> bool:
             # Fill in TPLs
             columns_tpl = {
                 'metadata_table_name': self._athena_table_name,
-                'cur_table_name': self.cur.tableName # only for trends
+                'cur_table_name': self.cur.table_name # only for trends
             }
             for key, val in self.mappings.get(name).get(self._athena_table_name).items():
                 logger.debug(f'Mapping field {key} to {val}')
@@ -185,7 +185,7 @@ def get_dummy_account_mapping_sql(self, name) -> list:
         ).decode('utf-8'))
         columns_tpl = {
             'athena_view_name': name,
-            'cur_table_name': self.cur.tableName
+            'cur_table_name': self.cur.table_name
         }
         compiled_query = template.safe_substitute(columns_tpl)
         return compiled_query