From 56feea0a8aed608cceadab735cae3487ca4c3d4a Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Thu, 10 Oct 2019 19:45:21 +0530 Subject: [PATCH 001/107] Basic changes required for terraform latest version(0.12) support --- .../core/lib/python_terraform/__init__.py | 3 +- installer/core/terraform/utils.py | 34 +++++++++++++++++-- installer/resources/vpc/security_group.py | 14 ++++++-- installer/settings/default.local.py | 9 ++--- 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/installer/core/lib/python_terraform/__init__.py b/installer/core/lib/python_terraform/__init__.py index 28b69a2fd..506cde9ee 100644 --- a/installer/core/lib/python_terraform/__init__.py +++ b/installer/core/lib/python_terraform/__init__.py @@ -432,7 +432,8 @@ def create(self, variables): self.files.append(temp) log.debug( 'variables wrote to tempfile: {0}'.format(str(variables))) - temp.write(json.dumps(variables)) + if variables: + temp.write(json.dumps(variables)) file_name = temp.name return file_name diff --git a/installer/core/terraform/utils.py b/installer/core/terraform/utils.py index 50489208e..274f4bba6 100644 --- a/installer/core/terraform/utils.py +++ b/installer/core/terraform/utils.py @@ -163,6 +163,29 @@ def get_resource_created_status_op_file(resource_id): return _get_resource_status_file_name(resource_id, '1') + +def get_type_corrected_tags(tags): + """ + Get tags type corrected since earlier version used list and now changed to dict for terraform compatibility + + Args: + tags (List/Dict): Tags + + Returns: + type_corrected_tags (dict): Dict of tags + """ + type_corrected_tags = {} + + if isinstance(tags, list): # To make tags compatible with earlier version + for tag in tags: + for key, value in tag.items(): + type_corrected_tags[key] = value + else: + return tags + + return type_corrected_tags + + def get_system_default_resource_tags(): """ Get all tags required for resources @@ -170,7 +193,9 @@ def get_system_default_resource_tags(): Returns: tags (list): List of tags """ - return [Settings.DEFAULT_RESOURCE_TAG] + type_corrected_tags = get_type_corrected_tags(Settings.DEFAULT_RESOURCE_TAG) + + return type_corrected_tags def get_user_defined_resource_tags(): @@ -180,7 +205,9 @@ def get_user_defined_resource_tags(): Returns: tags (list): List of tags """ - return Settings.CUSTOM_RESOURCE_TAGS + type_corrected_tags = get_type_corrected_tags(Settings.CUSTOM_RESOURCE_TAGS) + + return type_corrected_tags def get_all_resource_tags(): @@ -192,5 +219,6 @@ def get_all_resource_tags(): """ default_tags = get_system_default_resource_tags() user_defined_tags = get_user_defined_resource_tags() + default_tags.update(user_defined_tags) - return default_tags + user_defined_tags + return default_tags diff --git a/installer/resources/vpc/security_group.py b/installer/resources/vpc/security_group.py index 899a0a3f5..7b9bf84b9 100644 --- a/installer/resources/vpc/security_group.py +++ b/installer/resources/vpc/security_group.py @@ -11,7 +11,12 @@ class InfraSecurityGroupResource(SecurityGroupResource): 'from_port': 0, 'to_port': 0, 'protocol': "-1", - 'cidr_blocks': Settings.get('VPC')['CIDR_BLOCKS'] + 'cidr_blocks': Settings.get('VPC')['CIDR_BLOCKS'], + 'ipv6_cidr_blocks': [], + 'prefix_list_ids': [], + 'description': "", + 'self': False, + 'security_groups': [] } ] @@ -20,6 +25,11 @@ class InfraSecurityGroupResource(SecurityGroupResource): 'from_port': 0, 'to_port': 0, 'protocol': "-1", - 'cidr_blocks': ["0.0.0.0/0"] + 'cidr_blocks': ["0.0.0.0/0"], + 'ipv6_cidr_blocks': [], + 'prefix_list_ids': [], + 'description': "", + 'self': False, + 'security_groups': [] } ] diff --git a/installer/settings/default.local.py b/installer/settings/default.local.py index d8483ad63..a742c27bc 100644 --- a/installer/settings/default.local.py +++ b/installer/settings/default.local.py @@ -8,10 +8,11 @@ # CUstom tags that can be defined by user -CUSTOM_RESOURCE_TAGS = [ - {'Application': "PacBot"}, - {'Environment': "Prod"} -] +CUSTOM_RESOURCE_TAGS = { + 'Application': "PacBot", + 'Environment': "Prod", + 'Created By': "customer-name" +} # RDS Related Configurations From 300eeafcb1cfac8924df76a670660a45e33fd8e4 Mon Sep 17 00:00:00 2001 From: Sajeer Date: Tue, 15 Oct 2019 17:55:30 +0530 Subject: [PATCH 002/107] Resource collection keys are passed as arguments to work with reinstall --- installer/core/commands/__init__.py | 4 ++-- installer/core/commands/destroy.py | 2 +- installer/core/commands/install.py | 2 +- installer/core/commands/status.py | 2 +- installer/custom/commands/redeploy.py | 2 +- installer/custom/commands/reinstall.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/installer/core/commands/__init__.py b/installer/core/commands/__init__.py index 5d1a77798..c9c2a9310 100644 --- a/installer/core/commands/__init__.py +++ b/installer/core/commands/__init__.py @@ -51,7 +51,7 @@ def get_complete_resources(self, input_instance): return resources_to_process - def get_resources_to_process(self, input_instance): + def get_resources_to_process(self, resource_tags_list, input_instance): """ This returns the resources to be processed currently. This can either be full resources or part of resources @@ -61,7 +61,7 @@ def get_resources_to_process(self, input_instance): Returns: resources_to_process (list): List of resources """ - resource_keys_to_process = self.get_resource_keys_to_process(self.resource_tags_list, self.category_field_name) + resource_keys_to_process = self.get_resource_keys_to_process(resource_tags_list, self.category_field_name) resources_to_process = self.get_resources_from_the_keys(resource_keys_to_process, input_instance) return resources_to_process diff --git a/installer/core/commands/destroy.py b/installer/core/commands/destroy.py index 3899de7e5..aa7389f67 100644 --- a/installer/core/commands/destroy.py +++ b/installer/core/commands/destroy.py @@ -36,7 +36,7 @@ def execute(self, provider): if self.check_pre_requisites() is False: self.exit_system_with_pre_requisites_fail() - resources_to_process = self.get_resources_to_process(input_instance) + resources_to_process = self.get_resources_to_process(self.resource_tags_list, input_instance) if resources_to_process: self.destroy_class(self.args, input_instance).execute( resources_to_process, diff --git a/installer/core/commands/install.py b/installer/core/commands/install.py index 621f8f472..077f5d7ca 100644 --- a/installer/core/commands/install.py +++ b/installer/core/commands/install.py @@ -40,7 +40,7 @@ def execute(self, provider): if self.check_pre_requisites() is False: self.exit_system_with_pre_requisites_fail() - resources_to_process = self.get_resources_to_process(input_instance) + resources_to_process = self.get_resources_to_process(self.resource_tags_list, input_instance) if resources_to_process: self.install_class(self.args, input_instance).execute( resources_to_process, diff --git a/installer/core/commands/status.py b/installer/core/commands/status.py index 994fddd0a..7515f2420 100644 --- a/installer/core/commands/status.py +++ b/installer/core/commands/status.py @@ -43,7 +43,7 @@ def execute(self, provider): need_instance = False display_op_list = [] - resources = self.get_resources_to_process(input_instance) + resources = self.get_resources_to_process(self.resource_tags_list, input_instance) terraform_outputs = py_terraform.save_terraform_output() status = py_terraform.get_current_status() if not status and not terraform_outputs: diff --git a/installer/custom/commands/redeploy.py b/installer/custom/commands/redeploy.py index 14a754d80..4f081bcdf 100644 --- a/installer/custom/commands/redeploy.py +++ b/installer/custom/commands/redeploy.py @@ -85,7 +85,7 @@ def re_deploy_pacbot(self, input_instance): Args: input_instance (Input object): User input values """ - resources_to_process = self.get_resources_to_process(input_instance) + resources_to_process = self.get_resources_to_process(self.resource_tags_list, input_instance) try: resources_to_taint = self.get_resources_with_given_tags(input_instance, ["deploy"]) resources_to_taint = [resource for resource in resources_to_taint if resource.PROCESS is True] diff --git a/installer/custom/commands/reinstall.py b/installer/custom/commands/reinstall.py index e59ca1c50..706b370bc 100644 --- a/installer/custom/commands/reinstall.py +++ b/installer/custom/commands/reinstall.py @@ -63,7 +63,7 @@ def re_deploy_pacbot(self, input_instance): Args: input_instance (Input object): User input values """ - resources_to_process = self.get_resources_to_process(input_instance) + resources_to_process = self.get_resources_to_process(self.resource_tags_list, input_instance) terraform_with_targets = True self.install_class( From 33dacb6a00d8f62a02ad6ed1c2d0710bd8672ee7 Mon Sep 17 00:00:00 2001 From: Sajeer Date: Tue, 15 Oct 2019 18:10:50 +0530 Subject: [PATCH 003/107] Terraform tf extension is relaced with .tf.json --- installer/core/providers/aws/__init__.py | 2 +- installer/core/terraform/resources/__init__.py | 2 +- installer/core/terraform/utils.py | 2 +- installer/custom/commands/redeploy.py | 6 ++++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/installer/core/providers/aws/__init__.py b/installer/core/providers/aws/__init__.py index 2939b1594..65ca5f61b 100644 --- a/installer/core/providers/aws/__init__.py +++ b/installer/core/providers/aws/__init__.py @@ -82,7 +82,7 @@ def _delete_terraform_provider_file(self): def _delete_all_terraform_files(self): """"Delete all terraform files before terraform regeneration if the install is done on all resources""" for file in os.listdir(Settings.TERRAFORM_DIR): - if file.endswith(".tf"): + if file.endswith(".tf") or file.endswith(".tf.json"): file_abs_path = os.path.join(Settings.TERRAFORM_DIR, file) os.remove(file_abs_path) diff --git a/installer/core/terraform/resources/__init__.py b/installer/core/terraform/resources/__init__.py index 39273f5d6..497cd0c95 100644 --- a/installer/core/terraform/resources/__init__.py +++ b/installer/core/terraform/resources/__init__.py @@ -374,7 +374,7 @@ class TerraformResource(BaseTerraformResource, metaclass=ABCMeta): """ terraform_type = 'resource' MANDATORY_OUTPUT = 'id' - tf_file_extension = 'tf' + tf_file_extension = 'tf.json' tags = get_all_resource_tags() def check_exists_before(self, input, outputs): diff --git a/installer/core/terraform/utils.py b/installer/core/terraform/utils.py index 274f4bba6..be6f1b817 100644 --- a/installer/core/terraform/utils.py +++ b/installer/core/terraform/utils.py @@ -12,7 +12,7 @@ def get_terraform_provider_file(): """ return os.path.join( Settings.TERRAFORM_DIR, - 'provider.tf' + 'provider.tf.json' ) diff --git a/installer/custom/commands/redeploy.py b/installer/custom/commands/redeploy.py index 4f081bcdf..acefa3d9d 100644 --- a/installer/custom/commands/redeploy.py +++ b/installer/custom/commands/redeploy.py @@ -43,8 +43,10 @@ def __init__(self, args): def _need_complete_installation(self): need_complete_install = False - redshift_cluster_file = os.path.join(Settings.TERRAFORM_DIR, "datastore_redshift_RedshiftCluster.tf") - if os.path.exists(redshift_cluster_file): + redshift_cluster_file_tf = os.path.join(Settings.TERRAFORM_DIR, "datastore_redshift_RedshiftCluster.tf") + redshift_cluster_file_tf_json = os.path.join(Settings.TERRAFORM_DIR, "datastore_redshift_RedshiftCluster.tf.json") + + if os.path.exists(redshift_cluster_file) or os.path.exists(redshift_cluster_file_tf_json): need_complete_install = True return need_complete_install From a297ecbdc6fdf76d65e1033298c26470ef2ee502 Mon Sep 17 00:00:00 2001 From: Sajeer Date: Tue, 15 Oct 2019 18:55:27 +0530 Subject: [PATCH 004/107] Reinstall command modified to work with latest terraform support --- installer/core/commands/__init__.py | 1 - installer/core/providers/aws/reinstall.py | 23 ++++--- installer/custom/commands/reinstall.py | 82 +++++++++++++++++++++-- 3 files changed, 91 insertions(+), 15 deletions(-) diff --git a/installer/core/commands/__init__.py b/installer/core/commands/__init__.py index c9c2a9310..90f329bdb 100644 --- a/installer/core/commands/__init__.py +++ b/installer/core/commands/__init__.py @@ -31,7 +31,6 @@ def __init__(self, args): Args: args (List): List of key- value pair of args supplied to the command """ - self.args = args self.resource_tags_list = [v for (k, v) in args if k == self.category_field_name] if self.resource_tags_list: diff --git a/installer/core/providers/aws/reinstall.py b/installer/core/providers/aws/reinstall.py index 7fdd67926..0aaaca5fd 100644 --- a/installer/core/providers/aws/reinstall.py +++ b/installer/core/providers/aws/reinstall.py @@ -8,7 +8,7 @@ import sys -class ReInstall(Install): +class ReInstall(Install): # Do not inherit Destroy """ AWS provider for destroy command @@ -22,7 +22,7 @@ class ReInstall(Install): """ destroy = False - def run_tf_execution_and_status_threads(self, resources, terraform_with_targets, dry_run): + def run_tf_execution_and_status_threads(self, resources_to_destroy, resources_to_install, terraform_with_targets, dry_run): """ Creates 2 thread 1. For actualy installation @@ -34,8 +34,11 @@ def run_tf_execution_and_status_threads(self, resources, terraform_with_targets, terraform_with_targets (boolean): If partial install is to be done (if --tags is supplied) dry_run (boolean): Decides whether original install should be done """ - self.terraform_thread = Thread(target=self.re_create_resources, args=(list(resources), terraform_with_targets, dry_run)) - progressbar_thread = Thread(target=self.show_progress_status, args=(list(resources), terraform_with_targets, dry_run)) + self.terraform_thread = Thread( + target=self.re_create_resources, + args=(list(resources_to_destroy), list(resources_to_install), terraform_with_targets, dry_run) + ) + progressbar_thread = Thread(target=self.show_progress_status_all, args=(list(resources_to_install), terraform_with_targets, dry_run)) self.terraform_thread.start() progressbar_thread.start() @@ -43,7 +46,7 @@ def run_tf_execution_and_status_threads(self, resources, terraform_with_targets, self.terraform_thread.join() progressbar_thread.join() - def re_create_resources(self, resources, terraform_with_targets, dry_run): + def re_create_resources(self, resources_to_destroy, resources_to_install, terraform_with_targets, dry_run): """ Start installing the resources by calling PyTerraform class destroy @@ -54,9 +57,9 @@ def re_create_resources(self, resources, terraform_with_targets, dry_run): """ try: if not dry_run: - PyTerraform().terraform_destroy(resources) + PyTerraform().terraform_destroy(resources_to_destroy) self.destroy = True - self.terraform_apply(resources, terraform_with_targets, dry_run) + self.terraform_apply(resources_to_install, terraform_with_targets, dry_run) except Exception as e: self.executed_with_error = True self.exception = e @@ -64,7 +67,7 @@ def re_create_resources(self, resources, terraform_with_targets, dry_run): self._cleanup_installation_process(dry_run) - def show_progress_status(self, resources, terraform_with_targets, dry_run): + def show_progress_status_all(self, resources, terraform_with_targets, dry_run): """ Show the status of installation continously in this thread @@ -73,8 +76,8 @@ def show_progress_status(self, resources, terraform_with_targets, dry_run): terraform_with_targets (boolean): If partial install is to be done (if --tags is supplied) dry_run (boolean): Decides whether original install should be done """ - self.render_terraform_destroy_progress() - super().show_progress_status(resources, terraform_with_targets, dry_run) + self.render_terraform_destroy_progress() # Show destroy progress + self.show_progress_status(resources, terraform_with_targets, dry_run) # Show install progress def render_terraform_destroy_progress(self): """Show the status of terraform init command execution""" diff --git a/installer/custom/commands/reinstall.py b/installer/custom/commands/reinstall.py index 706b370bc..8502e658a 100644 --- a/installer/custom/commands/reinstall.py +++ b/installer/custom/commands/reinstall.py @@ -17,15 +17,38 @@ class Reinstall(BaseCommand): install_class (class): Provider based install class """ def __init__(self, args): + + Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True) + + + args.append((K.CATEGORY_FIELD_NAME, "deploy")) + args.append((K.CATEGORY_FIELD_NAME, "upload_tf")) + self.destroy_resource_tags_list = [v for (k, v) in args if k == self.category_field_name] + args.append((K.CATEGORY_FIELD_NAME, "deploy")) + args.append((K.CATEGORY_FIELD_NAME, "roles")) + args.append((K.CATEGORY_FIELD_NAME, "all_read_role")) args.append((K.CATEGORY_FIELD_NAME, "batch-ecr")) args.append((K.CATEGORY_FIELD_NAME, "batch-job")) args.append((K.CATEGORY_FIELD_NAME, "submit-job")) args.append((K.CATEGORY_FIELD_NAME, "rule-engine-job")) args.append((K.CATEGORY_FIELD_NAME, "upload_tf")) + self.reinstall_resource_tags_list = [v for (k, v) in args if k == self.category_field_name] - Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True) - super().__init__(args) + self.need_complete_install = self._need_complete_installation() + + self.dry_run = True if any([x[1] for x in args if x[0] == "dry-run"]) else self.dry_run + + def _need_complete_installation(self): + need_complete_install = False + + redshift_cluster_file_tf = os.path.join(Settings.TERRAFORM_DIR, "datastore_redshift_RedshiftCluster.tf") + redshift_cluster_file_tf_json = os.path.join(Settings.TERRAFORM_DIR, "datastore_redshift_RedshiftCluster.tf.json") + + if os.path.exists(redshift_cluster_file) or os.path.exists(redshift_cluster_file_tf_json): + need_complete_install = True + + return need_complete_install def execute(self, provider): """ @@ -63,9 +86,60 @@ def re_deploy_pacbot(self, input_instance): Args: input_instance (Input object): User input values """ - resources_to_process = self.get_resources_to_process(self.resource_tags_list, input_instance) - terraform_with_targets = True + resources_to_destroy = self.destroy_resource_tags_list(self.destroy_resource_tags_list, input_instance) + resources_to_install = self.destroy_resource_tags_list(self.reinstall_resource_tags_list, input_instance) + + try: + resources_to_taint = self.get_resources_with_given_tags(input_instance, ["deploy"]) + resources_to_taint = [resource for resource in resources_to_taint if resource.PROCESS is True] + response = PyTerraform().terraform_taint(resources_to_taint) # If tainted or destroyed already then skip it + except Exception as e: + pass + terraform_with_targets = False if self.need_complete_install else True + resources_to_install = self.get_complete_resources(input_instance) if self.need_complete_install else resources_to_install + + # self.run_pre_deployment_process(resources_to_process) + self.run_real_deployment(input_instance, resources_to_process, terraform_with_targets) + + def run_real_deployment(self, input_instance, resources_to_process, terraform_with_targets): + """ + Main thread method which invokes the 2 thread: one for actual execution and another for displaying status + + Args: + input_instance (Input obj): Input object with values read from user + resources_to_process (list): List of resources to be created/updated + terraform_with_targets (boolean): This is True since redeployment is happening + """ + self.terraform_thread = Thread(target=self.run_tf_apply, args=(input_instance, list(resources_to_process), terraform_with_targets)) + # Dt-run variable is passed as it is rquired otherwise argument parsing issue will occur + stop_related_task_thread = Thread(target=self.inactivate_required_services_for_redeploy, args=(list(resources_to_process), self.dry_run)) + + self.terraform_thread.start() + stop_related_task_thread.start() + + self.terraform_thread.join() + stop_related_task_thread.join() + + def inactivate_required_services_for_redeploy(self, resources_to_process, dry_run): + """ + Before redeploy get started or on redeploy happens stop the tasks and deregister task definition + + Args: + resources_to_process (list): List of resources to be created/updated + only_tasks (boolean): This flasg decides whther to deregister task definition or not + """ + pass + + def run_tf_apply(self, input_instance, resources_to_process, terraform_with_targets): + """ + Execute the installation of resources by invoking the execute method of provider class + + Args: + input_instance (Input obj): Input object with values read from user + resources_to_process (list): List of resources to be created/updated + terraform_with_targets (boolean): This is True since redeployment is happening + """ self.install_class( self.args, input_instance, From b2bdd11b219b644e11e4622f2f0629db6c8c5b9b Mon Sep 17 00:00:00 2001 From: Sajeer Date: Tue, 15 Oct 2019 19:22:17 +0530 Subject: [PATCH 005/107] Tf file extension corected and reinstall command functions corrected --- installer/core/providers/aws/reinstall.py | 17 +++++++++++ .../core/terraform/resources/__init__.py | 2 +- installer/custom/commands/reinstall.py | 30 +++++++++++-------- 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/installer/core/providers/aws/reinstall.py b/installer/core/providers/aws/reinstall.py index 0aaaca5fd..13801557a 100644 --- a/installer/core/providers/aws/reinstall.py +++ b/installer/core/providers/aws/reinstall.py @@ -22,6 +22,23 @@ class ReInstall(Install): # Do not inherit Destroy """ destroy = False + def execute(self, resources_to_destroy, resources_to_install, terraform_with_targets, dry_run): + """ + This is the starting method where install begins. This is the actual method called from the main install class + + Args: + resources (list): Resources to be installed + terraform_with_targets (boolean): If partial install is to be done (if --tags is supplied) + dry_run (boolean): Decides whether original install should be done + """ + self.generate_terraform_files(resources_to_install, terraform_with_targets) + self.run_tf_execution_and_status_threads(resources_to_destroy, resources_to_install, terraform_with_targets, dry_run) + + if not self.executed_with_error: + self.render_resource_outputs(resources) + else: + raise self.exception + def run_tf_execution_and_status_threads(self, resources_to_destroy, resources_to_install, terraform_with_targets, dry_run): """ Creates 2 thread diff --git a/installer/core/terraform/resources/__init__.py b/installer/core/terraform/resources/__init__.py index 497cd0c95..9048ae317 100644 --- a/installer/core/terraform/resources/__init__.py +++ b/installer/core/terraform/resources/__init__.py @@ -440,7 +440,7 @@ class TerraformData(BaseTerraformResource, metaclass=ABCMeta): tf_file_extension (str): File extension for the terraform file """ terraform_type = 'data' - tf_file_extension = 'tf' + tf_file_extension = 'tf.json' @classmethod def get_output_attr(cls, key): diff --git a/installer/custom/commands/reinstall.py b/installer/custom/commands/reinstall.py index 8502e658a..3ae853603 100644 --- a/installer/custom/commands/reinstall.py +++ b/installer/custom/commands/reinstall.py @@ -1,6 +1,7 @@ from core.commands import BaseCommand from core.config import Settings from core import constants as K +from threading import Thread import time import importlib import sys @@ -45,7 +46,7 @@ def _need_complete_installation(self): redshift_cluster_file_tf = os.path.join(Settings.TERRAFORM_DIR, "datastore_redshift_RedshiftCluster.tf") redshift_cluster_file_tf_json = os.path.join(Settings.TERRAFORM_DIR, "datastore_redshift_RedshiftCluster.tf.json") - if os.path.exists(redshift_cluster_file) or os.path.exists(redshift_cluster_file_tf_json): + if os.path.exists(redshift_cluster_file_tf) or os.path.exists(redshift_cluster_file_tf_json): need_complete_install = True return need_complete_install @@ -86,8 +87,8 @@ def re_deploy_pacbot(self, input_instance): Args: input_instance (Input object): User input values """ - resources_to_destroy = self.destroy_resource_tags_list(self.destroy_resource_tags_list, input_instance) - resources_to_install = self.destroy_resource_tags_list(self.reinstall_resource_tags_list, input_instance) + resources_to_destroy = self.get_resources_to_process(self.destroy_resource_tags_list, input_instance) + resources_to_install = self.get_resources_to_process(self.reinstall_resource_tags_list, input_instance) try: resources_to_taint = self.get_resources_with_given_tags(input_instance, ["deploy"]) @@ -100,9 +101,9 @@ def re_deploy_pacbot(self, input_instance): resources_to_install = self.get_complete_resources(input_instance) if self.need_complete_install else resources_to_install # self.run_pre_deployment_process(resources_to_process) - self.run_real_deployment(input_instance, resources_to_process, terraform_with_targets) + self.run_real_deployment(input_instance, resources_to_destroy, resources_to_install, terraform_with_targets) - def run_real_deployment(self, input_instance, resources_to_process, terraform_with_targets): + def run_real_deployment(self, input_instance, resources_to_destroy, resources_to_install, terraform_with_targets): """ Main thread method which invokes the 2 thread: one for actual execution and another for displaying status @@ -111,9 +112,13 @@ def run_real_deployment(self, input_instance, resources_to_process, terraform_wi resources_to_process (list): List of resources to be created/updated terraform_with_targets (boolean): This is True since redeployment is happening """ - self.terraform_thread = Thread(target=self.run_tf_apply, args=(input_instance, list(resources_to_process), terraform_with_targets)) + self.terraform_thread = Thread( + target=self.run_tf_apply, + args=(input_instance, list(resources_to_destroy), list(resources_to_install), terraform_with_targets)) # Dt-run variable is passed as it is rquired otherwise argument parsing issue will occur - stop_related_task_thread = Thread(target=self.inactivate_required_services_for_redeploy, args=(list(resources_to_process), self.dry_run)) + stop_related_task_thread = Thread( + target=self.inactivate_required_services_for_redeploy, + args=(list(resources_to_destroy), list(resources_to_install), self.dry_run)) self.terraform_thread.start() stop_related_task_thread.start() @@ -121,7 +126,7 @@ def run_real_deployment(self, input_instance, resources_to_process, terraform_wi self.terraform_thread.join() stop_related_task_thread.join() - def inactivate_required_services_for_redeploy(self, resources_to_process, dry_run): + def inactivate_required_services_for_redeploy(self, resources_to_destroy, resources_to_install, dry_run): """ Before redeploy get started or on redeploy happens stop the tasks and deregister task definition @@ -130,8 +135,8 @@ def inactivate_required_services_for_redeploy(self, resources_to_process, dry_ru only_tasks (boolean): This flasg decides whther to deregister task definition or not """ pass - - def run_tf_apply(self, input_instance, resources_to_process, terraform_with_targets): + + def run_tf_apply(self, input_instance, resources_to_destroy, resources_to_install, terraform_with_targets): """ Execute the installation of resources by invoking the execute method of provider class @@ -141,11 +146,12 @@ def run_tf_apply(self, input_instance, resources_to_process, terraform_with_targ terraform_with_targets (boolean): This is True since redeployment is happening """ self.install_class( - self.args, + [], input_instance, check_dependent_resources=False ).execute( - resources_to_process, + resources_to_destroy, + resources_to_install, terraform_with_targets, self.dry_run ) From dd2106765a21f9eddb4e6cced579627358bd8b2d Mon Sep 17 00:00:00 2001 From: johnrexj Date: Fri, 18 Oct 2019 14:53:15 +0530 Subject: [PATCH 006/107] Azure changes for batch commons --- commons/pac-batch-commons/pom.xml | 6 +- .../pacman/commons/PacmanSdkConstants.java | 6 ++ .../azure/clients/AzureCredentialManager.java | 85 ++++++++++++++++ .../sqlserver/CloudInsightSqlServer.java | 92 +++++++++++++++++ .../pacman/commons/utils/CommonUtils.java | 99 +++++++++++++++++++ 5 files changed, 287 insertions(+), 1 deletion(-) create mode 100644 commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/azure/clients/AzureCredentialManager.java create mode 100644 commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/azure/sqlserver/CloudInsightSqlServer.java diff --git a/commons/pac-batch-commons/pom.xml b/commons/pac-batch-commons/pom.xml index 933b15554..7f2cae5e5 100644 --- a/commons/pac-batch-commons/pom.xml +++ b/commons/pac-batch-commons/pom.xml @@ -33,7 +33,11 @@ - + + com.microsoft.azure + azure + 1.22.0 + com.amazonaws diff --git a/commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/PacmanSdkConstants.java b/commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/PacmanSdkConstants.java index 2e848d235..dba38ab7c 100644 --- a/commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/PacmanSdkConstants.java +++ b/commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/PacmanSdkConstants.java @@ -291,4 +291,10 @@ public interface PacmanSdkConstants { String SOURCE = "source"; String TAGGING_MANDATORY_TAGS = "tagging.mandatoryTags"; + + String CLOUD_INSIGHT_SQL_SERVER = "CLOUD_INSIGHT_SQL_SERVER"; + + String CLOUD_INSIGHT_USER = "CLOUD_INSIGHT_USER"; + + String CLOUD_INSIGHT_PASSWORD = "CLOUD_INSIGHT_PASSWORD"; } diff --git a/commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/azure/clients/AzureCredentialManager.java b/commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/azure/clients/AzureCredentialManager.java new file mode 100644 index 000000000..b660ff38a --- /dev/null +++ b/commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/azure/clients/AzureCredentialManager.java @@ -0,0 +1,85 @@ +package com.tmobile.pacman.commons.azure.clients; + +import java.util.HashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import com.microsoft.azure.AzureEnvironment; +import com.microsoft.azure.credentials.ApplicationTokenCredentials; +import com.microsoft.azure.management.Azure; +import com.tmobile.pacman.commons.utils.CommonUtils; + +public class AzureCredentialManager { + + /** The Constant logger. */ + static final Logger logger = LoggerFactory.getLogger(AzureCredentialManager.class); + + public static Azure authenticate(String subscription) { + return Azure.authenticate(getCredentials()).withSubscription(subscription); + } + + public static String getAuthToken() throws Exception { + String url = "https://login.microsoftonline.com/%s/oauth2/token"; + + String clientId = System.getProperty("azure.clientId"); + String domain = System.getProperty("azure.domain"); + String secret = System.getProperty("azure.secret"); + + + Map params = new HashMap<>(); + params.put("client_id", clientId); + params.put("client_secret", secret); + params.put("resource", "https://management.azure.com"); + params.put("grant_type", "client_credentials"); + url = String.format(url, domain); + + try { + String jsonResponse = CommonUtils.doHttpPost(url, params); + Map respMap = new Gson().fromJson(jsonResponse, new TypeToken>() {}.getType() ); + return respMap.get("access_token"); + } catch (Exception e) { + logger.error("Error getting mangement API token from Azure",e); + throw e; + } + } + + public static String getGraphApiAuthToken() throws Exception { + String url = "https://login.microsoftonline.com/%s/oauth2/v2.0/token"; + + String clientId = System.getProperty("azure.clientId"); + String domain = System.getProperty("azure.domain"); + String secret = System.getProperty("azure.secret"); + + Map params = new HashMap<>(); + params.put("client_id", clientId); + params.put("client_secret", secret); + params.put("scope", "https://graph.microsoft.com/.default"); + params.put("grant_type", "client_credentials"); + url = String.format(url, domain); + + try { + String jsonResponse = CommonUtils.doHttpPost(url, params); + Map respMap = new Gson().fromJson(jsonResponse, new TypeToken>() {}.getType() ); + return respMap.get("access_token"); + } catch (Exception e) { + logger.error("Error getting Grpah API token from Azure",e); + throw e; + } + + } + + + private static ApplicationTokenCredentials getCredentials(){ + String clientId = System.getProperty("azure.clientId"); + String domain = System.getProperty("azure.domain"); + String secret = System.getProperty("azure.secret"); + return new ApplicationTokenCredentials(clientId, + domain, secret, AzureEnvironment.AZURE); + } + + +} diff --git a/commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/azure/sqlserver/CloudInsightSqlServer.java b/commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/azure/sqlserver/CloudInsightSqlServer.java new file mode 100644 index 000000000..8218ae586 --- /dev/null +++ b/commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/azure/sqlserver/CloudInsightSqlServer.java @@ -0,0 +1,92 @@ +package com.tmobile.pacman.commons.azure.sqlserver; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.util.StringUtils; +import com.tmobile.pacman.commons.PacmanSdkConstants; +import com.tmobile.pacman.commons.utils.CommonUtils; + +/** + * @author Raghavendra + * + */ +public class CloudInsightSqlServer { + + private static final Logger logger = LoggerFactory.getLogger(CloudInsightSqlServer.class); + + public static Connection getDBConnection() throws SQLException { + String hostName = getClouldInsightSqlServer(); + String dbName = "cloudinsightbillingdb"; + String user = getClouldInsightUser(); + String password = getClouldInsightPassWord(); + + if (StringUtils.isNullOrEmpty(hostName) || StringUtils.isNullOrEmpty(user) + || StringUtils.isNullOrEmpty(password)) { + throw new RuntimeException( + " Cloud insight server mandatory configuration CLOUD_INSIGHT_SQL_SERVER/CLOUD_INSIGHT_USER/CLOUD_INSIGHT_PASSWORD "); + } + String url = String.format( + "jdbc:sqlserver://%s:1433;database=%s;user=%s;password=%s;encrypt=true;" + + "hostNameInCertificate=*.database.windows.net;loginTimeout=30;", + hostName, dbName, user, password); + Connection connection = null; + + connection = DriverManager.getConnection(url); + return connection; + } + + public static String getClouldInsightSqlServer() { + return CommonUtils.getEnvVariableValue(PacmanSdkConstants.CLOUD_INSIGHT_SQL_SERVER); + } + + public static String getClouldInsightUser() { + return CommonUtils.getEnvVariableValue(PacmanSdkConstants.CLOUD_INSIGHT_USER); + } + + public static String getClouldInsightPassWord() { + return CommonUtils.getEnvVariableValue(PacmanSdkConstants.CLOUD_INSIGHT_PASSWORD); + } + + /** + * @param appTag + * @return + */ + public static String getValidAppTag(String appTag) { + Connection connection = null; + try { + connection = getDBConnection(); + } catch (SQLException ex) { + logger.error("exception while getting connection ", ex); + return null; + } + String validAppTag = null; + String userAppTag = null; + String selectSql = "SELECT * FROM DimAppAlias where UserApplication='" + appTag + "'"; + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(selectSql)) { + if (resultSet.next()) { + userAppTag = resultSet.getString(1); + validAppTag = resultSet.getString(2); + } + if (userAppTag != null && userAppTag.equals(appTag)) { + logger.debug("apptag is not valid current tag : {} correct tag : {}",appTag,validAppTag); + return validAppTag; + } + } catch (Exception e) { + logger.error("exception while executing query ", e); + } finally { + try { + connection.close(); + } catch (SQLException e) { + } + } + return null; + } +} diff --git a/commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/utils/CommonUtils.java b/commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/utils/CommonUtils.java index c53ea9bd8..a6bbf309c 100644 --- a/commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/utils/CommonUtils.java +++ b/commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/utils/CommonUtils.java @@ -25,22 +25,28 @@ import java.io.IOException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import org.apache.http.Consts; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; +import org.apache.http.NameValuePair; import org.apache.http.ParseException; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.HttpClient; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -333,4 +339,97 @@ public static String getJsonString(final Object annotation) { } return null; } + + /** + * Do http post. + * + * @param url the url + * @param requestBody the request body + * @return String + * @throws Exception the exception + */ + public static String doHttpPost(final String url, final Map requestBody) throws Exception { + try { + + HttpClient client = HttpClientBuilder.create().build(); + HttpPost httppost = new HttpPost(url); + + List form = new ArrayList<>(); + requestBody.forEach((k,v)-> { + form.add(new BasicNameValuePair(k,v)); + }); + UrlEncodedFormEntity entity = new UrlEncodedFormEntity(form, Consts.UTF_8); + httppost.setEntity(entity); + + HttpResponse httpresponse = client.execute(httppost); + int statusCode = httpresponse.getStatusLine().getStatusCode(); + if(statusCode==HttpStatus.SC_OK || statusCode==HttpStatus.SC_CREATED) + { + return EntityUtils.toString(httpresponse.getEntity()); + }else{ + logger.error(httpresponse.getStatusLine().getStatusCode() + "---" + httpresponse.getStatusLine().getReasonPhrase()); + throw new Exception("unable to execute post request because " + httpresponse.getStatusLine().getReasonPhrase()); + } + } catch (ParseException parseException) { + logger.error("ParseException in getHttpPost :"+parseException.getMessage()); + throw parseException; + } catch (Exception exception) { + logger.error("Exception in getHttpPost :"+exception.getMessage()); + throw exception; + } + } + + public static String doHttpGet(String uri ,String tokeType, String token) throws Exception { + + HttpGet httpGet = new HttpGet(uri); + httpGet.addHeader("content-type", "application/json"); + httpGet.addHeader("cache-control", "no-cache"); + if(!Strings.isNullOrEmpty(token)){ + httpGet.addHeader("Authorization", tokeType+" "+token); + } + HttpClient httpClient = HttpClientBuilder.create().build(); + if(httpClient!=null){ + HttpResponse httpResponse; + try { + + httpResponse = httpClient.execute(httpGet); + if( httpResponse.getStatusLine().getStatusCode()==HttpStatus.SC_OK){ + return EntityUtils.toString(httpResponse.getEntity()); + }else { + throw new Exception("unable to execute put request caused by"+EntityUtils.toString(httpResponse.getEntity())); + } + } catch (Exception e) { + logger.error("Error getting the data " , e); + throw e; + } + } + return "{}"; + } + + public static String doHttpPost(String uri, String token, String accessToken) throws Exception { + + HttpPost httpPost = new HttpPost(uri); + httpPost.addHeader("content-type", "application/json"); + httpPost.addHeader("cache-control", "no-cache"); + if (!Strings.isNullOrEmpty(token)) { + httpPost.addHeader("Authorization", token + " " + accessToken); + } + HttpClient httpClient = HttpClientBuilder.create().build(); + if (httpClient != null) { + HttpResponse httpResponse; + try { + httpResponse = httpClient.execute(httpPost); + if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + return EntityUtils.toString(httpResponse.getEntity()); + } else { + throw new Exception("unable to execute post request caused by" + + EntityUtils.toString(httpResponse.getEntity())); + } + } catch (Exception e) { + logger.error("Error getting the data ", e); + throw e; + } + } + return "{}"; + } } From 286846e1703e7c7c1f67c692a50c63b3b1a1b0f1 Mon Sep 17 00:00:00 2001 From: johnrexj Date: Fri, 18 Oct 2019 14:54:11 +0530 Subject: [PATCH 007/107] Azure discovery job --- .../inventory/AzureDiscoveryApplication.java | 21 + .../azure/inventory/AzureDiscoveryJob.java | 69 ++ .../inventory/AzureFetchOrchestrator.java | 96 +++ .../azure/inventory/ErrorManageUtil.java | 136 ++++ .../azure/inventory/InventoryConstants.java | 28 + .../inventory/auth/AWSCredentialProvider.java | 80 +++ .../BatchAccountInventoryCollector.java | 89 +++ .../BlobContainerInventoryCollector.java | 78 +++ .../collector/CosmosDBInventoryCollector.java | 74 +++ .../DatabricksInventoryCollector.java | 71 +++ .../collector/DiskInventoryCollector.java | 43 ++ .../LoadBalancerInventoryCollector.java | 47 ++ .../collector/MariaDBInventoryCollector.java | 71 +++ .../collector/MySQLInventoryCollector.java | 74 +++ .../collector/NSGInventoryCollector.java | 96 +++ .../NamespaceInventoryCollector.java | 85 +++ .../NetworkInterfaceInventoryCollector.java | 76 +++ .../collector/NetworkInventoryCollector.java | 47 ++ .../PolicyDefinitionInventoryCollector.java | 38 ++ .../PolicyStatesInventoryCollector.java | 100 +++ .../PostgreSQLInventoryCollector.java | 70 +++ .../PublicIpAddressInventoryCollector.java | 51 ++ ...gisteredApplicationInventoryCollector.java | 161 +++++ .../ResourceGroupInventoryCollector.java | 39 ++ .../RouteTableInventoryCollector.java | 79 +++ .../collector/SCRecommendationsCollector.java | 93 +++ .../SQLDatabaseInventoryCollector.java | 140 +++++ .../SQLServerInventoryCollector.java | 121 ++++ .../SearchServiceInventoryCollector.java | 79 +++ .../SecurityAlertsInventoryCollector.java | 67 ++ .../collector/SitesInventoryCollector.java | 78 +++ .../collector/SnapshotInventoryCollector.java | 44 ++ .../StorageAccountInventoryCollector.java | 66 ++ .../collector/SubnetInventoryCollector.java | 84 +++ .../azure/inventory/collector/Util.java | 162 +++++ .../collector/VMInventoryCollector.java | 226 +++++++ .../collector/VaultInventoryCollector.java | 86 +++ .../collector/WorkflowInventoryCollector.java | 79 +++ .../azure/inventory/config/ConfigUtil.java | 68 ++ .../inventory/file/AssetFileGenerator.java | 595 ++++++++++++++++++ .../azure/inventory/file/FileGenerator.java | 127 ++++ .../azure/inventory/file/FileManager.java | 297 +++++++++ .../azure/inventory/file/S3Uploader.java | 193 ++++++ .../pacbot/azure/inventory/vo/AzureVH.java | 71 +++ .../azure/inventory/vo/BatchAccountVH.java | 143 +++++ .../azure/inventory/vo/BlobContainerVH.java | 57 ++ .../pacbot/azure/inventory/vo/CosmosDBVH.java | 74 +++ .../pacbot/azure/inventory/vo/DataDiskVH.java | 84 +++ .../azure/inventory/vo/DatabricksVH.java | 57 ++ .../azure/inventory/vo/ElasticPoolVH.java | 68 ++ .../pacbot/azure/inventory/vo/ErrorVH.java | 98 +++ .../azure/inventory/vo/FailoverGroupVH.java | 68 ++ .../azure/inventory/vo/FirewallRules.java | 69 ++ .../azure/inventory/vo/IPconfigurationVH.java | 87 +++ .../azure/inventory/vo/LoadBalancerVH.java | 118 ++++ .../pacbot/azure/inventory/vo/MariaDBVH.java | 53 ++ .../azure/inventory/vo/MySQLServerVH.java | 54 ++ .../azure/inventory/vo/NIIPConfigVH.java | 56 ++ .../azure/inventory/vo/NSGSecurityRule.java | 141 +++++ .../pacbot/azure/inventory/vo/NSGSubnet.java | 34 + .../azure/inventory/vo/NamespaceVH.java | 71 +++ .../inventory/vo/NetworkInterfaceVH.java | 157 +++++ .../pacbot/azure/inventory/vo/NetworkVH.java | 104 +++ .../inventory/vo/PolicyDefinitionVH.java | 51 ++ .../azure/inventory/vo/PolicyStatesVH.java | 266 ++++++++ .../inventory/vo/PostgreSQLServerVH.java | 53 ++ .../azure/inventory/vo/PublicIpAddressVH.java | 106 ++++ .../azure/inventory/vo/RecommendationVH.java | 17 + .../inventory/vo/RegAppCertificateVH.java | 76 +++ .../azure/inventory/vo/RegAppSecretVH.java | 67 ++ .../inventory/vo/RegisteredApplicationVH.java | 70 +++ .../azure/inventory/vo/ResourceGroupVH.java | 53 ++ .../azure/inventory/vo/RouteTableSubnet.java | 35 ++ .../azure/inventory/vo/RouteTableVH.java | 80 +++ .../pacbot/azure/inventory/vo/RouteVH.java | 32 + .../azure/inventory/vo/SQLDatabaseVH.java | 138 ++++ .../azure/inventory/vo/SQLServerVH.java | 122 ++++ .../azure/inventory/vo/SearchServiceVH.java | 62 ++ .../azure/inventory/vo/SecurityAlertsVH.java | 35 ++ .../azure/inventory/vo/SecurityGroupVH.java | 78 +++ .../pacbot/azure/inventory/vo/SitesVH.java | 61 ++ .../pacbot/azure/inventory/vo/SnapshotVH.java | 61 ++ .../azure/inventory/vo/StorageAccountVH.java | 261 ++++++++ .../pacbot/azure/inventory/vo/SubnetVH.java | 90 +++ .../azure/inventory/vo/SubscriptionVH.java | 23 + .../pacbot/azure/inventory/vo/VMDiskVH.java | 41 ++ .../pacbot/azure/inventory/vo/VaultVH.java | 115 ++++ .../azure/inventory/vo/VirtualMachineVH.java | 323 ++++++++++ .../inventory/vo/VirtualNetworkRuleVH.java | 23 + .../pacbot/azure/inventory/vo/WorkflowVH.java | 51 ++ 90 files changed, 8478 insertions(+) create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureDiscoveryApplication.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureDiscoveryJob.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureFetchOrchestrator.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/ErrorManageUtil.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/InventoryConstants.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/auth/AWSCredentialProvider.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/BatchAccountInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/BlobContainerInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/CosmosDBInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/DatabricksInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/DiskInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/LoadBalancerInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/MariaDBInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/MySQLInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NSGInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NamespaceInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NetworkInterfaceInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NetworkInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PolicyDefinitionInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PolicyStatesInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PostgreSQLInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PublicIpAddressInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/RegisteredApplicationInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/ResourceGroupInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/RouteTableInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SCRecommendationsCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLDatabaseInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLServerInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SearchServiceInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SecurityAlertsInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SitesInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SnapshotInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/StorageAccountInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SubnetInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/Util.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/VMInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/VaultInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/WorkflowInventoryCollector.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/config/ConfigUtil.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/AssetFileGenerator.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/FileGenerator.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/FileManager.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/S3Uploader.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/AzureVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/BatchAccountVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/BlobContainerVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/CosmosDBVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/DataDiskVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/DatabricksVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/ElasticPoolVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/ErrorVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/FailoverGroupVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/FirewallRules.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/IPconfigurationVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/LoadBalancerVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/MariaDBVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/MySQLServerVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NIIPConfigVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NSGSecurityRule.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NSGSubnet.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NamespaceVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NetworkInterfaceVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NetworkVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PolicyDefinitionVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PolicyStatesVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PostgreSQLServerVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PublicIpAddressVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RecommendationVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RegAppCertificateVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RegAppSecretVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RegisteredApplicationVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/ResourceGroupVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RouteTableSubnet.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RouteTableVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RouteVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SQLDatabaseVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SQLServerVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SearchServiceVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SecurityAlertsVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SecurityGroupVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SitesVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SnapshotVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/StorageAccountVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SubnetVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SubscriptionVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VMDiskVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VaultVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VirtualMachineVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VirtualNetworkRuleVH.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/WorkflowVH.java diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureDiscoveryApplication.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureDiscoveryApplication.java new file mode 100644 index 000000000..9ab350802 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureDiscoveryApplication.java @@ -0,0 +1,21 @@ +package com.tmobile.pacbot.azure.inventory; + +import java.util.Map; + +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + + +@Configuration +@ComponentScan +public class AzureDiscoveryApplication { + + public static Map collect(String[] args) { + ApplicationContext context = new AnnotationConfigApplicationContext(AzureDiscoveryApplication.class); + AzureFetchOrchestrator orchestrator = context.getBean(AzureFetchOrchestrator.class); + return orchestrator.orchestrate(); + } +} + diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureDiscoveryJob.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureDiscoveryJob.java new file mode 100644 index 000000000..76639b631 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureDiscoveryJob.java @@ -0,0 +1,69 @@ +/******************************************************************************* + * Copyright 2018 T Mobile, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +package com.tmobile.pacbot.azure.inventory; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.tmobile.pacbot.azure.inventory.config.ConfigUtil; +import com.tmobile.pacman.commons.jobs.PacmanJob; + +/** + * The Class InventoryCollectionJob. + */ +@PacmanJob(methodToexecute="execute",jobName="AWS Data Collector", desc="Job to fetch aws info and load to Redshift" ,priority=5) +public class AzureDiscoveryJob { + + /** The log. */ + private static Logger log = LoggerFactory.getLogger(AzureDiscoveryJob.class); + /** + * The main method. + * + * @param args the arguments + */ + public static void main(String[] args){ + Map params = new HashMap<>(); + Arrays.asList(args).stream().forEach(obj-> { + String[] keyValue = obj.split("[:]"); + params.put(keyValue[0], keyValue[1]); + }); + execute(params); + } + + /** + * Execute. + * + * @param params the params + * @return + */ + public static Map execute(Map params){ + try { + ConfigUtil.setConfigProperties(params.get(InventoryConstants.CONFIG_CREDS)); + if( !(params==null || params.isEmpty())){ + params.forEach((k,v) -> System.setProperty(k, v)); + } + } catch (Exception e) { + log.error("Error fetching config", e); + ErrorManageUtil.uploadError("all", "all", "all", "Error fetching config "+ e.getMessage()); + //return ErrorManageUtil.formErrorCode(); + } + return AzureDiscoveryApplication.collect( new String[]{}); + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureFetchOrchestrator.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureFetchOrchestrator.java new file mode 100644 index 000000000..71cbbd652 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureFetchOrchestrator.java @@ -0,0 +1,96 @@ +package com.tmobile.pacbot.azure.inventory; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import com.tmobile.pacbot.azure.inventory.file.AssetFileGenerator; +import com.tmobile.pacbot.azure.inventory.file.S3Uploader; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; + +@Component +public class AzureFetchOrchestrator { + + @Autowired + AssetFileGenerator fileGenerator; + + /** The s 3 uploader. */ + @Autowired + S3Uploader s3Uploader; + + + @Value("${file.path}") + private String filePath ; + /** The target types. */ + @Value("${subscriptions:}") + private String subscriptions; + + @Value("${s3}") + private String s3Bucket ; + + @Value("${s3.data}") + private String s3Data ; + + @Value("${s3.processed}") + private String s3Processed ; + + @Value("${s3.region}") + private String s3Region ; + + /** The log. */ + private static Logger log = LoggerFactory.getLogger(AzureFetchOrchestrator.class); + + public Map orchestrate(){ + + try{ + List subscriptions = fetchSubscriptions(); + if(subscriptions.isEmpty()){ + ErrorManageUtil.uploadError("all", "all", "all", "Error fetching subscription Info "); + return ErrorManageUtil.formErrorCode(); + } + + log.info("Start : FIle Generation"); + fileGenerator.generateFiles(subscriptions,filePath); + log.info("End : FIle Generation"); + + log.info("Start : Backup Current Files"); + s3Uploader.backUpFiles(s3Bucket, s3Region, s3Data, s3Processed+ "/"+ new SimpleDateFormat("yyyyMMdd-HHmmss").format(new Date())); + log.info("End : Backup Current Files"); + + log.info("Start : Upload Files to S3"); + s3Uploader.uploadFiles(s3Bucket,s3Data,s3Region,filePath); + log.info("End : Upload Files to S3"); + + + + }catch(Exception e){ + + } + return null; + } + + private List fetchSubscriptions() { + + List subscriptionList = new ArrayList<>(); + + if(subscriptions != null && !"".equals(subscriptions)){ + String[] subscriptionsArray = subscriptions.split(","); + for(String subcritpionInfo : subscriptionsArray){ + SubscriptionVH subscription= new SubscriptionVH(); + String[] subIdName = subcritpionInfo.split("~"); + subscription.setSubscriptionId(subIdName[0].trim()); + subscription.setSubscriptionName(subIdName.length>1?subIdName[1].trim():""); + subscriptionList.add(subscription); + } + } + return subscriptionList; + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/ErrorManageUtil.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/ErrorManageUtil.java new file mode 100644 index 000000000..7dc7a7b0f --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/ErrorManageUtil.java @@ -0,0 +1,136 @@ +/******************************************************************************* + * Copyright 2018 T Mobile, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +package com.tmobile.pacbot.azure.inventory; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.tmobile.pacbot.azure.inventory.file.FileGenerator; +import com.tmobile.pacbot.azure.inventory.file.FileManager; +import com.tmobile.pacbot.azure.inventory.vo.ErrorVH; + + +/** + * The Class ErrorManageUtil. + */ +public class ErrorManageUtil { + + /** The log. */ + private static Logger log = LoggerFactory.getLogger(ErrorManageUtil.class); + + /** The error map. */ + private static Map> errorMap = new HashMap<>(); + + /** + * Instantiates a new error manage util. + */ + private ErrorManageUtil() { + + } + + /** + * Initialise. + */ + public static void initialise(){ + try { + FileGenerator.writeToFile("azure-loaderror.data",InventoryConstants.OPEN_ARRAY, false); + } catch (IOException e) { + log.error("Error in Initialise",e); + } + } + + /** + * Finalise. + */ + public static void finalise(){ + try { + FileGenerator.writeToFile("azure-loaderror.data",InventoryConstants.CLOSE_ARRAY, true); + } catch (IOException e) { + log.error("Error in finalise",e); + } + } + + /** + * Upload error. + * + * @param account the account + * @param region the region + * @param type the type + * @param exception the exception + */ + public static synchronized void uploadError(String account, String region, String type, String exception) { + try{ + List errorList = errorMap.get(account); + if(errorList==null){ + errorList = new ArrayList<>(); + errorMap.put(account, errorList); + } + ErrorVH error = new ErrorVH(); + error.setException(exception); + error.setRegion(region); + error.setType(type); + errorList.add(error); + }catch(Exception e){ + log.error("Error in uploadError",e); + } + } + + + + public static Map formErrorCode() { + Map errorCode = new HashMap<>(); + errorCode.put("jobName", System.getProperty("jobName")); + + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + errorCode.put("executionEndDate", sdf.format(new Date())); + + List> errors = new ArrayList<>(); + for(Entry> errorDetail :errorMap.entrySet()) { + Map error = new HashMap<>(); + List> details = new ArrayList<>(); + + error.put("error", "Error while fetching Inventory for account "+errorDetail.getKey()); + for(ErrorVH errorVH : errorDetail.getValue()) { + Map detail = new HashMap<>(); + detail.put("type",errorVH.getType()); + detail.put("region",errorVH.getRegion()); + detail.put("exception",errorVH.getException()); + detail.put("account",errorDetail.getKey()); + details.add(detail); + } + error.put("details",details); + errors.add(error); + } + + errorCode.put("errors", errors); + if(errors.isEmpty()) { + errorCode.put("status","Success"); + } else { + errorCode.put("status","Partial Success"); + } + log.info("Return Info {}",errorCode); + return errorCode; + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/InventoryConstants.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/InventoryConstants.java new file mode 100644 index 000000000..f9ed565e9 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/InventoryConstants.java @@ -0,0 +1,28 @@ +package com.tmobile.pacbot.azure.inventory; + +public class InventoryConstants { + + public static final String OPEN_ARRAY = "["; + public static final String CLOSE_ARRAY = "]"; + /** The Constant NAME. */ + public static final String NAME = "name"; + + /** The Constant SOURCE. */ + public static final String SOURCE = "source"; + + /** The Constant APPLICATION. */ + public static final String APPLICATION = "application"; + + /** The Constant BATCH. */ + public static final String BATCH = "batch"; + + /** The Constant INVENTORY. */ + public static final String INVENTORY = "azure-discovery"; + + /** The Constant INVENTORY. */ + public static final String SUBSCRIPTION_ID = "subscriptionId"; + + /** The Constant INVENTORY. */ + public static final String SUBSCRIPTION_NAME = "subscriptionName"; + public static final String CONFIG_CREDS = "config_creds"; +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/auth/AWSCredentialProvider.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/auth/AWSCredentialProvider.java new file mode 100644 index 000000000..229ca9a90 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/auth/AWSCredentialProvider.java @@ -0,0 +1,80 @@ +/******************************************************************************* + * Copyright 2018 T Mobile, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +package com.tmobile.pacbot.azure.inventory.auth; + +import org.springframework.stereotype.Component; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; +import com.amazonaws.services.securitytoken.AWSSecurityTokenService; +import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; +import com.amazonaws.services.securitytoken.model.AssumeRoleRequest; +import com.amazonaws.services.securitytoken.model.AssumeRoleResult; + + +/** + * The Class CredentialProvider. + */ +@Component +public class AWSCredentialProvider { + + /** The dev mode. */ + private static boolean devMode = System.getProperty("PIC_DEV_MODE")==null?false:true; + + + /** + * Gets the base account credentials. + * + * @param roleName the role name + * @return the base account credentials + */ + public BasicSessionCredentials getCredentials (String baseAccount, String baseRegion,String roleName){ + if(devMode){ + String accessKey = System.getProperty("ACCESS_KEY"); + String secretKey = System.getProperty("SECRET_KEY"); + BasicAWSCredentials awsCreds = new BasicAWSCredentials(accessKey, secretKey); + AWSSecurityTokenServiceClientBuilder stsBuilder = AWSSecurityTokenServiceClientBuilder.standard().withCredentials( new AWSStaticCredentialsProvider(awsCreds)).withRegion(baseRegion); + AWSSecurityTokenService sts = stsBuilder.build(); + AssumeRoleRequest assumeRequest = new AssumeRoleRequest().withRoleArn(getRoleArn(baseAccount,roleName)).withRoleSessionName("pic-base-ro"); + AssumeRoleResult assumeResult = sts.assumeRole(assumeRequest); + return new BasicSessionCredentials( + assumeResult.getCredentials().getAccessKeyId(), assumeResult.getCredentials().getSecretAccessKey(), + assumeResult.getCredentials().getSessionToken()); + + } + else{ + System.out.println("inside"); + AWSSecurityTokenService sts = AWSSecurityTokenServiceClientBuilder.defaultClient(); + AssumeRoleRequest assumeRequest = new AssumeRoleRequest().withRoleArn(getRoleArn(baseAccount,roleName)).withRoleSessionName("pic-base-ro"); + AssumeRoleResult assumeResult = sts.assumeRole(assumeRequest); + return new BasicSessionCredentials( + assumeResult.getCredentials().getAccessKeyId(), assumeResult.getCredentials().getSecretAccessKey(), + assumeResult.getCredentials().getSessionToken()); + } + } + + /** + * Gets the role arn. + * + * @param accout the accout + * @param role the role + * @return the role arn + */ + private String getRoleArn(String accout, String role){ + return "arn:aws:iam::"+accout+":role/"+role; + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/BatchAccountInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/BatchAccountInventoryCollector.java new file mode 100644 index 000000000..ae99f886a --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/BatchAccountInventoryCollector.java @@ -0,0 +1,89 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.tmobile.pacbot.azure.inventory.vo.BatchAccountVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class BatchAccountInventoryCollector { + + private static Logger log = LoggerFactory.getLogger(BatchAccountInventoryCollector.class); + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.Batch/batchAccounts?api-version=2019-08-01"; + + public List fetchBatchAccountDetails(SubscriptionVH subscription) throws Exception { + + List batchAccountList = new ArrayList(); + String accessToken; + try { + accessToken = AzureCredentialManager.getAuthToken(); + + } catch (Exception e1) { + return batchAccountList; + } + + String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); + try { + + String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); + JsonObject responseObj = new JsonParser().parse(response).getAsJsonObject(); + JsonArray batchAccountObjects = responseObj.getAsJsonArray("value"); + if (batchAccountObjects != null) { + for (JsonElement batchAccountElement : batchAccountObjects) { + BatchAccountVH batchAccountVH = new BatchAccountVH(); + JsonObject batchAccountObject = batchAccountElement.getAsJsonObject(); + batchAccountVH.setSubscription(subscription.getSubscriptionId()); + batchAccountVH.setSubscriptionName(subscription.getSubscriptionName()); + batchAccountVH.setId(batchAccountObject.get("id").getAsString()); + batchAccountVH.setLocation(batchAccountObject.get("location").getAsString()); + batchAccountVH.setName(batchAccountObject.get("name").getAsString()); + batchAccountVH.setType(batchAccountObject.get("type").getAsString()); + JsonObject properties = batchAccountObject.getAsJsonObject("properties"); + JsonObject tags = batchAccountObject.getAsJsonObject("tags"); + if (properties != null) { + HashMap propertiesMap = new Gson().fromJson(properties.toString(), + HashMap.class); + batchAccountVH.setProvisioningState(propertiesMap.get("provisioningState").toString()); + batchAccountVH.setAccountEndpoint(propertiesMap.get("accountEndpoint").toString()); + batchAccountVH.setPoolQuota(propertiesMap.get("poolQuota").toString()); + batchAccountVH.setPoolAllocationMode(propertiesMap.get("poolAllocationMode").toString()); + batchAccountVH.setDedicatedCoreQuotaPerVMFamily(propertiesMap.get("dedicatedCoreQuotaPerVMFamilyEnforced").toString()); + batchAccountVH.setDedicatedCoreQuota(propertiesMap.get("dedicatedCoreQuota").toString()); + batchAccountVH.setLowPriorityCoreQuota(propertiesMap.get("lowPriorityCoreQuota").toString()); + batchAccountVH.setActiveJobAndJobScheduleQuota(propertiesMap.get("activeJobAndJobScheduleQuota").toString()); + batchAccountVH.setAutoStorage((Map) propertiesMap.get("autoStorage")); + } + if (tags != null) { + HashMap tagsMap = new Gson().fromJson(tags.toString(), HashMap.class); + batchAccountVH.setTags(tagsMap); + + + + } + + batchAccountList.add(batchAccountVH); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return batchAccountList; + } + + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/BlobContainerInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/BlobContainerInventoryCollector.java new file mode 100644 index 000000000..162b7e897 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/BlobContainerInventoryCollector.java @@ -0,0 +1,78 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.storage.StorageAccount; +import com.tmobile.pacbot.azure.inventory.vo.BlobContainerVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class BlobContainerInventoryCollector { + + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Storage/storageAccounts/%s/blobServices/default/containers?api-version=2019-04-01"; + private static Logger log = LoggerFactory.getLogger(BlobContainerInventoryCollector.class); + + public List fetchBlobContainerDetails(SubscriptionVH subscription,Map> tagMap) { + + List blobContainerList = new ArrayList(); + String accessToken; + try { + accessToken = AzureCredentialManager.getAuthToken(); + } catch (Exception e1) { + return blobContainerList; + } + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + PagedList storageAccounts = azure.storageAccounts().list(); + for (StorageAccount storageAccount : storageAccounts) { + String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId()), + URLEncoder.encode(storageAccount.resourceGroupName()), URLEncoder.encode(storageAccount.name())); + try { + String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); + JsonObject responseObj = new JsonParser().parse(response).getAsJsonObject(); + JsonArray blobObjects = responseObj.getAsJsonArray("value"); + for (JsonElement blobObjectElement : blobObjects) { + Map tags= new HashMap(); + BlobContainerVH blobContainerVH = new BlobContainerVH(); + blobContainerVH.setSubscription(subscription.getSubscriptionId()); + blobContainerVH.setSubscriptionName(subscription.getSubscriptionName()); + JsonObject blobObject = blobObjectElement.getAsJsonObject(); + JsonObject properties = blobObject.getAsJsonObject("properties"); + blobContainerVH.setId(blobObject.get("id").getAsString()); + blobContainerVH.setName(blobObject.get("name").getAsString()); + blobContainerVH.setType(blobObject.get("type").getAsString()); + blobContainerVH.setTag(blobObject.get("etag").getAsString()); + blobContainerVH.setTags(Util.tagsList(tagMap, storageAccount.resourceGroupName(), tags)); + if (properties!=null) { + HashMap propertiesMap = new Gson().fromJson(properties.toString(), + HashMap.class); + blobContainerVH.setPropertiesMap(propertiesMap); + } + blobContainerList.add(blobContainerVH); + } + } catch (Exception e) { + log.error(" Error fetching blobcontainers for storage account {} Cause : {}" ,storageAccount.name(),e.getMessage()); + + } + } + System.out.println(blobContainerList.size()); + return blobContainerList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/CosmosDBInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/CosmosDBInventoryCollector.java new file mode 100644 index 000000000..0f0099c97 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/CosmosDBInventoryCollector.java @@ -0,0 +1,74 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Component; + +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.cosmosdb.CosmosDBAccount; +import com.microsoft.azure.management.cosmosdb.VirtualNetworkRule; +import com.tmobile.pacbot.azure.inventory.vo.CosmosDBVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacbot.azure.inventory.vo.VirtualNetworkRuleVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; + +@Component +public class CosmosDBInventoryCollector { + + public List fetchCosmosDBDetails(SubscriptionVH subscription, Map> tagMap) { + List cosmosDBList = new ArrayList(); + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + PagedList CosmosDB = azure.cosmosDBAccounts().list(); + System.out.println(CosmosDB.size()); + for (CosmosDBAccount cosmosDB : CosmosDB) { + CosmosDBVH cosmosDBVH = new CosmosDBVH(); + cosmosDBVH.setSubscription(subscription.getSubscriptionId()); + cosmosDBVH.setSubscriptionName(subscription.getSubscriptionName()); + cosmosDBVH.setId(cosmosDB.id()); + cosmosDBVH.setKey(cosmosDB.key()); + cosmosDBVH.setName(cosmosDB.name()); + cosmosDBVH.setResourceGroupName(cosmosDB.resourceGroupName()); + cosmosDBVH.setRegion(cosmosDB.regionName()); + cosmosDBVH.setTags(Util.tagsList(tagMap, cosmosDB.resourceGroupName(), cosmosDB.tags())); + cosmosDBVH.setType(cosmosDB.type()); + cosmosDBVH.setIpRangeFilter(cosmosDB.ipRangeFilter()); + cosmosDBVH.setMultipleWriteLocationsEnabled(cosmosDB.multipleWriteLocationsEnabled()); + cosmosDBVH.setVirtualNetworkRuleList(getVirtualNetworkRule(cosmosDB.virtualNetworkRules())); + cosmosDBList.add(cosmosDBVH); + /* + * boolean flag = false; Map tagsFinal = new HashMap(); + * + * for (Map.Entry> resourceGroupTag : + * tagMap.entrySet()) { + * + * if (resourceGroupTag.getKey().equalsIgnoreCase(cosmosDB.resourceGroupName())) + * { flag = true; tagsFinal.putAll(resourceGroupTag.getValue()); + * tagsFinal.putAll(cosmosDB.tags()); break; } + * + * } if (flag == true) { cosmosDBVH.setTags(tagsFinal); } else { + * cosmosDBVH.setTags(cosmosDB.tags()); } + */ + + } + return cosmosDBList; + } + + private List getVirtualNetworkRule(List virtualNetworkRuleList) { + List virtualNetworkRuleVHlist = new ArrayList<>(); + for (VirtualNetworkRule virtualNetworkRule : virtualNetworkRuleList) { + VirtualNetworkRuleVH virtualNetworkRuleVH = new VirtualNetworkRuleVH(); + virtualNetworkRuleVH.setId(virtualNetworkRule.id()); + virtualNetworkRuleVH + .setIgnoreMissingVNetServiceEndpoint(virtualNetworkRule.ignoreMissingVNetServiceEndpoint()); + virtualNetworkRuleVHlist.add(virtualNetworkRuleVH); + + } + return virtualNetworkRuleVHlist; + + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/DatabricksInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/DatabricksInventoryCollector.java new file mode 100644 index 000000000..f752b527d --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/DatabricksInventoryCollector.java @@ -0,0 +1,71 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.springframework.stereotype.Component; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.microsoft.azure.management.Azure; +import com.tmobile.pacbot.azure.inventory.vo.DatabricksVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class DatabricksInventoryCollector { + + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.Databricks/workspaces?api-version=2018-04-01"; + + public List fetchDatabricksDetails(SubscriptionVH subscription) { + + List databricksList = new ArrayList(); + String accessToken; + try { + accessToken = AzureCredentialManager.getAuthToken(); + } catch (Exception e1) { + return databricksList; + } + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + + String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); + try { + String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); + JsonObject responseObj = new JsonParser().parse(response).getAsJsonObject(); + JsonArray databricksObjects = responseObj.getAsJsonArray("value"); + for (JsonElement databricksElement : databricksObjects) { + DatabricksVH databricksVH = new DatabricksVH(); + JsonObject databricksObject = databricksElement.getAsJsonObject(); + JsonObject properties = databricksObject.getAsJsonObject("properties"); + JsonObject sku = databricksObject.getAsJsonObject("sku"); + databricksVH.setId(databricksObject.get("id").getAsString()); + databricksVH.setLocation(databricksObject.get("location").getAsString()); + databricksVH.setName(databricksObject.get("name").getAsString()); + databricksVH.setType(databricksObject.get("type").getAsString()); + databricksVH.setSubscription(subscription.getSubscriptionId()); + databricksVH.setSubscriptionName(subscription.getSubscriptionName()); + if (sku!=null) { + HashMap skuMap = new Gson().fromJson(sku.toString(), HashMap.class); + databricksVH.setSkuMap(skuMap); + } + if (properties!=null) { + HashMap propertiesMap = new Gson().fromJson(properties.toString(), HashMap.class); + databricksVH.setPropertiesMap(propertiesMap); + } + databricksList.add(databricksVH); + } + } catch (Exception e) { + e.printStackTrace(); + } + + System.out.println(databricksList.size()); + return databricksList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/DiskInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/DiskInventoryCollector.java new file mode 100644 index 000000000..42d064845 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/DiskInventoryCollector.java @@ -0,0 +1,43 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Component; + +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.compute.Disk; +import com.tmobile.pacbot.azure.inventory.vo.DataDiskVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; + +@Component +public class DiskInventoryCollector { + public List fetchDataDiskDetails(SubscriptionVH subscription, Map> tagMap) { + List dataDiskList = new ArrayList(); + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + PagedList dataDisks = azure.disks().list(); + System.out.println(dataDisks.size()); + for (Disk dataDisk : dataDisks) { + DataDiskVH dataDiskVH = new DataDiskVH(); + dataDiskVH.setId(dataDisk.id()); + dataDiskVH.setIsAttachedToVirtualMachine(dataDisk.isAttachedToVirtualMachine()); + dataDiskVH.setKey(dataDisk.key()); + dataDiskVH.setName(dataDisk.name()); + dataDiskVH.setDiskInner(dataDisk.inner()); + dataDiskVH.setRegion(dataDisk.region().toString()); + dataDiskVH.setResourceGroupName(dataDisk.resourceGroupName()); + dataDiskVH.setSizeInGB(dataDisk.sizeInGB()); + dataDiskVH.setTags(Util.tagsList(tagMap, dataDisk.resourceGroupName(), dataDisk.tags())); + dataDiskVH.setType(dataDisk.type()); + dataDiskVH.setVirtualMachineId(dataDisk.virtualMachineId()); + dataDiskVH.setSubscription(subscription.getSubscriptionId()); + dataDiskVH.setSubscriptionName(subscription.getSubscriptionName()); + dataDiskList.add(dataDiskVH); + } + return dataDiskList; + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/LoadBalancerInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/LoadBalancerInventoryCollector.java new file mode 100644 index 000000000..395446559 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/LoadBalancerInventoryCollector.java @@ -0,0 +1,47 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Component; + +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.network.LoadBalancer; +import com.tmobile.pacbot.azure.inventory.vo.LoadBalancerVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; + +@Component +public class LoadBalancerInventoryCollector { + + public List fetchLoadBalancerDetails(SubscriptionVH subscription, + Map> tagMap) { + List loadBalancerList = new ArrayList(); + + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + PagedList loadBalancers = azure.loadBalancers().list(); + System.out.println(loadBalancers.size()); + for (LoadBalancer loadBalancer : loadBalancers) { + LoadBalancerVH loadBalancerVH = new LoadBalancerVH(); + loadBalancerVH.setHashCode(loadBalancer.hashCode()); + loadBalancerVH.setId(loadBalancer.id()); + loadBalancerVH.setKey(loadBalancer.key()); + loadBalancerVH.setPublicIPAddressIds(loadBalancer.publicIPAddressIds()); + loadBalancerVH.setName(loadBalancer.name()); + loadBalancerVH.setRegionName(loadBalancer.regionName()); + loadBalancerVH.setResourceGroupName(loadBalancer.resourceGroupName()); + loadBalancerVH.setTags(Util.tagsList(tagMap, loadBalancer.resourceGroupName(), loadBalancer.tags())); + loadBalancerVH.setType(loadBalancer.type()); + loadBalancerVH.setSubscription(subscription.getSubscriptionId()); + loadBalancerVH.setSubscriptionName(subscription.getSubscriptionName()); + loadBalancerList.add(loadBalancerVH); + + } + + return loadBalancerList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/MariaDBInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/MariaDBInventoryCollector.java new file mode 100644 index 000000000..690957e28 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/MariaDBInventoryCollector.java @@ -0,0 +1,71 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.springframework.stereotype.Component; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.microsoft.azure.management.Azure; +import com.tmobile.pacbot.azure.inventory.vo.MariaDBVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class MariaDBInventoryCollector { + + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.DBforMariaDB/servers?api-version=2018-06-01-preview"; + + public List fetchMariaDBDetails(SubscriptionVH subscription) { + + List mariaDBList = new ArrayList(); + String accessToken; + try { + accessToken = AzureCredentialManager.getAuthToken(); + } catch (Exception e1) { + return mariaDBList; + } + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + + String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); + try { + String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); + JsonObject responseObj = new JsonParser().parse(response).getAsJsonObject(); + JsonArray mariaDBObjects = responseObj.getAsJsonArray("value"); + for (JsonElement mariaDBElement : mariaDBObjects) { + MariaDBVH mariaDBVH = new MariaDBVH(); + JsonObject mariaDBObject = mariaDBElement.getAsJsonObject(); + JsonObject properties = mariaDBObject.getAsJsonObject("properties"); + JsonObject sku = mariaDBObject.getAsJsonObject("sku"); + mariaDBVH.setId(mariaDBObject.get("id").getAsString()); + mariaDBVH.setLocation(mariaDBObject.get("location").getAsString()); + mariaDBVH.setName(mariaDBObject.get("name").getAsString()); + mariaDBVH.setType(mariaDBObject.get("type").getAsString()); + mariaDBVH.setSubscription(subscription.getSubscriptionId()); + mariaDBVH.setSubscriptionName(subscription.getSubscriptionName()); + if (sku!=null) { + HashMap skuMap = new Gson().fromJson(sku.toString(), HashMap.class); + mariaDBVH.setSkuMap(skuMap); + } + if (properties!=null) { + HashMap propertiesMap = new Gson().fromJson(properties.toString(), HashMap.class); + mariaDBVH.setPropertiesMap(propertiesMap); + } + mariaDBList.add(mariaDBVH); + } + } catch (Exception e) { + e.printStackTrace(); + } + + System.out.println(mariaDBList.size()); + return mariaDBList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/MySQLInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/MySQLInventoryCollector.java new file mode 100644 index 000000000..e34c322d2 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/MySQLInventoryCollector.java @@ -0,0 +1,74 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.springframework.stereotype.Component; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.storage.StorageAccount; +import com.tmobile.pacbot.azure.inventory.vo.BlobContainerVH; +import com.tmobile.pacbot.azure.inventory.vo.MySQLServerVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class MySQLInventoryCollector { + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.DBforMySQL/servers?api-version=2017-12-01"; + + public List fetchMySQLServerDetails(SubscriptionVH subscription) { + + List mySqlServerList = new ArrayList(); + String accessToken; + try { + accessToken = AzureCredentialManager.getAuthToken(); + } catch (Exception e1) { + return mySqlServerList; + } + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + + String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); + try { + String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); + JsonObject responseObj = new JsonParser().parse(response).getAsJsonObject(); + JsonArray sqlServerObjects = responseObj.getAsJsonArray("value"); + for (JsonElement sqlServerObjectElement : sqlServerObjects) { + MySQLServerVH mySQLServerVH = new MySQLServerVH(); + mySQLServerVH.setSubscription(subscription.getSubscriptionId()); + mySQLServerVH.setSubscriptionName(subscription.getSubscriptionName()); + JsonObject sqlServerObject = sqlServerObjectElement.getAsJsonObject(); + JsonObject properties = sqlServerObject.getAsJsonObject("properties"); + JsonObject sku = sqlServerObject.getAsJsonObject("sku"); + mySQLServerVH.setId(sqlServerObject.get("id").getAsString()); + mySQLServerVH.setLocation(sqlServerObject.get("location").getAsString()); + mySQLServerVH.setName(sqlServerObject.get("name").getAsString()); + mySQLServerVH.setType(sqlServerObject.get("type").getAsString()); + if (sku!=null) { + HashMap skuMap = new Gson().fromJson(sku.toString(), HashMap.class); + mySQLServerVH.setSkuMap(skuMap); + } + if (properties!=null) { + HashMap propertiesMap = new Gson().fromJson(properties.toString(), HashMap.class); + mySQLServerVH.setPropertiesMap(propertiesMap); + } + + mySqlServerList.add(mySQLServerVH); + } + } catch (Exception e) { + e.printStackTrace(); + } + + System.out.println(mySqlServerList.size()); + return mySqlServerList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NSGInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NSGInventoryCollector.java new file mode 100644 index 000000000..6bc168e1d --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NSGInventoryCollector.java @@ -0,0 +1,96 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Component; + +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.network.NetworkSecurityGroup; +import com.microsoft.azure.management.network.NetworkSecurityRule; +import com.microsoft.azure.management.network.Subnet; +import com.tmobile.pacbot.azure.inventory.vo.NSGSubnet; +import com.tmobile.pacbot.azure.inventory.vo.SecurityGroupVH; +import com.tmobile.pacbot.azure.inventory.vo.NSGSecurityRule; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; + +@Component +public class NSGInventoryCollector { + public List fetchNetworkSecurityGroupDetails(SubscriptionVH subscription, + Map> tagMap) { + List securityGroupsList = new ArrayList(); + + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + PagedList securityGroups = azure.networkSecurityGroups().list(); + for (NetworkSecurityGroup securityGroup : securityGroups) { + SecurityGroupVH securityGroupVH = new SecurityGroupVH(); + securityGroupVH.setId(securityGroup.id()); + securityGroupVH.setKey(securityGroup.key()); + securityGroupVH.setName(securityGroup.name()); + securityGroupVH.setRegion(securityGroup.regionName()); + securityGroupVH.setResourceGroupName(securityGroup.resourceGroupName()); + securityGroupVH.setTags(Util.tagsList(tagMap, securityGroup.resourceGroupName(), securityGroup.tags())); + securityGroupVH.setSubnetList(getNetworkSecuritySubnetDetails(securityGroup.listAssociatedSubnets())); + securityGroupVH.setNetworkInterfaceIds(securityGroup.networkInterfaceIds()); + securityGroupVH.setSubscription(subscription.getSubscriptionId()); + securityGroupVH.setSubscriptionName(subscription.getSubscriptionName()); + setSecurityRules(securityGroup, securityGroupVH); + securityGroupsList.add(securityGroupVH); + + } + + return securityGroupsList; + } + + private void setSecurityRules(NetworkSecurityGroup securityGroup, SecurityGroupVH securityGroupVH) { + List inBoundSecurityList = new ArrayList(); + List outBoundSecurityList = new ArrayList(); + + for (Map.Entry entry : securityGroup.securityRules().entrySet()) { + populateRuleInfo(inBoundSecurityList, outBoundSecurityList, entry.getValue(), false); + } + for (Map.Entry entry : securityGroup.defaultSecurityRules().entrySet()) { + populateRuleInfo(inBoundSecurityList, outBoundSecurityList, entry.getValue(), true); + } + securityGroupVH.setOutBoundSecurityRules(outBoundSecurityList); + securityGroupVH.setInBoundSecurityRules(inBoundSecurityList); + + } + + private void populateRuleInfo(List inBoundSecurityList, List outBoundSecurityList, + NetworkSecurityRule securityRule, boolean isDefault) { + NSGSecurityRule securityListVH = new NSGSecurityRule(); + securityListVH.setName(securityRule.name()); + securityListVH.setDescription(securityRule.description()); + securityListVH.setAccess(securityRule.access().toString()); + securityListVH.setPriority(securityRule.priority()); + securityListVH.setProtocol(securityRule.protocol().toString()); + securityListVH.listValue(securityRule); + securityListVH.setDestinationApplicationSecurityGroupIds(securityRule.destinationApplicationSecurityGroupIds()); + securityListVH.setSourceApplicationSecurityGroupIds(securityRule.sourceApplicationSecurityGroupIds()); + securityListVH.setDefault(isDefault); + if (securityRule.direction().toString().equals("Inbound")) { + inBoundSecurityList.add(securityListVH); + } else if (securityRule.direction().toString().equals("Outbound")) { + outBoundSecurityList.add(securityListVH); + } + } + + private List getNetworkSecuritySubnetDetails(List subnetList) { + List subnetVHlist = new ArrayList<>(); + for (Subnet subnet : subnetList) { + NSGSubnet subnetVH = new NSGSubnet(); + subnetVH.setAddressPrefix(subnet.addressPrefix()); + subnetVH.setName(subnet.name()); + subnetVH.setVnet(subnet.parent().id()); + subnetVHlist.add(subnetVH); + + } + return subnetVHlist; + + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NamespaceInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NamespaceInventoryCollector.java new file mode 100644 index 000000000..2db13bf02 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NamespaceInventoryCollector.java @@ -0,0 +1,85 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.tmobile.pacbot.azure.inventory.vo.NamespaceVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class NamespaceInventoryCollector { + + private static Logger log = LoggerFactory.getLogger(NamespaceInventoryCollector.class); + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.EventHub/namespaces?api-version=2017-04-01"; + + public List fetchNamespaceDetails(SubscriptionVH subscription) throws Exception { + + List namespaceList = new ArrayList(); + String accessToken; + try { + accessToken = AzureCredentialManager.getAuthToken(); + + } catch (Exception e1) { + return namespaceList; + } + + String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); + try { + + String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); + JsonObject responseObj = new JsonParser().parse(response).getAsJsonObject(); + JsonArray namespaceObjects = responseObj.getAsJsonArray("value"); + if (namespaceObjects != null) { + for (JsonElement namespaceElement : namespaceObjects) { + NamespaceVH namespaceVH = new NamespaceVH(); + JsonObject namespaceObject = namespaceElement.getAsJsonObject(); + namespaceVH.setSubscription(subscription.getSubscriptionId()); + namespaceVH.setSubscriptionName(subscription.getSubscriptionName()); + namespaceVH.setId(namespaceObject.get("id").getAsString()); + namespaceVH.setLocation(namespaceObject.get("location").getAsString()); + namespaceVH.setName(namespaceObject.get("name").getAsString()); + namespaceVH.setType(namespaceObject.get("type").getAsString()); + JsonObject properties = namespaceObject.getAsJsonObject("properties"); + JsonObject tags = namespaceObject.getAsJsonObject("tags"); + JsonObject sku = namespaceObject.getAsJsonObject("sku"); + if (properties != null) { + HashMap propertiesMap = new Gson().fromJson(properties.toString(), + HashMap.class); + namespaceVH.setProperties(propertiesMap); + } + if (tags != null) { + HashMap tagsMap = new Gson().fromJson(tags.toString(), HashMap.class); + namespaceVH.setTags(tagsMap); + } + if (sku != null) { + HashMap skuMap = new Gson().fromJson(sku.toString(), HashMap.class); + namespaceVH.setSku(skuMap); + } + + + namespaceList.add(namespaceVH); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + System.out.println(namespaceList.size()); + return namespaceList; + } + + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NetworkInterfaceInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NetworkInterfaceInventoryCollector.java new file mode 100644 index 000000000..5dde5638f --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NetworkInterfaceInventoryCollector.java @@ -0,0 +1,76 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Component; + +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.network.NetworkInterface; +import com.microsoft.azure.management.network.NicIPConfiguration; +import com.tmobile.pacbot.azure.inventory.vo.NIIPConfigVH; +import com.tmobile.pacbot.azure.inventory.vo.NetworkInterfaceVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; + +@Component +public class NetworkInterfaceInventoryCollector { + + public List fetchNetworkInterfaceDetails(SubscriptionVH subscription, + Map> tagMap) { + List networkInterfaceList = new ArrayList(); + + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + PagedList networkInterfaces = azure.networkInterfaces().list(); + + for (NetworkInterface networkInterface : networkInterfaces) { + NetworkInterfaceVH networkInterfaceVH = new NetworkInterfaceVH(); + networkInterfaceVH.setAppliedDnsServers(networkInterface.appliedDnsServers()); + networkInterfaceVH.setDnsServers(networkInterface.dnsServers()); + networkInterfaceVH.setId(networkInterface.id()); + networkInterfaceVH.setInternalDnsNameLabel(networkInterface.internalDnsNameLabel()); + networkInterfaceVH.setInternalDomainNameSuffix(networkInterface.internalDomainNameSuffix()); + networkInterfaceVH.setInternalFqdn(networkInterface.internalFqdn()); + networkInterfaceVH.setAcceleratedNetworkingEnabled(networkInterface.isAcceleratedNetworkingEnabled()); + networkInterfaceVH.setKey(networkInterface.key()); + networkInterfaceVH.setMacAddress(networkInterface.macAddress()); + networkInterfaceVH.setName(networkInterface.name()); + networkInterfaceVH.setNetworkSecurityGroupId(networkInterface.networkSecurityGroupId()); + networkInterfaceVH.setPrimaryPrivateIP(networkInterface.primaryPrivateIP()); + networkInterfaceVH + .setTags(Util.tagsList(tagMap, networkInterface.resourceGroupName(), networkInterface.tags())); + networkInterfaceVH.setVirtualMachineId(networkInterface.virtualMachineId()); + networkInterfaceVH.setSubscription(subscription.getSubscriptionId()); + networkInterfaceVH.setSubscriptionName(subscription.getSubscriptionName()); + networkInterfaceVH.setIPForwardingEnabled(networkInterface.isIPForwardingEnabled()); + setipConfigurations(networkInterface.ipConfigurations(), networkInterfaceVH); + networkInterfaceList.add(networkInterfaceVH); + + } + return networkInterfaceList; + } + + private void setipConfigurations(Map ipConfigurations, + NetworkInterfaceVH networkInterfaceVH) { + List ipConfigurationList = new ArrayList<>(); + for (Map.Entry entry : ipConfigurations.entrySet()) { + NIIPConfigVH niipConfigVH = new NIIPConfigVH(); + niipConfigVH.setName(entry.getValue().name()); + niipConfigVH.setPrivateIPAddress(entry.getValue().privateIPAddress()); + niipConfigVH.setPrivateIPAddressVersion(entry.getValue().privateIPAddressVersion() != null + ? entry.getValue().privateIPAddressVersion().toString() + : ""); + niipConfigVH.setNetworkName(entry.getValue().getNetwork().name()); + niipConfigVH.setSubnetName(entry.getValue().subnetName()); + niipConfigVH.setPrimary(entry.getValue().isPrimary()); + niipConfigVH.setPublicIPAddress( + entry.getValue().getPublicIPAddress() != null ? entry.getValue().getPublicIPAddress().ipAddress() + : ""); + ipConfigurationList.add(niipConfigVH); + } + networkInterfaceVH.setIpConfigurationList(ipConfigurationList); + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NetworkInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NetworkInventoryCollector.java new file mode 100644 index 000000000..fc2da4388 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NetworkInventoryCollector.java @@ -0,0 +1,47 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Component; + +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.network.Network; +import com.tmobile.pacbot.azure.inventory.vo.NetworkVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; + +@Component +public class NetworkInventoryCollector { + + public List fetchNetworkDetails(SubscriptionVH subscription, Map> tagMap) { + List networkList = new ArrayList(); + + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + PagedList networks = azure.networks().list(); + + for (Network network : networks) { + NetworkVH networkVH = new NetworkVH(); + networkVH.setAddressSpaces(network.addressSpaces()); + networkVH.setDdosProtectionPlanId(network.ddosProtectionPlanId()); + networkVH.setDnsServerIPs(network.dnsServerIPs()); + networkVH.setHashCode(network.hashCode()); + networkVH.setId(network.id()); + networkVH.setDdosProtectionEnabled(network.isDdosProtectionEnabled()); + networkVH.setVmProtectionEnabled(network.isVmProtectionEnabled()); + networkVH.setKey(network.key()); + networkVH.setName(network.name()); + networkVH.setRegion(network.region().name()); + networkVH.setResourceGroupName(network.resourceGroupName()); + networkVH.setTags(Util.tagsList(tagMap, network.resourceGroupName(), network.tags())); + networkVH.setSubscription(subscription.getSubscriptionId()); + networkVH.setSubscriptionName(subscription.getSubscriptionName()); + networkList.add(networkVH); + } + + return networkList; + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PolicyDefinitionInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PolicyDefinitionInventoryCollector.java new file mode 100644 index 000000000..52ecacdc4 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PolicyDefinitionInventoryCollector.java @@ -0,0 +1,38 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Component; + +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.compute.Disk; +import com.microsoft.azure.management.resources.PolicyDefinition; +import com.tmobile.pacbot.azure.inventory.vo.PolicyDefinitionVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; + +@Component +public class PolicyDefinitionInventoryCollector { + public List fetchPolicyDefinitionDetails(SubscriptionVH subscription) { + List policyDefinitionList = new ArrayList(); + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + PagedList policyDefinitions = azure.policyDefinitions().list(); + System.out.println(policyDefinitions.size()); + for (PolicyDefinition policyDefinition : policyDefinitions) { + PolicyDefinitionVH policyDefinitionVH = new PolicyDefinitionVH(); + policyDefinitionVH.setId(policyDefinition.id()); + policyDefinitionVH.setName(policyDefinition.name()); + policyDefinitionVH.setDescription(policyDefinition.description()); + policyDefinitionVH.setDisplayName(policyDefinition.displayName()); + policyDefinitionVH.setPolicyType(policyDefinition.policyType().toString()); + policyDefinitionVH.setPolicyRule(policyDefinition.policyRule().toString()); + policyDefinitionVH.setSubscription(subscription.getSubscriptionId()); + policyDefinitionVH.setSubscriptionName(subscription.getSubscriptionName()); + policyDefinitionList.add(policyDefinitionVH); + } + return policyDefinitionList; + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PolicyStatesInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PolicyStatesInventoryCollector.java new file mode 100644 index 000000000..1a455e4e4 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PolicyStatesInventoryCollector.java @@ -0,0 +1,100 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.List; + +import org.springframework.stereotype.Component; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.tmobile.pacbot.azure.inventory.vo.PolicyDefinitionVH; +import com.tmobile.pacbot.azure.inventory.vo.PolicyStatesVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class PolicyStatesInventoryCollector { + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.PolicyInsights/policyStates/latest/queryResults?api-version=2018-04-04"; + + public List fetchPolicyStatesDetails(SubscriptionVH subscription, + List policyDefinitionList) throws Exception { + + List policyStatesList = new ArrayList(); + String accessToken; + try { + accessToken = AzureCredentialManager.getAuthToken(); + + } catch (Exception e1) { + return policyStatesList; + } + + String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); + try { + + String response = CommonUtils.doHttpPost(url, "Bearer", accessToken); + JsonObject responseObj = new JsonParser().parse(response).getAsJsonObject(); + JsonArray policyStatesObjects = responseObj.getAsJsonArray("value"); + for (JsonElement policyStatesElement : policyStatesObjects) { + PolicyStatesVH policyStatesVH = new PolicyStatesVH(); + JsonObject policyStatesObject = policyStatesElement.getAsJsonObject(); + PolicyDefinitionVH PolicyDefinitionVH = policyDefinitionList.stream() + .filter(policyDefinitionObj -> policyDefinitionObj.getName() + .equals(policyStatesObject.get("policyDefinitionName").getAsString())) + .findFirst().get(); + policyStatesVH.setPolicyDescription(PolicyDefinitionVH.getDescription()); + policyStatesVH.setPolicyName(PolicyDefinitionVH.getDisplayName()); + policyStatesVH.setPolicyType(PolicyDefinitionVH.getPolicyType()); + policyStatesVH.setPolicyRule(PolicyDefinitionVH.getPolicyRule()); + policyStatesVH.setTimestamp(policyStatesObject.get("timestamp").getAsString()); + policyStatesVH.setId(policyStatesObject.get("resourceId").getAsString()); + policyStatesVH.setResourceId(policyStatesObject.get("resourceId").getAsString()); + policyStatesVH.setPolicyAssignmentId(policyStatesObject.get("policyAssignmentId").getAsString()); + policyStatesVH.setPolicyDefinitionId(policyStatesObject.get("policyDefinitionId").getAsString()); + policyStatesVH.setEffectiveParameters(policyStatesObject.get("effectiveParameters").getAsString()); + policyStatesVH.setIsCompliant(policyStatesObject.get("isCompliant").getAsBoolean()); + policyStatesVH.setSubscriptionId(policyStatesObject.get("subscriptionId").getAsString()); + policyStatesVH.setResourceType(policyStatesObject.get("resourceType").getAsString()); + policyStatesVH.setResourceLocation(policyStatesObject.get("resourceLocation").getAsString()); + policyStatesVH.setResourceGroup(policyStatesObject.get("resourceGroup").getAsString()); + policyStatesVH.setResourceTags(policyStatesObject.get("resourceTags").getAsString()); + policyStatesVH.setPolicyAssignmentName(policyStatesObject.get("policyAssignmentName").getAsString()); + policyStatesVH.setPolicyAssignmentOwner(policyStatesObject.get("policyAssignmentOwner").getAsString()); + policyStatesVH.setPolicyAssignmentParameters( + policyStatesObject.get("policyAssignmentParameters").getAsString()); + policyStatesVH.setPolicyAssignmentScope(policyStatesObject.get("policyAssignmentScope").getAsString()); + policyStatesVH.setPolicyDefinitionName(policyStatesObject.get("policyDefinitionName").getAsString()); + policyStatesVH + .setPolicyDefinitionAction(policyStatesObject.get("policyDefinitionAction").getAsString()); + policyStatesVH + .setPolicyDefinitionCategory(policyStatesObject.get("policyDefinitionCategory").getAsString()); + policyStatesVH.setPolicySetDefinitionId(policyStatesObject.get("policySetDefinitionId").getAsString()); + policyStatesVH + .setPolicySetDefinitionName(policyStatesObject.get("policySetDefinitionName").getAsString()); + policyStatesVH + .setPolicySetDefinitionOwner(policyStatesObject.get("policySetDefinitionOwner").getAsString()); + policyStatesVH.setPolicySetDefinitionCategory( + policyStatesObject.get("policySetDefinitionCategory").getAsString()); + policyStatesVH.setPolicySetDefinitionParameters( + policyStatesObject.get("policySetDefinitionParameters").getAsString()); + policyStatesVH.setManagementGroupIds(policyStatesObject.get("managementGroupIds").getAsString()); + policyStatesVH.setPolicyDefinitionReferenceId( + policyStatesObject.get("policyDefinitionReferenceId").getAsString()); + + policyStatesVH.setSubscription(subscription.getSubscriptionId()); + policyStatesVH.setSubscriptionName(subscription.getSubscriptionName()); + + policyStatesList.add(policyStatesVH); + } + } catch (Exception e) { + e.printStackTrace(); + } + + System.out.println(policyStatesList.size()); + return policyStatesList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PostgreSQLInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PostgreSQLInventoryCollector.java new file mode 100644 index 000000000..2394b310d --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PostgreSQLInventoryCollector.java @@ -0,0 +1,70 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.springframework.stereotype.Component; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.microsoft.azure.management.Azure; +import com.tmobile.pacbot.azure.inventory.vo.PostgreSQLServerVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class PostgreSQLInventoryCollector { + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.DBforPostgreSQL/servers?api-version=2017-12-01"; + + public List fetchPostgreSQLServerDetails(SubscriptionVH subscription) { + + List postgreSQLServerList = new ArrayList(); + String accessToken; + try { + accessToken = AzureCredentialManager.getAuthToken(); + } catch (Exception e1) { + return postgreSQLServerList; + } + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + + String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); + try { + String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); + JsonObject responseObj = new JsonParser().parse(response).getAsJsonObject(); + JsonArray postgreSQLServerObjects = responseObj.getAsJsonArray("value"); + for (JsonElement postgreSQLServerObjectElement : postgreSQLServerObjects) { + PostgreSQLServerVH postgreSQLServerVH = new PostgreSQLServerVH(); + postgreSQLServerVH.setSubscription(subscription.getSubscriptionId()); + postgreSQLServerVH.setSubscriptionName(subscription.getSubscriptionName()); + JsonObject postgreSQLServerObject = postgreSQLServerObjectElement.getAsJsonObject(); + JsonObject properties = postgreSQLServerObject.getAsJsonObject("properties"); + JsonObject sku = postgreSQLServerObject.getAsJsonObject("sku"); + postgreSQLServerVH.setId(postgreSQLServerObject.get("id").getAsString()); + postgreSQLServerVH.setLocation(postgreSQLServerObject.get("location").getAsString()); + postgreSQLServerVH.setName(postgreSQLServerObject.get("name").getAsString()); + postgreSQLServerVH.setType(postgreSQLServerObject.get("type").getAsString()); + if (sku!=null) { + HashMap skuMap = new Gson().fromJson(sku.toString(), HashMap.class); + postgreSQLServerVH.setSkuMap(skuMap); + } + if (properties!=null) { + HashMap propertiesMap = new Gson().fromJson(properties.toString(), HashMap.class); + postgreSQLServerVH.setPropertiesMap(propertiesMap); + } + postgreSQLServerList.add(postgreSQLServerVH); + } + } catch (Exception e) { + e.printStackTrace(); + } + + System.out.println(postgreSQLServerList.size()); + return postgreSQLServerList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PublicIpAddressInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PublicIpAddressInventoryCollector.java new file mode 100644 index 000000000..63d0016cb --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PublicIpAddressInventoryCollector.java @@ -0,0 +1,51 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Component; + +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.network.PublicIPAddress; +import com.tmobile.pacbot.azure.inventory.vo.PublicIpAddressVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; + +@Component +public class PublicIpAddressInventoryCollector { + + public List fetchPublicIpAddressDetails(SubscriptionVH subscription, + Map> tagMap) { + + List publicIpAddressList = new ArrayList(); + + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + PagedList publicIPAddresses = azure.publicIPAddresses().list(); + System.out.println(publicIPAddresses.size()); + for (PublicIPAddress publicIPAddress : publicIPAddresses) { + PublicIpAddressVH publicIpAddressVH = new PublicIpAddressVH(); + publicIpAddressVH.setId(publicIPAddress.id()); + publicIpAddressVH.setName(publicIPAddress.name()); + publicIpAddressVH.setResourceGroupName(publicIPAddress.resourceGroupName()); + publicIpAddressVH.setType(publicIPAddress.type()); + publicIpAddressVH + .setTags(Util.tagsList(tagMap, publicIPAddress.resourceGroupName(), publicIPAddress.tags())); + publicIpAddressVH.setSubscription(subscription.getSubscriptionId()); + publicIpAddressVH.setSubscriptionName(subscription.getSubscriptionName()); + publicIpAddressVH.setIdleTimeoutInMinutes(publicIPAddress.idleTimeoutInMinutes()); + publicIpAddressVH.setFqdn(publicIPAddress.fqdn()); + publicIpAddressVH.setIpAddress(publicIPAddress.ipAddress()); + publicIpAddressVH.setKey(publicIPAddress.key()); + publicIpAddressVH.setRegionName(publicIPAddress.regionName()); + publicIpAddressVH.setReverseFqdn(publicIPAddress.reverseFqdn()); + publicIpAddressVH.setVersion(publicIPAddress.version().toString()); + publicIpAddressList.add(publicIpAddressVH); + + } + + return publicIpAddressList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/RegisteredApplicationInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/RegisteredApplicationInventoryCollector.java new file mode 100644 index 000000000..5145c46dc --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/RegisteredApplicationInventoryCollector.java @@ -0,0 +1,161 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.tmobile.pacbot.azure.inventory.vo.RegAppCertificateVH; +import com.tmobile.pacbot.azure.inventory.vo.RegAppSecretVH; +import com.tmobile.pacbot.azure.inventory.vo.RegisteredApplicationVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class RegisteredApplicationInventoryCollector { + + // constans for API data + private static final String VALUE = "value"; + private static final String NEXT_DATASET = "@odata.nextLink"; + + // constants for the application details + private static final String OBJECT_ID = "id"; + private static final String APP_ID = "appId"; + private static final String CREATE_DATETIME = "createdDateTime"; + private static final String DISPLAYNAME = "displayName"; + private static final String PUBLISHERDOMAIN = "publisherDomain"; + private static final String CERTIFICATE_DATA = "keyCredentials"; + private static final String SECRET_DATA = "passwordCredentials"; + + // constants for secret keys + private static final String CUSTOM_KEY_IDENTIFIER = "customKeyIdentifier"; + private static final String END_DATETIME = "endDateTime"; + private static final String START_DATETIME = "startDateTime"; + private static final String KEY_ID = "keyId"; + private static final String SECRET_TEXT = "secretText"; + private static final String HINT = "hint"; + + // constants for certificate data + private static final String TYPE = "type"; + private static final String USAGE = "usage"; + private static final String KEY = "key"; + + private static final String API_URL_TEMPLATE = "https://graph.microsoft.com/beta/applications"; + private static final String TOKEN_TYPE = "Bearer"; + + public List fetchAzureRegisteredApplication() { + List registeredApplicationList = new ArrayList<>(); + String accessToken; + try { + accessToken = AzureCredentialManager.getGraphApiAuthToken(); + } catch (Exception e1) { + return registeredApplicationList; + } + + String url = API_URL_TEMPLATE; + try { + do { + String registeredApplicationString = CommonUtils.doHttpGet(url, TOKEN_TYPE, accessToken); + JsonObject responseObj = new JsonParser().parse(registeredApplicationString).getAsJsonObject(); + registeredApplicationList.addAll(createRegisteredApplicationInfo(responseObj.getAsJsonArray(VALUE))); + url = responseObj.has(NEXT_DATASET) ? responseObj.get(NEXT_DATASET).getAsString() : null; + } while (!StringUtils.isEmpty(url)); + System.out.println("Registered Application Collected " + registeredApplicationList.size()); + } catch (Exception e) { + // TODO Auto-generated catch block + System.out.println("Error in collecting Registered application list"); + e.printStackTrace(); + } + return registeredApplicationList; + + } + + /** + * To create the list of register applications fromt the register application json + * @param registeredApplicationJsonArray + * @return + */ + private List createRegisteredApplicationInfo(JsonArray registeredApplicationJsonArray) { + + List registeredApplicationList = new ArrayList<>(); + for (JsonElement registeredApplicationElement : registeredApplicationJsonArray) { + + RegisteredApplicationVH registeredApplication = new RegisteredApplicationVH(); + JsonObject registeredApplicationInfo = registeredApplicationElement.getAsJsonObject(); + + registeredApplication.setObjectId(getStringValueforJsonElement(registeredApplicationInfo.get(OBJECT_ID))); + registeredApplication.setAppId(getStringValueforJsonElement(registeredApplicationInfo.get(APP_ID))); + registeredApplication.setCreatedDateTime(getStringValueforJsonElement(registeredApplicationInfo.get(CREATE_DATETIME))); + registeredApplication.setDisplayName(getStringValueforJsonElement(registeredApplicationInfo.get(DISPLAYNAME))); + registeredApplication.setPublisherDomain(getStringValueforJsonElement(registeredApplicationInfo.get(PUBLISHERDOMAIN))); + registeredApplication.setCertificateList(createRegisterApplicationCertificateList( + registeredApplicationInfo.getAsJsonArray(CERTIFICATE_DATA))); + registeredApplication.setSecretList(createRegisterApplicationSecretList( + registeredApplicationInfo.getAsJsonArray(SECRET_DATA))); + + registeredApplicationList.add(registeredApplication); + } + return registeredApplicationList; + } + + /** + * to create the register application certificate list for an application from the certificate json + * @param certificateJsonArray + * @return + */ + private List createRegisterApplicationCertificateList(JsonArray certificateJsonArray) { + List regAppCertificateList = new ArrayList<>(); + + for (JsonElement certificateJsonElement : certificateJsonArray) { + RegAppCertificateVH regAppCertificate = new RegAppCertificateVH(); + JsonObject regAppCertificateJsonObject = certificateJsonElement.getAsJsonObject(); + + regAppCertificate.setCustomKeyIdentifier(getStringValueforJsonElement(regAppCertificateJsonObject.get(CUSTOM_KEY_IDENTIFIER))); + regAppCertificate.setEndDateTime(getStringValueforJsonElement(regAppCertificateJsonObject.get(END_DATETIME))); + regAppCertificate.setDisplayName(getStringValueforJsonElement(regAppCertificateJsonObject.get(DISPLAYNAME))); + regAppCertificate.setKey(getStringValueforJsonElement(regAppCertificateJsonObject.get(KEY))); + regAppCertificate.setStartDateTime(getStringValueforJsonElement(regAppCertificateJsonObject.get(START_DATETIME))); + regAppCertificate.setType(getStringValueforJsonElement(regAppCertificateJsonObject.get(TYPE))); + regAppCertificate.setUsage(getStringValueforJsonElement(regAppCertificateJsonObject.get(USAGE))); + regAppCertificate.setKeyId(getStringValueforJsonElement(regAppCertificateJsonObject.get(KEY_ID))); + + regAppCertificateList.add(regAppCertificate); + } + + return regAppCertificateList; + } + + /** + * to create the register application secret list for an application from the secret json + * @param secretJsonArray + * @return + */ + private List createRegisterApplicationSecretList(JsonArray secretJsonArray) { + List regAppSecretList = new ArrayList<>(); + + for (JsonElement secretJsonElement : secretJsonArray) { + RegAppSecretVH regAppSecret = new RegAppSecretVH(); + JsonObject regAppSecretJsonObject = secretJsonElement.getAsJsonObject(); + + regAppSecret.setCustomKeyIdentifier(getStringValueforJsonElement(regAppSecretJsonObject.get(CUSTOM_KEY_IDENTIFIER))); + regAppSecret.setDisplayName(getStringValueforJsonElement(regAppSecretJsonObject.get(DISPLAYNAME))); + regAppSecret.setEndDateTime(getStringValueforJsonElement(regAppSecretJsonObject.get(END_DATETIME))); + regAppSecret.setHint(getStringValueforJsonElement(regAppSecretJsonObject.get(HINT))); + regAppSecret.setKeyId(getStringValueforJsonElement(regAppSecretJsonObject.get(KEY_ID))); + regAppSecret.setSecretText(getStringValueforJsonElement(regAppSecretJsonObject.get(SECRET_TEXT))); + regAppSecret.setStartDateTime(getStringValueforJsonElement(regAppSecretJsonObject.get(START_DATETIME))); + + regAppSecretList.add(regAppSecret); + } + return regAppSecretList; + } + + private String getStringValueforJsonElement (JsonElement jsonElement) { + return jsonElement.isJsonNull() ? null : jsonElement.getAsString(); + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/ResourceGroupInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/ResourceGroupInventoryCollector.java new file mode 100644 index 000000000..306075a9e --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/ResourceGroupInventoryCollector.java @@ -0,0 +1,39 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Component; + +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.resources.ResourceGroup; +import com.tmobile.pacbot.azure.inventory.vo.ResourceGroupVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; + +@Component +public class ResourceGroupInventoryCollector { + public List fetchResourceGroupDetails(SubscriptionVH subscription) { + List resourceGroupList = new ArrayList(); + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + PagedList resourceGroups = azure.resourceGroups().list(); + System.out.println(resourceGroups.size()); + for (ResourceGroup resourceGroup : resourceGroups) { + ResourceGroupVH resourceGroupVH = new ResourceGroupVH(); + resourceGroupVH.setSubscription(subscription.getSubscriptionId()); + resourceGroupVH.setSubscriptionName(subscription.getSubscriptionName()); + resourceGroupVH.setId(resourceGroup.id()); + resourceGroupVH.setResourceGroupName(resourceGroup.name()); + resourceGroupVH.setKey(resourceGroup.key()); + resourceGroupVH.setType(resourceGroup.type()); + resourceGroupVH.setProvisioningState(resourceGroup.provisioningState()); + resourceGroupVH.setRegionName(resourceGroup.regionName()); + resourceGroupVH.setTags(resourceGroup.tags()); + resourceGroupList.add(resourceGroupVH); + } + return resourceGroupList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/RouteTableInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/RouteTableInventoryCollector.java new file mode 100644 index 000000000..258c7e747 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/RouteTableInventoryCollector.java @@ -0,0 +1,79 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Component; + +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.network.Route; +import com.microsoft.azure.management.network.RouteTable; +import com.microsoft.azure.management.network.Subnet; +import com.tmobile.pacbot.azure.inventory.vo.RouteTableSubnet; +import com.tmobile.pacbot.azure.inventory.vo.RouteTableVH; +import com.tmobile.pacbot.azure.inventory.vo.RouteVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; + +@Component +public class RouteTableInventoryCollector { + + public List fetchRouteTableDetails(SubscriptionVH subscription, + Map> tagMap) { + List routeTableDetailsList = new ArrayList(); + + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + PagedList routTableList = azure.routeTables().list(); + System.out.println(routTableList.size()); + for (RouteTable routTable : routTableList) { + RouteTableVH routeTableVH = new RouteTableVH(); + routeTableVH.setHashCode(routTable.hashCode()); + routeTableVH.setId(routTable.id()); + routeTableVH.setKey(routTable.key()); + routeTableVH.setName(routTable.name()); + routeTableVH.setRegionName(routTable.regionName()); + routeTableVH.setResourceGroupName(routTable.resourceGroupName()); + routeTableVH.setTags(Util.tagsList(tagMap, routTable.resourceGroupName(), routTable.tags())); + routeTableVH.setSubnetList(getNetworkSecuritySubnetDetails(routTable.listAssociatedSubnets())); + routeTableVH.setType(routTable.type()); + getRouteDetails(routTable.routes(), routeTableVH); + routeTableVH.setSubscription(subscription.getSubscriptionId()); + routeTableVH.setSubscriptionName(subscription.getSubscriptionName()); + routeTableDetailsList.add(routeTableVH); + + } + + return routeTableDetailsList; + } + + private void getRouteDetails(Map routeDetails, RouteTableVH routeTableVH) { + List routeVHlist = new ArrayList<>(); + for (Map.Entry entry : routeDetails.entrySet()) { + RouteVH routeVH = new RouteVH(); + routeVH.setAddressPrefix(entry.getValue().destinationAddressPrefix()); + routeVH.setName(entry.getValue().name()); + routeVH.setNextHop(entry.getValue().nextHopType().toString()); + routeVHlist.add(routeVH); + } + + routeTableVH.setRouteVHlist(routeVHlist); + + } + + private List getNetworkSecuritySubnetDetails(List subnetList) { + List subnetVHlist = new ArrayList<>(); + for (Subnet subnet : subnetList) { + RouteTableSubnet routeTableSubnet = new RouteTableSubnet(); + routeTableSubnet.setAddressPrefix(subnet.addressPrefix()); + routeTableSubnet.setName(subnet.name()); + routeTableSubnet.setVnet(subnet.parent().id()); + subnetVHlist.add(routeTableSubnet); + + } + return subnetVHlist; + + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SCRecommendationsCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SCRecommendationsCollector.java new file mode 100644 index 000000000..a1cfe6e67 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SCRecommendationsCollector.java @@ -0,0 +1,93 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.springframework.stereotype.Component; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.reflect.TypeToken; +import com.tmobile.pacbot.azure.inventory.vo.RecommendationVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class SCRecommendationsCollector { + + Set policyList = new HashSet<>(); + Set nameList = new HashSet<>(); + Set baseNameList = new HashSet<>(); + + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.Security/tasks?api-version=2015-06-01-preview"; + public List fetchSecurityCenterRecommendations(SubscriptionVH subscription) { + List recommendations = new ArrayList<>(); + String accessToken; + try { + accessToken = AzureCredentialManager.getAuthToken(); + } catch (Exception e1) { + return recommendations; + } + String url = String.format(apiUrlTemplate, subscription.getSubscriptionId()); + + try { + String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); + recommendations = filterRecommendationInfo(response,subscription); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return recommendations; + + } + + private List filterRecommendationInfo(String response,SubscriptionVH subscription){ + + List recommendations = new ArrayList<>(); + JsonObject responseObj = new JsonParser().parse(response).getAsJsonObject(); + JsonArray recommedationObjects = responseObj.getAsJsonArray("value"); + + for(JsonElement recElmnt : recommedationObjects) { + JsonObject recommendObject = recElmnt.getAsJsonObject(); + JsonObject properties = recommendObject.getAsJsonObject("properties"); + String id = recommendObject.get("id").getAsString(); + if("Active".equals(properties.get("state").getAsString())){ + JsonObject secTaskParameters = properties.getAsJsonObject("securityTaskParameters"); + //String baseLineName = secTaskParameters.get("baselineName")!=null?secTaskParameters.get("baselineName").getAsString():null; + String policyName = secTaskParameters.get("policyName")!=null?secTaskParameters.get("policyName").getAsString():null; + //String name = secTaskParameters.get("name")!=null?secTaskParameters.get("name").getAsString():null; + String resourceType = secTaskParameters.get("resourceType")!=null?secTaskParameters.get("resourceType").getAsString():""; + + if(policyName !=null && "VirtualMachine".equals(resourceType)) { + + + Map recommendationMap = new Gson().fromJson(secTaskParameters, new TypeToken>() {}.getType() ); + Object resourceId = recommendationMap.get("resourceId"); + if(resourceId!=null) { + RecommendationVH recommendation = new RecommendationVH(); + recommendation.setSubscription(subscription.getSubscriptionId()); + recommendation.setSubscriptionName(subscription.getSubscriptionName()); + recommendationMap.put("resourceId",Util.removeFirstSlash(resourceId.toString())); + recommendationMap.put("_resourceIdLower",Util.removeFirstSlash(resourceId.toString()).toLowerCase()); + recommendation.setId(id); + recommendation.setRecommendation(recommendationMap); + recommendations.add(recommendation); + } + + } + + } + } + + return recommendations; + + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLDatabaseInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLDatabaseInventoryCollector.java new file mode 100644 index 000000000..ec8a37e7e --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLDatabaseInventoryCollector.java @@ -0,0 +1,140 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Component; + +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.sql.SqlDatabase; +import com.microsoft.azure.management.sql.SqlFirewallRule; +import com.microsoft.azure.management.sql.SqlServer; +import com.microsoft.azure.management.sql.SqlVirtualNetworkRule; +import com.tmobile.pacbot.azure.inventory.vo.FirewallRules; +import com.tmobile.pacbot.azure.inventory.vo.SQLDatabaseVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacbot.azure.inventory.vo.VirtualNetworkRuleVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; + +@Component +public class SQLDatabaseInventoryCollector { + + public List fetchSQLDatabaseDetails(SubscriptionVH subscription, + Map> tagMap) { + + List sqlDatabaseList = new ArrayList(); + + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + PagedList sqlServers = azure.sqlServers().list(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); + for (SqlServer sqlServer : sqlServers) { + List sqlDatabases = azure.sqlServers().databases().listBySqlServer(sqlServer); + for (SqlDatabase sqlDatabase : sqlDatabases) { + if (!sqlDatabase.name().contentEquals("master")) { + SQLDatabaseVH sqlDatabaseVH = new SQLDatabaseVH(); + if (sqlDatabase.creationDate() != null) { + + sqlDatabaseVH.setCreationDate(sdf.format(sqlDatabase.creationDate().toDate())); + } + sqlDatabaseVH.setCurrentServiceObjectiveId(sqlDatabase.currentServiceObjectiveId()); + sqlDatabaseVH.setCollation(sqlDatabase.collation()); + sqlDatabaseVH.setDatabaseId(sqlDatabase.databaseId()); + sqlDatabaseVH.setDefaultSecondaryLocation(sqlDatabase.defaultSecondaryLocation()); + if (sqlDatabase.earliestRestoreDate() != null) { + + sqlDatabaseVH.setEarliestRestoreDate(sdf.format(sqlDatabase.earliestRestoreDate().toDate())); + } + sqlDatabaseVH.setEdition(sqlDatabase.edition().toString()); + sqlDatabaseVH.setElasticPoolName(sqlDatabase.elasticPoolName()); + sqlDatabaseVH.setId(sqlDatabase.id()); + sqlDatabaseVH.setDataWarehouse(sqlDatabase.isDataWarehouse()); + sqlDatabaseVH.setName(sqlDatabase.name()); + sqlDatabaseVH.setStatus(sqlDatabase.status()); + sqlDatabaseVH.setSubscription(subscription.getSubscriptionId()); + sqlDatabaseVH.setSubscriptionName(subscription.getSubscriptionName()); + sqlDatabaseVH.setServerName(sqlDatabase.sqlServerName()); + sqlDatabaseVH.setResourceGroupName(sqlDatabase.resourceGroupName()); + + for (Map.Entry> resourceGroupTag : tagMap.entrySet()) { + + if (resourceGroupTag.getKey().equalsIgnoreCase(sqlDatabase.resourceGroupName())) { + sqlDatabaseVH.setTags(resourceGroupTag.getValue()); + break; + } + + } + + firewallRule(sqlServer, sqlDatabaseVH); + // sqlDatabaseVH.setFirewallRuleDetails(getFirewallRuleDetails(sqlServer.firewallRules().list())); + // sqlDatabaseVH.setVirtualNetworkRuleDetails( + // getVirtualNetworkRuleDetails(sqlServer.virtualNetworkRules().list())); + sqlDatabaseList.add(sqlDatabaseVH); + } + + } + + } + return sqlDatabaseList; + + } + + /* + * private List getFirewallRuleDetails(List + * sqlFirewallRuleList) { List firewallRulesList = new + * ArrayList<>(); for (SqlFirewallRule sqlFirewallRule : sqlFirewallRuleList) { + * FirewallRules firewallRuleVH = new FirewallRules(); + * firewallRuleVH.setName(sqlFirewallRule.name()); + * firewallRuleVH.setStartIPAddress(sqlFirewallRule.startIPAddress()); + * firewallRuleVH.setEndIPAddress(sqlFirewallRule.endIPAddress()); + * firewallRulesList.add(firewallRuleVH); } return firewallRulesList; + * + * } + * + * private List getVirtualNetworkRuleDetails( + * List sqlVirtualNetworkRuleList) { + * List virtualNetworkRuleList = new ArrayList<>(); for + * (SqlVirtualNetworkRule sqlVirtualNetworkRule : sqlVirtualNetworkRuleList) { + * VirtualNetworkRule virtualNetworkRuleVH = new VirtualNetworkRule(); + * virtualNetworkRuleVH.setName(sqlVirtualNetworkRule.name()); + * virtualNetworkRuleVH.setSubnetId(sqlVirtualNetworkRule.subnetId()); + * virtualNetworkRuleVH.setResourceGroupName(sqlVirtualNetworkRule. + * resourceGroupName()); + * virtualNetworkRuleVH.setState(sqlVirtualNetworkRule.state()); + * virtualNetworkRuleList.add(virtualNetworkRuleVH); } return + * virtualNetworkRuleList; + * + * } + */ + + private void firewallRule(SqlServer sqlServer, SQLDatabaseVH sqlDatabaseVH) { + List> firewallRuleList = new ArrayList<>(); + Map firewallMap; + for (SqlFirewallRule sqlFirewallRule : sqlServer.firewallRules().list()) { + firewallMap = new HashMap<>(); + firewallMap.put("name", sqlFirewallRule.name()); + firewallMap.put("startIPAddress", sqlFirewallRule.startIPAddress()); + firewallMap.put("endIPAddress", sqlFirewallRule.endIPAddress()); + firewallRuleList.add(firewallMap); + + } + for (SqlVirtualNetworkRule sqlVirtualNetworkRule : sqlServer.virtualNetworkRules().list()) { + firewallMap = new HashMap<>(); + + firewallMap.put("virtualNetworkRuleName", + sqlVirtualNetworkRule.name() != null ? sqlVirtualNetworkRule.name() : ""); + firewallMap.put("virtualNetworkSubnetId", + sqlVirtualNetworkRule.subnetId() != null ? sqlVirtualNetworkRule.subnetId() : ""); + firewallMap.put("virtualNetworkResourceGroupName", + sqlVirtualNetworkRule.resourceGroupName() != null ? sqlVirtualNetworkRule.resourceGroupName() : ""); + firewallMap.put("virtualNetworkState", + sqlVirtualNetworkRule.state() != null ? sqlVirtualNetworkRule.state() : ""); + + firewallRuleList.add(firewallMap); + } + sqlDatabaseVH.setFirewallRuleDetails(firewallRuleList); + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLServerInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLServerInventoryCollector.java new file mode 100644 index 000000000..2ff114552 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLServerInventoryCollector.java @@ -0,0 +1,121 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Component; + +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.sql.SqlElasticPool; +import com.microsoft.azure.management.sql.SqlFailoverGroup; +import com.microsoft.azure.management.sql.SqlFirewallRule; +import com.microsoft.azure.management.sql.SqlServer; +import com.microsoft.azure.management.sql.SqlVirtualNetworkRule; +import com.tmobile.pacbot.azure.inventory.vo.ElasticPoolVH; +import com.tmobile.pacbot.azure.inventory.vo.FailoverGroupVH; +import com.tmobile.pacbot.azure.inventory.vo.SQLServerVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; + +@Component +public class SQLServerInventoryCollector { + public List fetchSQLServerDetails(SubscriptionVH subscription, + Map> tagMap) { + + List sqlServerList = new ArrayList(); + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + PagedList sqlServers = azure.sqlServers().list(); + for (SqlServer sqlServer : sqlServers) { + SQLServerVH sqlServerVH = new SQLServerVH(); + sqlServerVH.setSubscription(subscription.getSubscriptionId()); + sqlServerVH.setSubscriptionName(subscription.getSubscriptionName()); + sqlServerVH.setId(sqlServer.id()); + sqlServerVH.setKind(sqlServer.kind()); + sqlServerVH.setName(sqlServer.name()); + sqlServerVH.setRegionName(sqlServer.regionName()); + sqlServerVH.setState(sqlServer.state()); + sqlServerVH.setSystemAssignedManagedServiceIdentityPrincipalId( + sqlServer.systemAssignedManagedServiceIdentityPrincipalId()); + sqlServerVH.setSystemAssignedManagedServiceIdentityTenantId( + sqlServer.systemAssignedManagedServiceIdentityTenantId()); + sqlServerVH.setTags(Util.tagsList(tagMap, sqlServer.resourceGroupName(), sqlServer.tags())); + sqlServerVH.setVersion(sqlServer.version()); + sqlServerVH.setAdministratorLogin(sqlServer.administratorLogin()); + firewallRule(sqlServer, sqlServerVH); + getElasticPoolList(sqlServer.elasticPools().list(), sqlServerVH); + getFailoverGroupList(sqlServer.failoverGroups().list(), sqlServerVH); + sqlServerList.add(sqlServerVH); + } + + return sqlServerList; + + } + + private void getElasticPoolList(List sqlElasticPoolList, SQLServerVH sqlServerVH) { + List elasticPoolList = new ArrayList<>(); + for (SqlElasticPool sqlElasticPool : sqlElasticPoolList) { + ElasticPoolVH elasticPoolVH = new ElasticPoolVH(); + elasticPoolVH.setName(sqlElasticPool.name()); + elasticPoolVH.setSize(sqlElasticPool.listDatabases().size()); + elasticPoolVH.setStorageCapacity(sqlElasticPool.storageCapacityInMB()); + elasticPoolVH.setId(sqlElasticPool.id()); + elasticPoolVH.setStorageMB(sqlElasticPool.storageMB()); + elasticPoolVH.setDtu(sqlElasticPool.dtu()); + elasticPoolVH.setEdition(sqlElasticPool.edition().toString()); + elasticPoolList.add(elasticPoolVH); + + } + sqlServerVH.setElasticPoolList(elasticPoolList); + + } + + private void firewallRule(SqlServer sqlServer, SQLServerVH sqlServerVH) { + List> firewallRuleList = new ArrayList<>(); + Map firewallMap; + for (SqlFirewallRule sqlFirewallRule : sqlServer.firewallRules().list()) { + firewallMap = new HashMap<>(); + firewallMap.put("name", sqlFirewallRule.name()); + firewallMap.put("startIPAddress", sqlFirewallRule.startIPAddress()); + firewallMap.put("endIPAddress", sqlFirewallRule.endIPAddress()); + firewallRuleList.add(firewallMap); + + } + for (SqlVirtualNetworkRule sqlVirtualNetworkRule : sqlServer.virtualNetworkRules().list()) { + firewallMap = new HashMap<>(); + + firewallMap.put("virtualNetworkRuleName", + sqlVirtualNetworkRule.name() != null ? sqlVirtualNetworkRule.name() : ""); + firewallMap.put("virtualNetworkSubnetId", + sqlVirtualNetworkRule.subnetId() != null ? sqlVirtualNetworkRule.subnetId() : ""); + firewallMap.put("virtualNetworkResourceGroupName", + sqlVirtualNetworkRule.resourceGroupName() != null ? sqlVirtualNetworkRule.resourceGroupName() : ""); + firewallMap.put("virtualNetworkState", + sqlVirtualNetworkRule.state() != null ? sqlVirtualNetworkRule.state() : ""); + + firewallRuleList.add(firewallMap); + } + sqlServerVH.setFirewallRuleDetails(firewallRuleList); + } + + private void getFailoverGroupList(List sqlFailoverGroupList, SQLServerVH sqlServerVH) { + List failoverGroupList = new ArrayList<>(); + for (SqlFailoverGroup sqlFailoverGroup : sqlFailoverGroupList) { + FailoverGroupVH failoverGroupVH = new FailoverGroupVH(); + failoverGroupVH.setSize(sqlFailoverGroup.databases().size()); + failoverGroupVH.setId(sqlFailoverGroup.id()); + failoverGroupVH.setName(sqlFailoverGroup.name()); + failoverGroupVH.setReplicationState(sqlFailoverGroup.replicationState()); + failoverGroupVH.setReadOnlyEndpointPolicy(sqlFailoverGroup.readOnlyEndpointPolicy().toString()); + failoverGroupVH.setReadWriteEndpointPolicy(sqlFailoverGroup.readWriteEndpointPolicy().toString()); + failoverGroupVH.setGracePeriod(sqlFailoverGroup.readWriteEndpointDataLossGracePeriodMinutes()); + failoverGroupList.add(failoverGroupVH); + + } + sqlServerVH.setFailoverGroupList(failoverGroupList); + + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SearchServiceInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SearchServiceInventoryCollector.java new file mode 100644 index 000000000..be6b9af52 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SearchServiceInventoryCollector.java @@ -0,0 +1,79 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.tmobile.pacbot.azure.inventory.vo.SearchServiceVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class SearchServiceInventoryCollector { + + private static Logger log = LoggerFactory.getLogger(SearchServiceInventoryCollector.class); + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.Search/searchServices?api-version=2015-08-19"; + + public List fetchSearchServiceDetails(SubscriptionVH subscription) throws Exception { + + List searchServiceList = new ArrayList(); + String accessToken; + try { + accessToken = AzureCredentialManager.getAuthToken(); + + } catch (Exception e1) { + return searchServiceList; + } + + String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); + try { + + String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); + JsonObject responseObj = new JsonParser().parse(response).getAsJsonObject(); + JsonArray searchServiceObjects = responseObj.getAsJsonArray("value"); + if (searchServiceObjects != null) { + for (JsonElement searchServiceElement : searchServiceObjects) { + SearchServiceVH searchServiceVH = new SearchServiceVH(); + JsonObject searchServiceObject = searchServiceElement.getAsJsonObject(); + searchServiceVH.setSubscription(subscription.getSubscriptionId()); + searchServiceVH.setSubscriptionName(subscription.getSubscriptionName()); + searchServiceVH.setId(searchServiceObject.get("id").getAsString()); + searchServiceVH.setLocation(searchServiceObject.get("location").getAsString()); + searchServiceVH.setName(searchServiceObject.get("name").getAsString()); + searchServiceVH.setType(searchServiceObject.get("type").getAsString()); + JsonObject properties = searchServiceObject.getAsJsonObject("properties"); + JsonObject sku = searchServiceObject.getAsJsonObject("sku"); + if (properties != null) { + HashMap propertiesMap = new Gson().fromJson(properties.toString(), + HashMap.class); + searchServiceVH.setProperties(propertiesMap); + } + + if (sku != null) { + HashMap skuMap = new Gson().fromJson(sku.toString(), HashMap.class); + searchServiceVH.setSku(skuMap); + } + + searchServiceList.add(searchServiceVH); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + System.out.println(searchServiceList.size()); + return searchServiceList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SecurityAlertsInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SecurityAlertsInventoryCollector.java new file mode 100644 index 000000000..49fcbd234 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SecurityAlertsInventoryCollector.java @@ -0,0 +1,67 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.springframework.stereotype.Component; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.microsoft.azure.management.Azure; +import com.tmobile.pacbot.azure.inventory.vo.DatabricksVH; +import com.tmobile.pacbot.azure.inventory.vo.SecurityAlertsVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class SecurityAlertsInventoryCollector { + + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.Security/alerts?api-version=2019-01-01"; + + public List fetchSecurityAlertsDetails(SubscriptionVH subscription) throws Exception { + + List securityAlertsList = new ArrayList(); + String accessToken; + try { + accessToken = AzureCredentialManager.getAuthToken(); + + } catch (Exception e1) { + return securityAlertsList; + } + + String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); + try { + String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); + JsonObject responseObj = new JsonParser().parse(response).getAsJsonObject(); + JsonArray securityAlertsObjects = responseObj.getAsJsonArray("value"); + for (JsonElement securityAlertsElement : securityAlertsObjects) { + SecurityAlertsVH securityAlertsVH = new SecurityAlertsVH(); + JsonObject databricksObject = securityAlertsElement.getAsJsonObject(); + JsonObject properties = databricksObject.getAsJsonObject("properties"); + securityAlertsVH.setId(databricksObject.get("id").getAsString()); + securityAlertsVH.setName(databricksObject.get("name").getAsString()); + securityAlertsVH.setType(databricksObject.get("type").getAsString()); + securityAlertsVH.setSubscription(subscription.getSubscriptionId()); + securityAlertsVH.setSubscriptionName(subscription.getSubscriptionName()); + + if (properties != null) { + HashMap propertiesMap = new Gson().fromJson(properties.toString(), HashMap.class); + securityAlertsVH.setPropertiesMap(propertiesMap); + } + securityAlertsList.add(securityAlertsVH); + } + } catch (Exception e) { + e.printStackTrace(); + } + + System.out.println(securityAlertsList.size()); + return securityAlertsList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SitesInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SitesInventoryCollector.java new file mode 100644 index 000000000..f333c023a --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SitesInventoryCollector.java @@ -0,0 +1,78 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.tmobile.pacbot.azure.inventory.vo.SitesVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class SitesInventoryCollector { + private static Logger log = LoggerFactory.getLogger(SitesInventoryCollector.class); + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.Network/vpnSites?api-version=2019-06-01"; + + public List fetchSitesDetails(SubscriptionVH subscription) throws Exception { + + List sitesList = new ArrayList(); + String accessToken; + try { + accessToken = AzureCredentialManager.getAuthToken(); + + } catch (Exception e1) { + return sitesList; + } + + String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); + try { + + String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); + JsonObject responseObj = new JsonParser().parse(response).getAsJsonObject(); + JsonArray sitesObjects = responseObj.getAsJsonArray("value"); + if (sitesObjects != null) { + for (JsonElement sitesElement : sitesObjects) { + SitesVH sitesVH = new SitesVH(); + JsonObject sitesObject = sitesElement.getAsJsonObject(); + sitesVH.setSubscription(subscription.getSubscriptionId()); + sitesVH.setSubscriptionName(subscription.getSubscriptionName()); + sitesVH.setId(sitesObject.get("id").getAsString()); + sitesVH.setEtag(sitesObject.get("etag").getAsString()); + sitesVH.setLocation(sitesObject.get("location").getAsString()); + sitesVH.setName(sitesObject.get("name").getAsString()); + sitesVH.setType(sitesObject.get("type").getAsString()); + JsonObject properties = sitesObject.getAsJsonObject("properties"); + JsonObject tags = sitesObject.getAsJsonObject("tags"); + if (properties!=null) { + HashMap propertiesMap = new Gson().fromJson(properties.toString(), HashMap.class); + sitesVH.setProperties(propertiesMap); + } + if (tags!=null) { + HashMap tagsMap = new Gson().fromJson(tags.toString(), HashMap.class); + sitesVH.setTags(tagsMap); + } + + + sitesList.add(sitesVH); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + System.out.println(sitesList.size()); + return sitesList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SnapshotInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SnapshotInventoryCollector.java new file mode 100644 index 000000000..620b92d37 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SnapshotInventoryCollector.java @@ -0,0 +1,44 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Component; + +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.compute.Snapshot; +import com.tmobile.pacbot.azure.inventory.vo.SnapshotVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; + +@Component +public class SnapshotInventoryCollector { + + public List fetchSnapshotDetails(SubscriptionVH subscription, Map> tagMap) { + List snapshotList = new ArrayList(); + + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + PagedList snapshots = azure.snapshots().list(); + System.out.println(snapshots.size()); + for (Snapshot snapshot : snapshots) { + SnapshotVH snapshotVH = new SnapshotVH(); + snapshotVH.setId(snapshot.id()); + snapshotVH.setName(snapshot.name()); + snapshotVH.setResourceGroupName(snapshot.resourceGroupName()); + snapshotVH.setType(snapshot.type()); + snapshotVH.setTags(Util.tagsList(tagMap, snapshot.resourceGroupName(), snapshot.tags())); + snapshotVH.setSubscription(subscription.getSubscriptionId()); + snapshotVH.setSubscriptionName(subscription.getSubscriptionName()); + snapshotVH.setKey(snapshot.key()); + snapshotVH.setRegionName(snapshot.regionName()); + snapshotVH.setSizeInGB(snapshot.sizeInGB()); + snapshotList.add(snapshotVH); + + } + + return snapshotList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/StorageAccountInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/StorageAccountInventoryCollector.java new file mode 100644 index 000000000..499b9a6af --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/StorageAccountInventoryCollector.java @@ -0,0 +1,66 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Component; + +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.storage.PublicEndpoints; +import com.microsoft.azure.management.storage.StorageAccount; +import com.tmobile.pacbot.azure.inventory.vo.StorageAccountVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; + +@Component +public class StorageAccountInventoryCollector { + + public List fetchStorageAccountDetails(SubscriptionVH subscription, + Map> tagMap) { + List storageAccountList = new ArrayList(); + + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + PagedList storageAccounts = azure.storageAccounts().list(); + for (StorageAccount storageAccount : storageAccounts) { + StorageAccountVH storageAccountVH = new StorageAccountVH(); + storageAccountVH.setResourceGroupName(storageAccount.resourceGroupName()); + storageAccountVH.setKind(storageAccount.kind().toString()); + storageAccountVH.setCanAccessFromAzureServices(storageAccount.canAccessFromAzureServices()); + storageAccountVH.setIpAddressesWithAccess(storageAccount.ipAddressesWithAccess()); + storageAccountVH.setId(storageAccount.id()); + storageAccountVH.setIpAddressRangesWithAccess(storageAccount.ipAddressRangesWithAccess()); + storageAccountVH.setAccessAllowedFromAllNetworks(storageAccount.isAccessAllowedFromAllNetworks()); + storageAccountVH.setAzureFilesAadIntegrationEnabled(storageAccount.isAzureFilesAadIntegrationEnabled()); + storageAccountVH.setHnsEnabled(storageAccount.isHnsEnabled()); + storageAccountVH.setName(storageAccount.name()); + storageAccountVH.setRegionName(storageAccount.regionName()); + storageAccountVH.setNetworkSubnetsWithAccess(storageAccount.networkSubnetsWithAccess()); + storageAccountVH.setSystemAssignedManagedServiceIdentityPrincipalId( + storageAccount.systemAssignedManagedServiceIdentityPrincipalId()); + storageAccountVH.setSystemAssignedManagedServiceIdentityTenantId( + storageAccount.systemAssignedManagedServiceIdentityTenantId()); + storageAccountVH.setTags(Util.tagsList(tagMap, storageAccount.resourceGroupName(), storageAccount.tags())); + storageAccountVH.setSubscription(subscription.getSubscriptionId()); + storageAccountVH.setSubscriptionName(subscription.getSubscriptionName()); + endPointDetails(storageAccount.endPoints(), storageAccountVH); + storageAccountList.add(storageAccountVH); + } + + return storageAccountList; + } + + private void endPointDetails(PublicEndpoints endpoints, StorageAccountVH storageAccountVH) { + Map endpointsMap = new HashMap(); + endpointsMap.put("blobEndPoint", endpoints.primary().blob()); + endpointsMap.put("fileEndPoint", endpoints.primary().file()); + endpointsMap.put("queueEndPoint", endpoints.primary().queue()); + endpointsMap.put("tableEndPoint", endpoints.primary().table()); + endpointsMap.put("dfsEndPoint", endpoints.primary().dfs()); + endpointsMap.put("webEndPoint", endpoints.primary().web()); + storageAccountVH.setEndpointsMap(endpointsMap); + + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SubnetInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SubnetInventoryCollector.java new file mode 100644 index 000000000..0966f9b9c --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SubnetInventoryCollector.java @@ -0,0 +1,84 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.network.Network; +import com.tmobile.pacbot.azure.inventory.vo.SubnetVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class SubnetInventoryCollector { + + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets?api-version=2019-07-01"; + private static Logger log = LoggerFactory.getLogger(SubnetInventoryCollector.class); + + public List fetchSubnetDetails(SubscriptionVH subscription) { + + List subnetList = new ArrayList(); + String accessToken; + try { + accessToken = AzureCredentialManager.getAuthToken(); + } catch (Exception e1) { + return subnetList; + } + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + PagedList networks = azure.networks().list(); + for (Network network : networks) { + String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId()), + URLEncoder.encode(network.resourceGroupName()), URLEncoder.encode(network.name())); + try { + String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); + JsonObject responseObj = new JsonParser().parse(response).getAsJsonObject(); + JsonArray subnetObjects = responseObj.getAsJsonArray("value"); + for (JsonElement subnetElement : subnetObjects) { + SubnetVH subnetVH = new SubnetVH(); + subnetVH.setSubscription(subscription.getSubscriptionId()); + subnetVH.setSubscriptionName(subscription.getSubscriptionName()); + JsonObject subnetObject = subnetElement.getAsJsonObject(); + JsonObject properties = subnetObject.getAsJsonObject("properties"); + subnetVH.setId(subnetObject.get("id").getAsString()); + subnetVH.setName(subnetObject.get("name").getAsString()); + subnetVH.setType(subnetObject.get("type").getAsString()); + subnetVH.setEtag(subnetObject.get("etag").getAsString()); + if (properties != null) { + HashMap propertiesMap = new Gson().fromJson(properties.toString(), + HashMap.class); + subnetVH.setIpConfigurations((List>) propertiesMap.get("ipConfigurations")); + subnetVH.setAddressPrefix(propertiesMap.get("addressPrefix").toString()); + subnetVH.setPrivateLinkServiceNetworkPolicies( + propertiesMap.get("privateLinkServiceNetworkPolicies").toString()); + subnetVH.setProvisioningState(propertiesMap.get("provisioningState").toString()); + subnetVH.setPrivateEndpointNetworkPolicies( + propertiesMap.get("privateEndpointNetworkPolicies").toString()); + + } + subnetList.add(subnetVH); + } + } catch (Exception e) { + log.error(" Error fetching subnets for network inventory {} Cause : {}", network.name(), + e.getMessage()); + + } + } + System.out.println(subnetList.size()); + return subnetList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/Util.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/Util.java new file mode 100644 index 000000000..9408b4823 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/Util.java @@ -0,0 +1,162 @@ +/******************************************************************************* + * Copyright 2018 T Mobile, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; + +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.entity.ContentType; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.TrustStrategy; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The Class Util. + */ +public class Util { + + private static Logger log = LoggerFactory.getLogger(Util.class); + + /** + * Instantiates a new util. + */ + private Util() { + + } + + /** + * Base 64 decode. + * + * @param encodedStr + * the encoded str + * @return the string + */ + public static String base64Decode(String encodedStr) { + return new String(Base64.getDecoder().decode(encodedStr)); + } + + public static String base64Encode(String str) { + return Base64.getEncoder().encodeToString(str.getBytes()); + } + + public static Map getHeader(String base64Creds) { + Map authToken = new HashMap<>(); + authToken.put("Content-Type", ContentType.APPLICATION_JSON.toString()); + authToken.put("Authorization", "Basic " + base64Creds); + return authToken; + } + + public static String httpGetMethodWithHeaders(String url, Map headers) throws Exception { + String json = null; + + HttpGet get = new HttpGet(url); + CloseableHttpClient httpClient = null; + if (headers != null && !headers.isEmpty()) { + for (Map.Entry entry : headers.entrySet()) { + get.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + try { + httpClient = getHttpClient(); + CloseableHttpResponse res = httpClient.execute(get); + if (res.getStatusLine().getStatusCode() == 200) { + json = EntityUtils.toString(res.getEntity()); + } + } finally { + if (httpClient != null) { + httpClient.close(); + } + } + return json; + } + + private static CloseableHttpClient getHttpClient() { + CloseableHttpClient httpClient = null; + try { + httpClient = HttpClientBuilder.create().setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .setSSLContext(new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() { + @Override + public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException { + return true; + } + }).build()).build(); + } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) { + log.error("Error in HttpUtil post ", e); + } + return httpClient; + } + + public static String httpPostMethodWithHeaders(String url, Map headers) throws Exception { + String json = null; + + HttpPost post = new HttpPost(url); + CloseableHttpClient httpClient = null; + if (headers != null && !headers.isEmpty()) { + for (Map.Entry entry : headers.entrySet()) { + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + try { + httpClient = getHttpClient(); + CloseableHttpResponse res = httpClient.execute(post); + if (res.getStatusLine().getStatusCode() == 200) { + json = EntityUtils.toString(res.getEntity()); + } + } finally { + if (httpClient != null) { + httpClient.close(); + } + } + return json; + } + + public static String removeFirstSlash(String resourceId) { + if (resourceId != null && resourceId.startsWith("/")) { + return resourceId.substring(1); + } + return resourceId; + + } + + public static Map tagsList(Map> tagMap, String resourceGroupName, + Map tags) { + + Map tagsFinal = new HashMap(); + if (tagMap.get(resourceGroupName.toLowerCase()) != null) { + tagsFinal.putAll(tagMap.get(resourceGroupName.toLowerCase())); + tagsFinal.putAll(tags); + return tagsFinal; + } else { + return tags; + } + + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/VMInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/VMInventoryCollector.java new file mode 100644 index 000000000..ea8ab26fb --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/VMInventoryCollector.java @@ -0,0 +1,226 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.compute.DataDisk; +import com.microsoft.azure.management.compute.OSDisk; +import com.microsoft.azure.management.compute.VirtualMachine; +import com.microsoft.azure.management.network.NetworkInterface; +import com.microsoft.azure.management.network.NicIPConfiguration; +import com.microsoft.azure.management.network.Subnet; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacbot.azure.inventory.vo.VMDiskVH; +import com.tmobile.pacbot.azure.inventory.vo.VirtualMachineVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; + +@Component +public class VMInventoryCollector { + + + private static Logger log = LoggerFactory.getLogger(VMInventoryCollector.class); + + public List fetchVMDetails(SubscriptionVH subscription, Map> tagMap) { + List vmList = new ArrayList(); + + Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + List networkInterfaces = azure.networkInterfaces().list(); + + PagedList vms = azure.virtualMachines().list(); + for (VirtualMachine virtualMachine : vms) { + try { + VirtualMachineVH vmVH = new VirtualMachineVH(); + + vmVH.setComputerName(virtualMachine.computerName() == null + ? virtualMachine.instanceView().computerName() == null ? virtualMachine.name() + : virtualMachine.instanceView().computerName() + : virtualMachine.computerName()); + vmVH.setName(virtualMachine.name()); + vmVH.setRegion(virtualMachine.regionName()); + vmVH.setSubscription(subscription.getSubscriptionId()); + vmVH.setSubscriptionName(subscription.getSubscriptionName()); + + virtualMachine.inner().networkProfile(); + vmVH.setVmSize(virtualMachine.size().toString()); + vmVH.setResourceGroupName(virtualMachine.resourceGroupName()); + + vmVH.setStatus(virtualMachine.powerState() != null + ? virtualMachine.powerState().toString().replace("PowerState/", "") + : "Unknown"); + + if(virtualMachine.instanceView()!=null) { + vmVH.setOs(virtualMachine.instanceView().osName()); + vmVH.setOsVersion(virtualMachine.instanceView().osVersion()); + } + vmVH.setOsType(virtualMachine.osType()!=null?virtualMachine.osType().toString():""); + + vmVH.setNetworkInterfaceIds(virtualMachine.networkInterfaceIds()); + vmVH.setAvailabilityZones(virtualMachine.availabilityZones()); + + vmVH.setVmId(virtualMachine.vmId()); + vmVH.setManagedDiskEnabled(virtualMachine.isManagedDiskEnabled()); + + vmVH.setPrivateIpAddress(virtualMachine.getPrimaryNetworkInterface().primaryPrivateIP()); + vmVH.setPublicIpAddress(virtualMachine.getPrimaryPublicIPAddress() != null + ? virtualMachine.getPrimaryPublicIPAddress().ipAddress() + : ""); + + vmVH.setAvailabilitySetId(virtualMachine.availabilitySetId()); + vmVH.setProvisioningState(virtualMachine.provisioningState()); + vmVH.setLicenseType(virtualMachine.licenseType()); + vmVH.setId(virtualMachine.id()); + + vmVH.setBootDiagnosticsEnabled(virtualMachine.isBootDiagnosticsEnabled()); + vmVH.setBootDiagnosticsStorageUri(virtualMachine.bootDiagnosticsStorageUri()); + vmVH.setManagedServiceIdentityEnabled(virtualMachine.isManagedServiceIdentityEnabled()); + vmVH.setSystemAssignedManagedServiceIdentityTenantId( + virtualMachine.systemAssignedManagedServiceIdentityTenantId()); + vmVH.setSystemAssignedManagedServiceIdentityPrincipalId( + virtualMachine.systemAssignedManagedServiceIdentityPrincipalId()); + vmVH.setUserAssignedManagedServiceIdentityIds(virtualMachine.userAssignedManagedServiceIdentityIds()); + vmVH.setTags(Util.tagsList(tagMap, virtualMachine.resourceGroupName(), virtualMachine.tags())); + vmVH.setPrimaryNetworkIntefaceId(virtualMachine.primaryNetworkInterfaceId()); + vmVH.setPrimaryNCIMacAddress(virtualMachine.getPrimaryNetworkInterface().macAddress()); + + setVmDisks(virtualMachine, vmVH); + setNsgs(virtualMachine, vmVH, networkInterfaces); + setVnetInfo(virtualMachine, vmVH); + setOtherVnets(virtualMachine, vmVH, networkInterfaces); + + + vmList.add(vmVH); + }catch(Exception e) { + log.error("Error Collecting info for {} {} ",virtualMachine.computerName(), virtualMachine.name(),e.getMessage()); + } + } + log.info("Target Type : {} Total: {} ", "virtualmachine", vmList.size()); + return vmList; + } + + private void setVnetInfo(VirtualMachine virtualMachine, VirtualMachineVH vmVH) { + + NicIPConfiguration ipConfiguration = virtualMachine.getPrimaryNetworkInterface().primaryIPConfiguration(); + + vmVH.setVnet(ipConfiguration.networkId()); + vmVH.setVnetName(ipConfiguration.getNetwork().name()); + vmVH.setSubnet(ipConfiguration.subnetName()); + + } + + private void setOtherVnets(VirtualMachine virtualMachine, VirtualMachineVH vmVH, + List networkInterfaces) { + String primaryNetworkIntefaceId = virtualMachine.getPrimaryNetworkInterface().id(); + + List nicIds = virtualMachine.networkInterfaceIds(); + List nics = networkInterfaces.stream() + .filter(nic -> nicIds.contains(nic.id()) && !primaryNetworkIntefaceId.equals(nic.id())) + .collect(Collectors.toList()); + List> vnetInfoList = new ArrayList<>(); + for (NetworkInterface nic : nics) { + NicIPConfiguration ipConfiguration = nic.primaryIPConfiguration(); + String subnet = ipConfiguration.subnetName(); + String vnet = ipConfiguration.networkId(); + Map vnetInfo = new HashMap<>(); + vnetInfo.put("vnet", vnet); + vnetInfo.put("subnet", subnet); + vnetInfoList.add(vnetInfo); + } + vmVH.setSecondaryNetworks(vnetInfoList); + + } + + private void setNsgs(VirtualMachine virtualMachine, VirtualMachineVH vmVH, + List networkInterfaces) { + List nicIds = virtualMachine.networkInterfaceIds(); + List nics = networkInterfaces.stream().filter(nic -> nicIds.contains(nic.id())) + .collect(Collectors.toList()); + + List> nsgList = new ArrayList<>(); + String nsg; + Map nsgMap; + for (NetworkInterface nic : nics) { + NicIPConfiguration ipConfiguration = nic.primaryIPConfiguration(); + String subnet = ipConfiguration.subnetName(); + Optional subnetOptional = ipConfiguration.getNetwork().subnets().values().stream() + .filter(subnetObj -> subnet.equals(subnetObj.name())).findAny(); + Subnet subnetObj = null; + ; + if (subnetOptional.isPresent()) { + subnetObj = subnetOptional.get(); + } + nsg = nic.networkSecurityGroupId(); + if (nsg != null) { + nsgMap = new HashMap<>(); + nsgMap.put("nsg", nsg); + nsgMap.put("attachedTo", nic.id()); + nsgMap.put("attachedToType", "nic"); + nsgMap.put("nicSubet", subnetObj.parent().id() + "/" + subnetObj.name()); + nsgList.add(nsgMap); + } + if (subnetObj != null) { + nsg = subnetObj.networkSecurityGroupId(); + if (nsg != null) { + nsgMap = new HashMap<>(); + nsgMap.put("nsg", nsg); + nsgMap.put("attachedTo", subnetObj.parent().id() + "/" + subnetObj.name()); + nsgMap.put("attachedToType", "subnet"); + nsgList.add(nsgMap); + } + } + } + vmVH.setNetworkSecurityGroups(nsgList); + + } + + private void setVmDisks(VirtualMachine vm, VirtualMachineVH vmVH) { + List vmDisks = new ArrayList<>(); + OSDisk osDisk = vm.storageProfile().osDisk(); + VMDiskVH vmDisk = new VMDiskVH(); + vmDisk.setName(osDisk.name()); + vmDisk.setSizeInGB(osDisk.diskSizeGB()); + vmDisk.setCachingType(osDisk.caching().toString()); + vmDisk.setStorageAccountType( + osDisk.managedDisk().storageAccountType() != null ? osDisk.managedDisk().storageAccountType().toString() + : "Unknown"); + vmDisk.setType("OSDisk"); + vmDisks.add(vmDisk); + + List dataDisks = vm.storageProfile().dataDisks(); + for (DataDisk dataDisk : dataDisks) { + vmDisk = new VMDiskVH(); + vmDisk.setName(dataDisk.name()); + vmDisk.setSizeInGB(dataDisk.diskSizeGB()); + vmDisk.setStorageAccountType(dataDisk.managedDisk().storageAccountType() != null + ? dataDisk.managedDisk().storageAccountType().toString() + : "Unknown"); + vmDisk.setCachingType(dataDisk.caching().toString()); + vmDisk.setType("DataDisk"); + vmDisks.add(vmDisk); + } + vmVH.setDisks(vmDisks); + + } + + @SuppressWarnings("unused") + private String identifyPlatform(String os) { + try{ + if(os.toLowerCase().contains("windows")) { + return "windows"; + } + }catch(Exception e) { + + } + return ""; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/VaultInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/VaultInventoryCollector.java new file mode 100644 index 000000000..631c31481 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/VaultInventoryCollector.java @@ -0,0 +1,86 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacbot.azure.inventory.vo.VaultVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class VaultInventoryCollector { + private static Logger log = LoggerFactory.getLogger(VaultInventoryCollector.class); + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.KeyVault/vaults?api-version=2018-02-14"; + + public List fetchVaultDetails(SubscriptionVH subscription) throws Exception { + + List vaultList = new ArrayList(); + String accessToken; + try { + accessToken = AzureCredentialManager.getAuthToken(); + + } catch (Exception e1) { + return vaultList; + } + + String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); + try { + + String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); + JsonObject responseObj = new JsonParser().parse(response).getAsJsonObject(); + JsonArray vaultObjects = responseObj.getAsJsonArray("value"); + if (vaultObjects != null) { + for (JsonElement vaultElement : vaultObjects) { + VaultVH vaultVH = new VaultVH(); + JsonObject vaultObject = vaultElement.getAsJsonObject(); + vaultVH.setSubscription(subscription.getSubscriptionId()); + vaultVH.setSubscriptionName(subscription.getSubscriptionName()); + vaultVH.setId(vaultObject.get("id").getAsString()); + vaultVH.setLocation(vaultObject.get("location").getAsString()); + vaultVH.setName(vaultObject.get("name").getAsString()); + vaultVH.setType(vaultObject.get("type").getAsString()); + JsonObject properties = vaultObject.getAsJsonObject("properties"); + JsonObject tags = vaultObject.getAsJsonObject("tags"); + if (properties != null) { + HashMap propertiesMap = new Gson().fromJson(properties.toString(), + HashMap.class); + vaultVH.setEnabledForDeployment((boolean) propertiesMap.get("enabledForDeployment")); + vaultVH.setEnabledForDiskEncryption((boolean) propertiesMap.get("enabledForDiskEncryption")); + vaultVH.setEnabledForTemplateDeployment( + (boolean) propertiesMap.get("enabledForTemplateDeployment")); + vaultVH.setTenantId(propertiesMap.get("tenantId").toString()); + vaultVH.setProvisioningState(propertiesMap.get("provisioningState").toString()); + vaultVH.setSku((Map) propertiesMap.get("sku")); + vaultVH.setVaultUri(propertiesMap.get("vaultUri").toString()); + + } + if (tags != null) { + HashMap tagsMap = new Gson().fromJson(tags.toString(), HashMap.class); + vaultVH.setTags(tagsMap); + } + + vaultList.add(vaultVH); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + System.out.println(vaultList.size()); + return vaultList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/WorkflowInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/WorkflowInventoryCollector.java new file mode 100644 index 000000000..2f23261de --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/WorkflowInventoryCollector.java @@ -0,0 +1,79 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; +import com.tmobile.pacbot.azure.inventory.vo.WorkflowVH; +import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class WorkflowInventoryCollector { + + private static Logger log = LoggerFactory.getLogger(WorkflowInventoryCollector.class); + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.Logic/workflows?api-version=2016-06-01"; + + public List fetchWorkflowDetails(SubscriptionVH subscription) throws Exception { + + List workflowList = new ArrayList(); + String accessToken; + try { + accessToken = AzureCredentialManager.getAuthToken(); + + } catch (Exception e1) { + return workflowList; + } + + String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); + try { + + String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); + JsonObject responseObj = new JsonParser().parse(response).getAsJsonObject(); + JsonArray workflowObjects = responseObj.getAsJsonArray("value"); + if (workflowObjects != null) { + for (JsonElement workflowElement : workflowObjects) { + WorkflowVH workflowVH = new WorkflowVH(); + JsonObject workflowObject = workflowElement.getAsJsonObject(); + workflowVH.setSubscription(subscription.getSubscriptionId()); + workflowVH.setSubscriptionName(subscription.getSubscriptionName()); + workflowVH.setId(workflowObject.get("id").getAsString()); + workflowVH.setLocation(workflowObject.get("location").getAsString()); + workflowVH.setName(workflowObject.get("name").getAsString()); + workflowVH.setType(workflowObject.get("type").getAsString()); + JsonObject properties = workflowObject.getAsJsonObject("properties"); + JsonObject tags = workflowObject.getAsJsonObject("tags"); + if (properties != null) { + HashMap propertiesMap = new Gson().fromJson(properties.toString(), + HashMap.class); + workflowVH.setProperties(propertiesMap); + } + if (tags != null) { + HashMap tagsMap = new Gson().fromJson(tags.toString(), HashMap.class); + workflowVH.setTags(tagsMap); + } + + workflowList.add(workflowVH); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + System.out.println(workflowList.size()); + return workflowList; + } + + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/config/ConfigUtil.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/config/ConfigUtil.java new file mode 100644 index 000000000..c88cdfb7c --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/config/ConfigUtil.java @@ -0,0 +1,68 @@ +package com.tmobile.pacbot.azure.inventory.config; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.tmobile.pacbot.azure.inventory.InventoryConstants; +import com.tmobile.pacbot.azure.inventory.collector.Util; + +public class ConfigUtil { + + private static Logger log = LoggerFactory.getLogger(ConfigUtil.class); + + private static String configUrl = System.getenv("CONFIG_URL"); + + public static void setConfigProperties(String base64Creds) throws Exception { + Properties properties = new Properties(); + properties.putAll(System.getProperties()); + properties.putAll(fetchConfigProperties(base64Creds)); + System.setProperties(properties); + } + + @SuppressWarnings("unchecked") + public static Map fetchConfigProperties(String base64Creds) throws Exception { + + Map properties = new HashMap<>(); + + + ObjectMapper objectMapper = new ObjectMapper(); + try { + Map appProps = new HashMap<>(); + Map batchProps = new HashMap<>(); + Map invProps = new HashMap<>(); + Map response = objectMapper.readValue(Util.httpGetMethodWithHeaders(configUrl, Util.getHeader(base64Creds)), new TypeReference>(){}); + List> propertySources = (List>)response.get("propertySources"); + for(Map propertySource : propertySources) { + if(propertySource.get(InventoryConstants.NAME).toString().contains(InventoryConstants.APPLICATION)) { + appProps.putAll((Map)propertySource.get(InventoryConstants.SOURCE)); + } + if(propertySource.get(InventoryConstants.NAME).toString().contains(InventoryConstants.BATCH)) { + batchProps.putAll((Map)propertySource.get(InventoryConstants.SOURCE)); + } + if(propertySource.get(InventoryConstants.NAME).toString().contains(InventoryConstants.INVENTORY)) { + invProps.putAll((Map)propertySource.get(InventoryConstants.SOURCE)); + } + properties.putAll(appProps); + properties.putAll(batchProps); + properties.putAll(invProps); + } + } catch (Exception e) { + log.error("Error in fetchConfigProperties",e); + throw e; + } + if(properties.isEmpty()){ + throw new Exception("No config properties fetched from "+configUrl); + } + + log.info("Config are feteched from {}",configUrl); + properties.forEach((k,v)-> log.debug(" {} : {} ",k,v)); + return properties; + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/AssetFileGenerator.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/AssetFileGenerator.java new file mode 100644 index 000000000..8edeabe8f --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/AssetFileGenerator.java @@ -0,0 +1,595 @@ +package com.tmobile.pacbot.azure.inventory.file; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import com.tmobile.pacbot.azure.inventory.collector.BatchAccountInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.BlobContainerInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.CosmosDBInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.DatabricksInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.DiskInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.LoadBalancerInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.MariaDBInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.MySQLInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.NSGInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.NamespaceInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.NetworkInterfaceInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.NetworkInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.PolicyDefinitionInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.PolicyStatesInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.PostgreSQLInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.PublicIpAddressInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.RegisteredApplicationInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.ResourceGroupInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.RouteTableInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.SCRecommendationsCollector; +import com.tmobile.pacbot.azure.inventory.collector.SQLDatabaseInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.SQLServerInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.SearchServiceInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.SecurityAlertsInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.SitesInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.SnapshotInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.StorageAccountInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.SubnetInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.VMInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.VaultInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.WorkflowInventoryCollector; +import com.tmobile.pacbot.azure.inventory.vo.PolicyDefinitionVH; +import com.tmobile.pacbot.azure.inventory.vo.ResourceGroupVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; + +@Component +public class AssetFileGenerator { + + /** The target types. */ + @Value("${targetTypes:virtualmachine}") + private String targetTypes; + + /** The log. */ + private static Logger log = LoggerFactory.getLogger(AssetFileGenerator.class); + + @Autowired + VMInventoryCollector vmInventoryCollector; + + @Autowired + DiskInventoryCollector diskInventoryCollector; + + @Autowired + LoadBalancerInventoryCollector loadBalancerInventoryCollector; + + @Autowired + NetworkInterfaceInventoryCollector networkInterfaceInventoryCollector; + + @Autowired + NSGInventoryCollector networkSecurityInventoryCollector; + + @Autowired + SQLDatabaseInventoryCollector sqlDatabaseInventoryCollector; + + @Autowired + StorageAccountInventoryCollector storageAccountInventoryCollector; + + @Autowired + NetworkInventoryCollector networkInventoryCollector; + + @Autowired + SCRecommendationsCollector scRecommendationsCollector; + + @Autowired + SQLServerInventoryCollector sqlServerInventoryCollector; + + @Autowired + BlobContainerInventoryCollector blobContainerInventoryCollector; + + @Autowired + ResourceGroupInventoryCollector resourceGroupInventoryCollector; + + @Autowired + CosmosDBInventoryCollector cosmosDBInventoryCollector; + + @Autowired + RegisteredApplicationInventoryCollector registeredApplicationInventoryCollector; + + @Autowired + MySQLInventoryCollector mySQLInventoryCollector; + + @Autowired + DatabricksInventoryCollector databricksInventoryCollector; + + @Autowired + MariaDBInventoryCollector mariaDBInventoryCollector; + + @Autowired + PostgreSQLInventoryCollector postgreSQLInventoryCollector; + + @Autowired + SnapshotInventoryCollector snapshotInventoryCollector; + + @Autowired + PublicIpAddressInventoryCollector publicIpAddressInventoryCollector; + + @Autowired + RouteTableInventoryCollector routeTableInventoryCollector; + + @Autowired + SecurityAlertsInventoryCollector securityAlertsInventoryCollector; + + @Autowired + PolicyStatesInventoryCollector policyStatesInventoryCollector; + + @Autowired + PolicyDefinitionInventoryCollector policyDefinitionInventoryCollector; + + @Autowired + SitesInventoryCollector sitesInventoryCollector; + + @Autowired + VaultInventoryCollector vaultInventoryCollector; + + @Autowired + WorkflowInventoryCollector workflowInventoryCollector; + + @Autowired + BatchAccountInventoryCollector batchAccountInventoryCollector; + + @Autowired + NamespaceInventoryCollector namespaceInventoryCollector; + + @Autowired + SearchServiceInventoryCollector searchServiceInventoryCollector; + + @Autowired + SubnetInventoryCollector subnetInventoryCollector; + + public void generateFiles(List subscriptions, String filePath) { + + try { + FileManager.initialise(filePath); + } catch (IOException e1) { + e1.printStackTrace(); + } + // generateAzureAplicationList(); + + for (SubscriptionVH subscription : subscriptions) { + log.info("Started Discovery for sub {}", subscription); + List resourceGroupList = new ArrayList(); + try { + resourceGroupList = resourceGroupInventoryCollector.fetchResourceGroupDetails(subscription); + + } catch (Exception e) { + e.printStackTrace(); + + } + Map> tagMap = resourceGroupList.stream() + .collect(Collectors.toMap(x -> x.getResourceGroupName().toLowerCase(), x -> x.getTags())); + + List policyDefinitionList = policyDefinitionInventoryCollector + .fetchPolicyDefinitionDetails(subscription); + + ExecutorService executor = Executors.newCachedThreadPool(); + + executor.execute(() -> { + if (!(isTypeInScope("virtualmachine"))) { + return; + } + try { + FileManager.generateVMFiles(vmInventoryCollector.fetchVMDetails(subscription, tagMap)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("storageaccount"))) { + return; + } + try { + FileManager.generateStorageAccountFiles( + storageAccountInventoryCollector.fetchStorageAccountDetails(subscription, tagMap)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + executor.execute(() -> { + if (!(isTypeInScope("sqldatabase"))) { + return; + } + try { + FileManager.generateSQLdatabaseFiles( + sqlDatabaseInventoryCollector.fetchSQLDatabaseDetails(subscription, tagMap)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + executor.execute(() -> { + if (!(isTypeInScope("nsg"))) { + return; + } + try { + FileManager.generateNetworkSecurityFiles( + networkSecurityInventoryCollector.fetchNetworkSecurityGroupDetails(subscription, tagMap)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + executor.execute(() -> { + if (!(isTypeInScope("disk"))) { + return; + } + try { + FileManager + .generateDataDiskFiles(diskInventoryCollector.fetchDataDiskDetails(subscription, tagMap)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + executor.execute(() -> { + if (!(isTypeInScope("networkinterface"))) { + return; + } + try { + FileManager.generateNetworkInterfaceFiles( + networkInterfaceInventoryCollector.fetchNetworkInterfaceDetails(subscription, tagMap)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("vnet"))) { + return; + } + try { + FileManager + .generateNetworkFiles(networkInventoryCollector.fetchNetworkDetails(subscription, tagMap)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("loadbalancer"))) { + return; + } + try { + FileManager.generateLoadBalancerFiles( + loadBalancerInventoryCollector.fetchLoadBalancerDetails(subscription, tagMap)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("securitycenter"))) { + return; + } + + try { + FileManager.generateSecurityCenterFiles( + scRecommendationsCollector.fetchSecurityCenterRecommendations(subscription)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("sqlserver"))) { + return; + } + + try { + FileManager.generateSQLServerFiles( + sqlServerInventoryCollector.fetchSQLServerDetails(subscription, tagMap)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("blobcontainer"))) { + return; + } + + try { + FileManager.generateBlobContainerFiles( + blobContainerInventoryCollector.fetchBlobContainerDetails(subscription, tagMap)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("resourcegroup"))) { + return; + } + + try { + FileManager.generateResourceGroupFiles( + resourceGroupInventoryCollector.fetchResourceGroupDetails(subscription)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("cosmosdb"))) { + return; + } + + try { + FileManager.generateCosmosDBFiles( + cosmosDBInventoryCollector.fetchCosmosDBDetails(subscription, tagMap)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + executor.execute(() -> { + if (!(isTypeInScope("mysqlserver"))) { + return; + } + + try { + FileManager.generateMySqlServerFiles(mySQLInventoryCollector.fetchMySQLServerDetails(subscription)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("databricks"))) { + return; + } + + try { + FileManager + .generateDatabricksFiles(databricksInventoryCollector.fetchDatabricksDetails(subscription)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("mariadb"))) { + return; + } + + try { + FileManager.generateMariaDBFiles(mariaDBInventoryCollector.fetchMariaDBDetails(subscription)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("postgresql"))) { + return; + } + + try { + FileManager.generatePostgreSQLServerFiles( + postgreSQLInventoryCollector.fetchPostgreSQLServerDetails(subscription)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("snapshot"))) { + return; + } + + try { + FileManager.generateSnapshotFiles( + snapshotInventoryCollector.fetchSnapshotDetails(subscription, tagMap)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("publicipaddress"))) { + return; + } + + try { + FileManager.generatePublicIpAddressFiles( + publicIpAddressInventoryCollector.fetchPublicIpAddressDetails(subscription, tagMap)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("routetable"))) { + return; + } + + try { + FileManager.generateRouteTableFiles( + routeTableInventoryCollector.fetchRouteTableDetails(subscription, tagMap)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("securityalerts"))) { + return; + } + + try { + FileManager.generateSecurityAlertsFiles( + securityAlertsInventoryCollector.fetchSecurityAlertsDetails(subscription)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("policyevaluationresults"))) { + return; + } + + try { + FileManager.generatePolicyStatesFiles(policyStatesInventoryCollector + .fetchPolicyStatesDetails(subscription, policyDefinitionList)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("policydefinitions"))) { + return; + } + + try { + FileManager.generatePolicyDefinitionFiles( + policyDefinitionInventoryCollector.fetchPolicyDefinitionDetails(subscription)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("sites"))) { + return; + } + + try { + FileManager.generateSiteFiles( + sitesInventoryCollector.fetchSitesDetails(subscription)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("vaults"))) { + return; + } + + try { + FileManager.generateVaultFiles( + vaultInventoryCollector.fetchVaultDetails(subscription)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("workflows"))) { + return; + } + + try { + FileManager.generateWorkflowFiles( + workflowInventoryCollector.fetchWorkflowDetails(subscription)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("batchaccounts"))) { + return; + } + + try { + FileManager.generateBatchAccountFiles( + batchAccountInventoryCollector.fetchBatchAccountDetails(subscription)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("namespaces"))) { + return; + } + + try { + FileManager.generateNamespaceFiles( + namespaceInventoryCollector.fetchNamespaceDetails(subscription)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("searchservices"))) { + return; + } + + try { + FileManager.generateSearchServiceFiles( + searchServiceInventoryCollector.fetchSearchServiceDetails(subscription)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + executor.execute(() -> { + if (!(isTypeInScope("subnets"))) { + return; + } + + try { + FileManager.generateSubnetFiles( + subnetInventoryCollector.fetchSubnetDetails(subscription)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + + executor.shutdown(); + while (!executor.isTerminated()) { + + } + + log.info("Finished Discovery for sub {}", subscription); + } + + try { + FileManager.finalise(); + } catch (IOException e) { + } + } + + /** + * function for generating registered application file + */ + private void generateAzureAplicationList() { + + if ((isTypeInScope("registeredApplication"))) { + try { + FileManager.generateRegisteredApplicationFiles( + registeredApplicationInventoryCollector.fetchAzureRegisteredApplication()); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + private boolean isTypeInScope(String type) { + if ("".equals(targetTypes)) { + return true; + } else { + List targetTypesList = Arrays.asList(targetTypes.split(",")); + return targetTypesList.contains(type); + } + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/FileGenerator.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/FileGenerator.java new file mode 100644 index 000000000..abd7096ec --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/FileGenerator.java @@ -0,0 +1,127 @@ +/******************************************************************************* + * Copyright 2018 T Mobile, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +package com.tmobile.pacbot.azure.inventory.file; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.tmobile.pacbot.azure.inventory.vo.AzureVH; + +/** + * The Class FileGenerator. + */ +public class FileGenerator { + + /** + * Instantiates a new file generator. + */ + private FileGenerator() { + + } + + /** The folder name. */ + protected static String folderName ; + + /** The Constant DELIMITER. */ + public static final String DELIMITER ="`"; + + /** The Constant LINESEPARATOR. */ + public static final String LINESEPARATOR ="\n"; + + public static final String COMMA =","; + + /** The current date. */ + protected static String discoveryDate = new SimpleDateFormat("yyyy-MM-dd HH:00:00Z").format(new java.util.Date()); + + /** The log. */ + private static Logger log = LoggerFactory.getLogger(FileGenerator.class); + + /** + * Write to file. + * + * @param filename the filename + * @param data the data + * @param appendto the appendto + * @throws IOException Signals that an I/O exception has occurred. + */ + public static void writeToFile(String filename ,String data,boolean appendto) throws IOException{ + log.debug("Write to File :"+filename ); + BufferedWriter bw = null ; + try { + bw = new BufferedWriter(new FileWriter(folderName+File.separator+filename,appendto)); + bw.write(data); + bw.flush(); + bw.close(); + } catch (IOException e) { + log.error("Write to File :{} failed",filename,e); + throw e; + } + finally { + if(bw != null) { + bw.close(); + } + } + } + + + /** + * Gets the line data. + * + * @param fieledNames the fieled names + * @param obj the obj + * @return the line data + */ + + + protected static boolean generateJson(List assetList,String fileName ){ + + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + StringBuilder sb = new StringBuilder(); + + for(AzureVH asset : assetList) { + asset.setDiscoverydate(discoveryDate); + try { + if(sb.length() == 0 && new File(folderName+File.separator+fileName).length() < 2) { + sb.append(objectMapper.writeValueAsString(asset)); + } else { + sb.append(COMMA+LINESEPARATOR+objectMapper.writeValueAsString(asset)); + } + } catch (Exception e) { + log.error("Error in generateJson ",e); + return false; + } + } + + try { + writeToFile(fileName, sb.toString(), true); + } catch (IOException e) { + log.error("Error in generateJson ",e); + return false; + } + return true; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/FileManager.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/FileManager.java new file mode 100644 index 000000000..48c410da6 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/FileManager.java @@ -0,0 +1,297 @@ +/******************************************************************************* + * Copyright 2018 T Mobile, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +package com.tmobile.pacbot.azure.inventory.file; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import com.tmobile.pacbot.azure.inventory.vo.BatchAccountVH; +import com.tmobile.pacbot.azure.inventory.vo.BlobContainerVH; +import com.tmobile.pacbot.azure.inventory.vo.CosmosDBVH; +import com.tmobile.pacbot.azure.inventory.vo.DataDiskVH; +import com.tmobile.pacbot.azure.inventory.vo.DatabricksVH; +import com.tmobile.pacbot.azure.inventory.vo.LoadBalancerVH; +import com.tmobile.pacbot.azure.inventory.vo.MariaDBVH; +import com.tmobile.pacbot.azure.inventory.vo.MySQLServerVH; +import com.tmobile.pacbot.azure.inventory.vo.NamespaceVH; +import com.tmobile.pacbot.azure.inventory.vo.NetworkInterfaceVH; +import com.tmobile.pacbot.azure.inventory.vo.NetworkVH; +import com.tmobile.pacbot.azure.inventory.vo.PolicyDefinitionVH; +import com.tmobile.pacbot.azure.inventory.vo.PolicyStatesVH; +import com.tmobile.pacbot.azure.inventory.vo.PostgreSQLServerVH; +import com.tmobile.pacbot.azure.inventory.vo.PublicIpAddressVH; +import com.tmobile.pacbot.azure.inventory.vo.RecommendationVH; +import com.tmobile.pacbot.azure.inventory.vo.RegisteredApplicationVH; +import com.tmobile.pacbot.azure.inventory.vo.ResourceGroupVH; +import com.tmobile.pacbot.azure.inventory.vo.RouteTableVH; +import com.tmobile.pacbot.azure.inventory.vo.SQLDatabaseVH; +import com.tmobile.pacbot.azure.inventory.vo.SQLServerVH; +import com.tmobile.pacbot.azure.inventory.vo.SearchServiceVH; +import com.tmobile.pacbot.azure.inventory.vo.SecurityAlertsVH; +import com.tmobile.pacbot.azure.inventory.vo.SecurityGroupVH; +import com.tmobile.pacbot.azure.inventory.vo.SitesVH; +import com.tmobile.pacbot.azure.inventory.vo.SnapshotVH; +import com.tmobile.pacbot.azure.inventory.vo.StorageAccountVH; +import com.tmobile.pacbot.azure.inventory.vo.SubnetVH; +import com.tmobile.pacbot.azure.inventory.vo.VaultVH; +import com.tmobile.pacbot.azure.inventory.vo.VirtualMachineVH; +import com.tmobile.pacbot.azure.inventory.vo.WorkflowVH;; + +/** + * The Class FileManager. + */ +public class FileManager { + + /** + * Instantiates a new file manager. + */ + private FileManager() { + + } + + /** + * Initialise. + * + * @param folderName + * the folder name + * @throws IOException + * Signals that an I/O exception has occurred. + */ + public static void initialise(String folderName) throws IOException { + FileGenerator.folderName = folderName; + new File(folderName).mkdirs(); + + FileGenerator.writeToFile("azure-virtualmachine.data", "[", false); + FileGenerator.writeToFile("azure-storageaccount.data", "[", false); + FileGenerator.writeToFile("azure-sqldatabase.data", "[", false); + FileGenerator.writeToFile("azure-nsg.data", "[", false); + FileGenerator.writeToFile("azure-disk.data", "[", false); + FileGenerator.writeToFile("azure-networkinterface.data", "[", false); + FileGenerator.writeToFile("azure-vnet.data", "[", false); + FileGenerator.writeToFile("azure-loadbalancer.data", "[", false); + FileGenerator.writeToFile("azure-securitycenter.data", "[", false); + FileGenerator.writeToFile("azure-sqlserver.data", "[", false); + FileGenerator.writeToFile("azure-blobcontainer.data", "[", false); + FileGenerator.writeToFile("azure-resourcegroup.data", "[", false); + FileGenerator.writeToFile("azure-cosmosdb.data", "[", false); + FileGenerator.writeToFile("azure-mysqlserver.data", "[", false); + FileGenerator.writeToFile("azure-databricks.data", "[", false); + FileGenerator.writeToFile("azure-mariadb.data", "[", false); + FileGenerator.writeToFile("azure-postgresql.data", "[", false); + FileGenerator.writeToFile("azure-registeredApplication.data", "[", false); + FileGenerator.writeToFile("azure-snapshot.data", "[", false); + FileGenerator.writeToFile("azure-publicipaddress.data", "[", false); + FileGenerator.writeToFile("azure-routetable.data", "[", false); + FileGenerator.writeToFile("azure-securityalerts.data", "[", false); + FileGenerator.writeToFile("azure-policyevaluationresults.data", "[", false); + FileGenerator.writeToFile("azure-policydefinitions.data", "[", false); + FileGenerator.writeToFile("azure-sites.data", "[", false); + FileGenerator.writeToFile("azure-vaults.data", "[", false); + FileGenerator.writeToFile("azure-workflows.data", "[", false); + FileGenerator.writeToFile("azure-batchaccounts.data", "[", false); + FileGenerator.writeToFile("azure-namespaces.data", "[", false); + FileGenerator.writeToFile("azure-searchservices.data", "[", false); + FileGenerator.writeToFile("azure-subnets.data", "[", false); + } + + public static void finalise() throws IOException { + + FileGenerator.writeToFile("azure-virtualmachine.data", "]", true); + FileGenerator.writeToFile("azure-storageaccount.data", "]", true); + FileGenerator.writeToFile("azure-sqldatabase.data", "]", true); + FileGenerator.writeToFile("azure-nsg.data", "]", true); + FileGenerator.writeToFile("azure-disk.data", "]", true); + FileGenerator.writeToFile("azure-networkinterface.data", "]", true); + FileGenerator.writeToFile("azure-vnet.data", "]", true); + FileGenerator.writeToFile("azure-securitycenter.data", "]", true); + FileGenerator.writeToFile("azure-loadbalancer.data", "]", true); + FileGenerator.writeToFile("azure-sqlserver.data", "]", true); + FileGenerator.writeToFile("azure-blobcontainer.data", "]", true); + FileGenerator.writeToFile("azure-resourcegroup.data", "]", true); + FileGenerator.writeToFile("azure-cosmosdb.data", "]", true); + FileGenerator.writeToFile("azure-mysqlserver.data", "]", true); + FileGenerator.writeToFile("azure-databricks.data", "]", true); + FileGenerator.writeToFile("azure-mariadb.data", "]", true); + FileGenerator.writeToFile("azure-postgresql.data", "]", true); + FileGenerator.writeToFile("azure-registeredApplication.data", "]", true); + FileGenerator.writeToFile("azure-snapshot.data", "]", true); + FileGenerator.writeToFile("azure-publicipaddress.data", "]", true); + FileGenerator.writeToFile("azure-routetable.data", "]", true); + FileGenerator.writeToFile("azure-securityalerts.data", "]", true); + FileGenerator.writeToFile("azure-policyevaluationresults.data", "]", true); + FileGenerator.writeToFile("azure-policydefinitions.data", "]", true); + FileGenerator.writeToFile("azure-sites.data", "]", true); + FileGenerator.writeToFile("azure-vaults.data", "]", true); + FileGenerator.writeToFile("azure-workflows.data", "]", true); + FileGenerator.writeToFile("azure-batchaccounts.data", "]", true); + FileGenerator.writeToFile("azure-namespaces.data", "]", true); + FileGenerator.writeToFile("azure-searchservices.data", "]", true); + FileGenerator.writeToFile("azure-subnets.data", "]", true); + + + } + + public static void generateVMFiles(List vmMap) throws IOException { + + FileGenerator.generateJson(vmMap, "azure-virtualmachine.data"); + + } + + public static void generateStorageAccountFiles(List storageAccountMap) throws IOException { + + FileGenerator.generateJson(storageAccountMap, "azure-storageaccount.data"); + + } + + public static void generateSQLdatabaseFiles(List sqlDatabaseMap) throws IOException { + + FileGenerator.generateJson(sqlDatabaseMap, "azure-sqldatabase.data"); + + } + + public static void generateNetworkSecurityFiles(List securityGroupMap) throws IOException { + + FileGenerator.generateJson(securityGroupMap, "azure-nsg.data"); + + } + + public static void generateDataDiskFiles(List dataDiskMap) throws IOException { + + FileGenerator.generateJson(dataDiskMap, "azure-disk.data"); + + } + + public static void generateNetworkInterfaceFiles(List networkInterfaceMap) throws IOException { + + FileGenerator.generateJson(networkInterfaceMap, "azure-networkinterface.data"); + + } + + public static void generateNetworkFiles(List networkMap) throws IOException { + + FileGenerator.generateJson(networkMap, "azure-vnet.data"); + + } + + public static void generateLoadBalancerFiles(List loadBalancerMap) throws IOException { + + FileGenerator.generateJson(loadBalancerMap, "azure-loadbalancer.data"); + + } + + public static void generateSecurityCenterFiles(List recommendations) throws IOException { + + FileGenerator.generateJson(recommendations, "azure-securitycenter.data"); + + } + + public static void generateSQLServerFiles(List sqlServerList) throws IOException { + FileGenerator.generateJson(sqlServerList, "azure-sqlserver.data"); + } + + public static void generateBlobContainerFiles(List blobDetailsList) throws IOException { + FileGenerator.generateJson(blobDetailsList, "azure-blobcontainer.data"); + } + + public static void generateResourceGroupFiles(List resourceGroupList) throws IOException { + FileGenerator.generateJson(resourceGroupList, "azure-resourcegroup.data"); + } + + public static void generateCosmosDBFiles(List cosmosDBList) throws IOException { + FileGenerator.generateJson(cosmosDBList, "azure-cosmosdb.data"); + } + + public static void generateRegisteredApplicationFiles(List registeredApplicationVHList) + throws IOException { + FileGenerator.generateJson(registeredApplicationVHList, "azure-registeredApplication.data"); + } + + public static void generateMySqlServerFiles(List mySqlServerList) throws IOException { + FileGenerator.generateJson(mySqlServerList, "azure-mysqlserver.data"); + } + + public static void generateDatabricksFiles(List databricksList) throws IOException { + FileGenerator.generateJson(databricksList, "azure-databricks.data"); + } + + public static void generateMariaDBFiles(List mariaDBList) throws IOException { + FileGenerator.generateJson(mariaDBList, "azure-mariadb.data"); + } + + public static void generatePostgreSQLServerFiles(List postgreSQLServerList) throws IOException { + FileGenerator.generateJson(postgreSQLServerList, "azure-postgresql.data"); + } + + public static void generateSnapshotFiles(List snapshotList) throws IOException { + FileGenerator.generateJson(snapshotList, "azure-snapshot.data"); + } + + public static void generatePublicIpAddressFiles(List publicIpAddressList) throws IOException { + FileGenerator.generateJson(publicIpAddressList, "azure-publicipaddress.data"); + } + + public static void generateRouteTableFiles(List routeTableDetailsList) throws IOException { + FileGenerator.generateJson(routeTableDetailsList, "azure-routetable.data"); + } + + public static void generateSecurityAlertsFiles(List securityAlertsList) throws IOException { + FileGenerator.generateJson(securityAlertsList, "azure-securityalerts.data"); + } + + public static void generatePolicyStatesFiles(List policyStatesList) throws IOException { + FileGenerator.generateJson(policyStatesList, "azure-policyevaluationresults.data"); + } + + public static void generatePolicyDefinitionFiles(List policyDefinitionList) throws IOException { + FileGenerator.generateJson(policyDefinitionList, "azure-policydefinitions.data"); + } + + public static void generateSiteFiles(List sitesList) throws IOException { + FileGenerator.generateJson(sitesList, "azure-sites.data"); + } + + public static void generateVaultFiles(List vaultList) throws IOException { + FileGenerator.generateJson(vaultList, "azure-vaults.data"); + + } + + public static void generateWorkflowFiles(List workflowList) throws IOException { + FileGenerator.generateJson(workflowList, "azure-workflows.data"); + + } + + public static void generateBatchAccountFiles(List batchAccountList) throws IOException { + FileGenerator.generateJson(batchAccountList, "azure-batchaccounts.data"); + + } + + public static void generateNamespaceFiles(List namespaceList) throws IOException { + FileGenerator.generateJson(namespaceList, "azure-namespaces.data"); + + } + + public static void generateSearchServiceFiles(List searchServiceList) throws IOException { + FileGenerator.generateJson(searchServiceList, "azure-searchservices.data"); + + } + + public static void generateSubnetFiles(List subnetList) throws IOException { + FileGenerator.generateJson(subnetList, "azure-subnets.data"); + + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/S3Uploader.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/S3Uploader.java new file mode 100644 index 000000000..d1128955e --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/S3Uploader.java @@ -0,0 +1,193 @@ +/******************************************************************************* + * Copyright 2018 T Mobile, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +package com.tmobile.pacbot.azure.inventory.file; + +import java.io.File; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicSessionCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.DeleteObjectsResult; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.transfer.MultipleFileUpload; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; +import com.tmobile.pacbot.azure.inventory.ErrorManageUtil; +import com.tmobile.pacbot.azure.inventory.auth.AWSCredentialProvider; + +/** + * The Class S3Uploader. + */ +@Component +public class S3Uploader { + + /** The log. */ + private static Logger log = LoggerFactory.getLogger(S3Uploader.class); + + + /** The account. */ + @Value("${base.account}") + private String account; + + /** The account. */ + @Value("${s3.role}") + private String s3Role; + + @Value("${base.region}") + private String region ; + + /** The cred provider. */ + @Autowired + AWSCredentialProvider credProvider; + + /** + * Upload files. + * + * @param s3Bucket the s 3 bucket + * @param dataFolder the data folder + * @param s3Region the s 3 region + * @param filePath the file path + */ + public void uploadFiles(String s3Bucket,String dataFolder, String s3Region,String filePath){ + BasicSessionCredentials credentials = credProvider.getCredentials(account,region,s3Role); + AmazonS3 s3client = AmazonS3ClientBuilder.standard().withRegion(s3Region).withCredentials(new AWSStaticCredentialsProvider(credentials)).build(); + uploadAllFiles(s3client,s3Bucket,dataFolder,filePath); + } + + /** + * Back up files. + * + * @param s3Bucket the s 3 bucket + * @param s3Region the s 3 region + * @param from the from + * @param to the to + */ + public void backUpFiles(String s3Bucket,String s3Region,String from,String to){ + BasicSessionCredentials credentials = credProvider.getCredentials(account,region,s3Role); + AmazonS3 s3client = AmazonS3ClientBuilder.standard().withRegion(s3Region).withCredentials(new AWSStaticCredentialsProvider(credentials)).build(); + log.info("Backing up files from : {} to : {} in bucket : {}",from,to,s3Bucket); + copytoBackUp(s3client,s3Bucket,from,to); + deleteFiles(s3client,s3Bucket,from); + } + + /** + * Upload all files. + * + * @param s3client the s 3 client + * @param s3Bucket the s 3 bucket + * @param dataFolderS3 the data folder S 3 + * @param filePath the file path + */ + private void uploadAllFiles(AmazonS3 s3client,String s3Bucket,String dataFolderS3, String filePath){ + log.info("Uploading files to bucket: {} folder: {}",s3Bucket,dataFolderS3); + TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3client).build(); + try { + MultipleFileUpload xfer = xferMgr.uploadDirectory(s3Bucket, + dataFolderS3, new File(filePath), false); + + while(!xfer.isDone()){ + try{ + Thread.sleep(3000); + }catch(InterruptedException e){ + log.error("Error in uploadAllFiles",e); + ErrorManageUtil.uploadError("all", "all", "all", e.getMessage()); + Thread.currentThread().interrupt(); + } + log.debug(" Transfer % Completed :" +xfer.getProgress().getPercentTransferred()); + } + xfer.waitForCompletion(); + + log.info("Transfer completed"); + } catch (Exception e) { + log.error("{\"errcode\": \"S3_UPLOAD_ERR\" ,\"account\": \"ANY\",\"Message\": \"Exception in loading files to S3\", \"cause\":\"" +e.getMessage()+"\"}") ; + ErrorManageUtil.uploadError("all", "all", "all", e.getMessage()); + } + xferMgr.shutdownNow(); + } + + /** + * Copyto back up. + * + * @param s3client the s 3 client + * @param s3Bucket the s 3 bucket + * @param from the from + * @param to the to + */ + private void copytoBackUp(AmazonS3 s3client,String s3Bucket,String from, String to){ + String[] keys = listKeys(s3client,s3Bucket,from); + String fileName =""; + for(String key:keys){ + try{ + fileName = key.substring(key.lastIndexOf('/')+1); + s3client.copyObject(s3Bucket,key,s3Bucket,to+"/"+fileName); + log.debug(" Copy "+fileName + " to backup folder"); + }catch(Exception e){ + log.info(" Copy "+fileName + "failed",e); + ErrorManageUtil.uploadError("all", "all", "all", e.getMessage()); + } + } + } + + /** + * Delete files. + * + * @param s3client the s 3 client + * @param s3Bucket the s 3 bucket + * @param folder the folder + */ + private void deleteFiles(AmazonS3 s3client,String s3Bucket,String folder){ + + String[] keys = listKeys(s3client,s3Bucket,folder); + DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(s3Bucket).withKeys((keys)); + + try{ + DeleteObjectsResult result = s3client.deleteObjects(multiObjectDeleteRequest); + log.debug("Files Deleted " +result.getDeletedObjects().stream().map(obj->obj.getKey()).collect(Collectors.toList())); + }catch(Exception e){ + log.error("Delete Failed",e); + ErrorManageUtil.uploadError("all", "all", "all", e.getMessage()); + } + } + + /** + * List keys. + * + * @param s3client the s 3 client + * @param s3Bucket the s 3 bucket + * @param folder the folder + * @return the string[] + */ + private String[] listKeys(AmazonS3 s3client,String s3Bucket,String folder){ + try{ + return s3client.listObjectsV2(new ListObjectsV2Request().withBucketName(s3Bucket).withPrefix(folder)).getObjectSummaries().stream().map(S3ObjectSummary::getKey).toArray(String[]::new); + }catch(Exception e){ + log.error("Error in listKeys",e); + ErrorManageUtil.uploadError("all", "all", "all", e.getMessage()); + } + return new String[0]; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/AzureVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/AzureVH.java new file mode 100644 index 000000000..cca33ece4 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/AzureVH.java @@ -0,0 +1,71 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import com.tmobile.pacbot.azure.inventory.collector.Util; + +public class AzureVH { + + private String discoverydate; + private String _cloudType = "Azure"; + private String subscription; + private String region; + private String subscriptionName; + private String resourceGroupName; + private String id; + + public String getSubscription() { + return subscription; + } + + public void setSubscription(String subscription) { + this.subscription = subscription; + } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getSubscriptionName() { + return subscriptionName; + } + + public void setSubscriptionName(String subscriptionName) { + this.subscriptionName = subscriptionName; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = Util.removeFirstSlash(id); + } + + public String getResourceGroupName() { + return resourceGroupName; + } + + public void setResourceGroupName(String resourceGroupName) { + this.resourceGroupName = resourceGroupName; + } + + public String getDiscoverydate() { + return discoverydate; + } + + public void setDiscoverydate(String discoverydate) { + this.discoverydate = discoverydate; + } + + public String get_cloudType() { + return _cloudType; + } + + public void set_cloudType(String _cloudType) { + this._cloudType = _cloudType; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/BatchAccountVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/BatchAccountVH.java new file mode 100644 index 000000000..e1351c68c --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/BatchAccountVH.java @@ -0,0 +1,143 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.Map; + +public class BatchAccountVH extends AzureVH { + + private String id; + private String name; + private String type; + private String location; + private Map tags; + private String provisioningState; + private String accountEndpoint; + private String poolQuota; + private String dedicatedCoreQuotaPerVMFamily; + private String poolAllocationMode; + private String dedicatedCoreQuota; + private String lowPriorityCoreQuota; + private String activeJobAndJobScheduleQuota; + private boolean dedicatedCoreQuotaPerVMFamilyEnforced; + private Map autoStorage; + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getLocation() { + return location; + } + + public Map getTags() { + return tags; + } + + public void setId(String id) { + this.id = id; + } + + public void setName(String name) { + this.name = name; + } + + public void setType(String type) { + this.type = type; + } + + public void setLocation(String location) { + this.location = location; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public String getProvisioningState() { + return provisioningState; + } + + public String getAccountEndpoint() { + return accountEndpoint; + } + + public String getPoolQuota() { + return poolQuota; + } + + public String getDedicatedCoreQuotaPerVMFamily() { + return dedicatedCoreQuotaPerVMFamily; + } + + public String getPoolAllocationMode() { + return poolAllocationMode; + } + + public String getDedicatedCoreQuota() { + return dedicatedCoreQuota; + } + + public String getLowPriorityCoreQuota() { + return lowPriorityCoreQuota; + } + + public String getActiveJobAndJobScheduleQuota() { + return activeJobAndJobScheduleQuota; + } + + public boolean isDedicatedCoreQuotaPerVMFamilyEnforced() { + return dedicatedCoreQuotaPerVMFamilyEnforced; + } + + public Map getAutoStorage() { + return autoStorage; + } + + public void setProvisioningState(String provisioningState) { + this.provisioningState = provisioningState; + } + + public void setAccountEndpoint(String accountEndpoint) { + this.accountEndpoint = accountEndpoint; + } + + public void setPoolQuota(String poolQuota) { + this.poolQuota = poolQuota; + } + + public void setDedicatedCoreQuotaPerVMFamily(String dedicatedCoreQuotaPerVMFamily) { + this.dedicatedCoreQuotaPerVMFamily = dedicatedCoreQuotaPerVMFamily; + } + + public void setPoolAllocationMode(String poolAllocationMode) { + this.poolAllocationMode = poolAllocationMode; + } + + public void setDedicatedCoreQuota(String dedicatedCoreQuota) { + this.dedicatedCoreQuota = dedicatedCoreQuota; + } + + public void setLowPriorityCoreQuota(String lowPriorityCoreQuota) { + this.lowPriorityCoreQuota = lowPriorityCoreQuota; + } + + public void setActiveJobAndJobScheduleQuota(String activeJobAndJobScheduleQuota) { + this.activeJobAndJobScheduleQuota = activeJobAndJobScheduleQuota; + } + + public void setDedicatedCoreQuotaPerVMFamilyEnforced(boolean dedicatedCoreQuotaPerVMFamilyEnforced) { + this.dedicatedCoreQuotaPerVMFamilyEnforced = dedicatedCoreQuotaPerVMFamilyEnforced; + } + + public void setAutoStorage(Map autoStorage) { + this.autoStorage = autoStorage; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/BlobContainerVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/BlobContainerVH.java new file mode 100644 index 000000000..6b910e464 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/BlobContainerVH.java @@ -0,0 +1,57 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@JsonSerialize +public class BlobContainerVH extends AzureVH { + + private String name; + private String type; + private String tag; + private HashMap propertiesMap; + private Map tags; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } + + public HashMap getPropertiesMap() { + return propertiesMap; + } + + public void setPropertiesMap(HashMap propertiesMap) { + this.propertiesMap = propertiesMap; + } + + public Map getTags() { + return tags; + } + + public void setTags(Map tags) { + this.tags = tags; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/CosmosDBVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/CosmosDBVH.java new file mode 100644 index 000000000..b3db88b1e --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/CosmosDBVH.java @@ -0,0 +1,74 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.List; +import java.util.Map; + +public class CosmosDBVH extends AzureVH { + private String key; + private String name; + private String type; + private Map tags; + private String ipRangeFilter; + private boolean multipleWriteLocationsEnabled; + private List virtualNetworkRuleList; + + + + public String getKey() { + return key; + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public Map getTags() { + return tags; + } + + + public void setKey(String key) { + this.key = key; + } + + public void setName(String name) { + this.name = name; + } + + public void setType(String type) { + this.type = type; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public String getIpRangeFilter() { + return ipRangeFilter; + } + + public boolean isMultipleWriteLocationsEnabled() { + return multipleWriteLocationsEnabled; + } + + public void setIpRangeFilter(String ipRangeFilter) { + this.ipRangeFilter = ipRangeFilter; + } + + public void setMultipleWriteLocationsEnabled(boolean multipleWriteLocationsEnabled) { + this.multipleWriteLocationsEnabled = multipleWriteLocationsEnabled; + } + + public List getVirtualNetworkRuleList() { + return virtualNetworkRuleList; + } + + public void setVirtualNetworkRuleList(List virtualNetworkRuleList) { + this.virtualNetworkRuleList = virtualNetworkRuleList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/DataDiskVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/DataDiskVH.java new file mode 100644 index 000000000..f368586be --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/DataDiskVH.java @@ -0,0 +1,84 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.Map; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.microsoft.azure.management.compute.implementation.DiskInner; + +@JsonSerialize +public class DataDiskVH extends AzureVH { + + private Boolean isAttachedToVirtualMachine; + private String key; + private String name; + private DiskInner diskInner; + private int sizeInGB; + private String type; + private String virtualMachineId; + private Map tags; + + public Boolean getIsAttachedToVirtualMachine() { + return isAttachedToVirtualMachine; + } + + public void setIsAttachedToVirtualMachine(Boolean isAttachedToVirtualMachine) { + this.isAttachedToVirtualMachine = isAttachedToVirtualMachine; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public DiskInner getDiskInner() { + return diskInner; + } + + public void setDiskInner(DiskInner diskInner) { + this.diskInner = diskInner; + } + + public int getSizeInGB() { + return sizeInGB; + } + + public void setSizeInGB(int sizeInGB) { + this.sizeInGB = sizeInGB; + } + + public Map getTags() { + return tags; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getVirtualMachineId() { + return virtualMachineId; + } + + public void setVirtualMachineId(String virtualMachineId) { + this.virtualMachineId = virtualMachineId; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/DatabricksVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/DatabricksVH.java new file mode 100644 index 000000000..95a1a4f72 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/DatabricksVH.java @@ -0,0 +1,57 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.Map; + +public class DatabricksVH extends AzureVH { + + + private String name; + private String type; + private String location; + private Map propertiesMap; + private Map skuMap; + + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getLocation() { + return location; + } + + public Map getPropertiesMap() { + return propertiesMap; + } + + public Map getSkuMap() { + return skuMap; + } + + + + public void setName(String name) { + this.name = name; + } + + public void setType(String type) { + this.type = type; + } + + public void setLocation(String location) { + this.location = location; + } + + public void setPropertiesMap(Map propertiesMap) { + this.propertiesMap = propertiesMap; + } + + public void setSkuMap(Map skuMap) { + this.skuMap = skuMap; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/ElasticPoolVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/ElasticPoolVH.java new file mode 100644 index 000000000..891ad208a --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/ElasticPoolVH.java @@ -0,0 +1,68 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +public class ElasticPoolVH { + private String id; + private String name; + private String edition; + private int size; + private int storageCapacity; + private int storageMB; + private int dtu; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getEdition() { + return edition; + } + + public void setEdition(String edition) { + this.edition = edition; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public int getStorageCapacity() { + return storageCapacity; + } + + public void setStorageCapacity(int storageCapacity) { + this.storageCapacity = storageCapacity; + } + + public int getStorageMB() { + return storageMB; + } + + public void setStorageMB(int storageMB) { + this.storageMB = storageMB; + } + + public int getDtu() { + return dtu; + } + + public void setDtu(int dtu) { + this.dtu = dtu; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/ErrorVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/ErrorVH.java new file mode 100644 index 000000000..8a2fc5ae5 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/ErrorVH.java @@ -0,0 +1,98 @@ +/******************************************************************************* + * Copyright 2018 T Mobile, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ + +package com.tmobile.pacbot.azure.inventory.vo; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +/** + * The Class ErrorVH. + */ +@JsonSerialize +public class ErrorVH { + + /** The type. */ + private String type; + + /** The region. */ + private String region; + + /** The exception. */ + private String exception; + + /** + * Gets the type. + * + * @return the type + */ + public String getType() { + return type; + } + + /** + * Sets the type. + * + * @param type the new type + */ + public void setType(String type) { + this.type = type; + } + + /** + * Gets the region. + * + * @return the region + */ + public String getRegion() { + return region; + } + + /** + * Sets the region. + * + * @param region the new region + */ + public void setRegion(String region) { + this.region = region; + } + + /** + * Gets the exception. + * + * @return the exception + */ + public String getException() { + return exception; + } + + /** + * Sets the exception. + * + * @param exception the new exception + */ + public void setException(String exception) { + this.exception = exception; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "ErrorVH [type=" + type + ", region=" + region + ", exception=" + + exception + "]"; + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/FailoverGroupVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/FailoverGroupVH.java new file mode 100644 index 000000000..3831c5769 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/FailoverGroupVH.java @@ -0,0 +1,68 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +public class FailoverGroupVH { + private String id; + private String name; + private String replicationState; + private String readOnlyEndpointPolicy; + private String readWriteEndpointPolicy; + private int size; + private int gracePeriod; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getReplicationState() { + return replicationState; + } + + public void setReplicationState(String replicationState) { + this.replicationState = replicationState; + } + + public String getReadOnlyEndpointPolicy() { + return readOnlyEndpointPolicy; + } + + public void setReadOnlyEndpointPolicy(String readOnlyEndpointPolicy) { + this.readOnlyEndpointPolicy = readOnlyEndpointPolicy; + } + + public String getReadWriteEndpointPolicy() { + return readWriteEndpointPolicy; + } + + public void setReadWriteEndpointPolicy(String readWriteEndpointPolicy) { + this.readWriteEndpointPolicy = readWriteEndpointPolicy; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public int getGracePeriod() { + return gracePeriod; + } + + public void setGracePeriod(int gracePeriod) { + this.gracePeriod = gracePeriod; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/FirewallRules.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/FirewallRules.java new file mode 100644 index 000000000..454852aa5 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/FirewallRules.java @@ -0,0 +1,69 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +public class FirewallRules { + private String name; + private String startIPAddress; + private String endIPAddress; +/* private String virtualNetworkName; + private String virtualNetworkSubnetId; + private String virtualNetworkResourceGroupName; + private String virtualNetworkState;*/ + + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getStartIPAddress() { + return startIPAddress; + } + + public void setStartIPAddress(String startIPAddress) { + this.startIPAddress = startIPAddress; + } + + public String getEndIPAddress() { + return endIPAddress; + } + + public void setEndIPAddress(String endIPAddress) { + this.endIPAddress = endIPAddress; + } + +/* public String getVirtualNetworkName() { + return virtualNetworkName; + } + + public void setVirtualNetworkName(String virtualNetworkName) { + this.virtualNetworkName = virtualNetworkName; + } + + public String getVirtualNetworkSubnetId() { + return virtualNetworkSubnetId; + } + + public void setVirtualNetworkSubnetId(String virtualNetworkSubnetId) { + this.virtualNetworkSubnetId = virtualNetworkSubnetId; + } + + public String getVirtualNetworkResourceGroupName() { + return virtualNetworkResourceGroupName; + } + + public void setVirtualNetworkResourceGroupName(String virtualNetworkResourceGroupName) { + this.virtualNetworkResourceGroupName = virtualNetworkResourceGroupName; + } + + public String getVirtualNetworkState() { + return virtualNetworkState; + } + + public void setVirtualNetworkState(String virtualNetworkState) { + this.virtualNetworkState = virtualNetworkState; + }*/ + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/IPconfigurationVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/IPconfigurationVH.java new file mode 100644 index 000000000..2c1d0b260 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/IPconfigurationVH.java @@ -0,0 +1,87 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +public class IPconfigurationVH { + + private String networkSecurityGroup; + private boolean isPrimary; + private String key; + private String name; + private String networkId; + private String privateIPAddress; + private String version; + private String publicIPAddressId; + private String type; + + public String getNetworkSecurityGroup() { + return networkSecurityGroup; + } + + public void setNetworkSecurityGroup(String networkSecurityGroup) { + this.networkSecurityGroup = networkSecurityGroup; + } + + public boolean isPrimary() { + return isPrimary; + } + + public void setPrimary(boolean isPrimary) { + this.isPrimary = isPrimary; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getNetworkId() { + return networkId; + } + + public void setNetworkId(String networkId) { + this.networkId = networkId; + } + + public String getPrivateIPAddress() { + return privateIPAddress; + } + + public void setPrivateIPAddress(String privateIPAddress) { + this.privateIPAddress = privateIPAddress; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getPublicIPAddressId() { + return publicIPAddressId; + } + + public void setPublicIPAddressId(String publicIPAddressId) { + this.publicIPAddressId = publicIPAddressId; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/LoadBalancerVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/LoadBalancerVH.java new file mode 100644 index 000000000..b2afea344 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/LoadBalancerVH.java @@ -0,0 +1,118 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.microsoft.azure.management.network.LoadBalancer; +import com.microsoft.azure.management.network.LoadBalancerPrivateFrontend; +import com.microsoft.azure.management.network.LoadBalancerPublicFrontend; +import com.microsoft.azure.management.network.LoadBalancingRule; +import com.microsoft.azure.management.resources.fluentcore.arm.Region; + +@JsonSerialize +public class LoadBalancerVH extends AzureVH { + + private int hashCode; + private String name; + + private String key; + private LoadBalancer refresh; + + private String regionName; + private String type; + private List publicIPAddressIds; + private Map tags; + private Map loadBalancingRules; + private Map privateFrontends; + private Map publicFrontends; + + public int getHashCode() { + return hashCode; + } + + public String getName() { + return name; + } + + public String getKey() { + return key; + } + + public Map getLoadBalancingRules() { + return loadBalancingRules; + } + + public Map getPrivateFrontends() { + return privateFrontends; + } + + public Map getPublicFrontends() { + return publicFrontends; + } + + public List getPublicIPAddressIds() { + return publicIPAddressIds; + } + + public LoadBalancer getRefresh() { + return refresh; + } + + public String getRegionName() { + return regionName; + } + + public Map getTags() { + return tags; + } + + public String getType() { + return type; + } + + public void setHashCode(int hashCode) { + this.hashCode = hashCode; + } + + public void setName(String name) { + this.name = name; + } + + public void setKey(String key) { + this.key = key; + } + + public void setLoadBalancingRules(Map loadBalancingRules) { + this.loadBalancingRules = loadBalancingRules; + } + + public void setPrivateFrontends(Map privateFrontends) { + this.privateFrontends = privateFrontends; + } + + public void setPublicFrontends(Map publicFrontends) { + this.publicFrontends = publicFrontends; + } + + public void setPublicIPAddressIds(List publicIPAddressIds) { + this.publicIPAddressIds = publicIPAddressIds; + } + + public void setRefresh(LoadBalancer refresh) { + this.refresh = refresh; + } + + public void setRegionName(String regionName) { + this.regionName = regionName; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public void setType(String type) { + this.type = type; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/MariaDBVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/MariaDBVH.java new file mode 100644 index 000000000..86187ded0 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/MariaDBVH.java @@ -0,0 +1,53 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.Map; + +public class MariaDBVH extends AzureVH { + + private String name; + private String type; + private String location; + private Map propertiesMap; + private Map skuMap; + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getLocation() { + return location; + } + + public Map getPropertiesMap() { + return propertiesMap; + } + + public Map getSkuMap() { + return skuMap; + } + + public void setName(String name) { + this.name = name; + } + + public void setType(String type) { + this.type = type; + } + + public void setLocation(String location) { + this.location = location; + } + + public void setPropertiesMap(Map propertiesMap) { + this.propertiesMap = propertiesMap; + } + + public void setSkuMap(Map skuMap) { + this.skuMap = skuMap; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/MySQLServerVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/MySQLServerVH.java new file mode 100644 index 000000000..515375745 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/MySQLServerVH.java @@ -0,0 +1,54 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.Map; + +public class MySQLServerVH extends AzureVH { + private String name; + private String type; + private String location; + private Map propertiesMap; + private Map skuMap; + + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getLocation() { + return location; + } + + public Map getPropertiesMap() { + return propertiesMap; + } + + public Map getSkuMap() { + return skuMap; + } + + + public void setName(String name) { + this.name = name; + } + + public void setType(String type) { + this.type = type; + } + + public void setLocation(String location) { + this.location = location; + } + + public void setPropertiesMap(Map propertiesMap) { + this.propertiesMap = propertiesMap; + } + + public void setSkuMap(Map skuMap) { + this.skuMap = skuMap; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NIIPConfigVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NIIPConfigVH.java new file mode 100644 index 000000000..e04e41ece --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NIIPConfigVH.java @@ -0,0 +1,56 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +public class NIIPConfigVH { + private String name; + private String privateIPAddress; + private String privateIPAddressVersion; + private String networkName; + private String subnetName; + private boolean isPrimary; + private String publicIPAddress; + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + public String getPrivateIPAddress() { + return privateIPAddress; + } + public void setPrivateIPAddress(String privateIPAddress) { + this.privateIPAddress = privateIPAddress; + } + public String getPrivateIPAddressVersion() { + return privateIPAddressVersion; + } + public void setPrivateIPAddressVersion(String privateIPAddressVersion) { + this.privateIPAddressVersion = privateIPAddressVersion; + } + public String getNetworkName() { + return networkName; + } + public void setNetworkName(String networkName) { + this.networkName = networkName; + } + public String getSubnetName() { + return subnetName; + } + public void setSubnetName(String subnetName) { + this.subnetName = subnetName; + } + public boolean isPrimary() { + return isPrimary; + } + public void setPrimary(boolean isPrimary) { + this.isPrimary = isPrimary; + } + public String getPublicIPAddress() { + return publicIPAddress; + } + public void setPublicIPAddress(String publicIPAddress) { + this.publicIPAddress = publicIPAddress; + } + + + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NSGSecurityRule.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NSGSecurityRule.java new file mode 100644 index 000000000..7bfa2477d --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NSGSecurityRule.java @@ -0,0 +1,141 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import com.microsoft.azure.management.network.NetworkSecurityRule; + +public class NSGSecurityRule { + + private String description; + private String access; + private int priority; + private String name; + private String protocol; + private List destinationAddressPrefixes = new ArrayList();; + private Set destinationApplicationSecurityGroupIds; + private List destinationPortRanges = new ArrayList();; + private List sourceAddressPrefixes = new ArrayList(); + private Set sourceApplicationSecurityGroupIds; + private List sourcePortRanges = new ArrayList();; + private boolean isDefault; + + public boolean isDefault() { + return isDefault; + } + + public void setDefault(boolean isDefault) { + this.isDefault = isDefault; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getAccess() { + return access; + } + + public void setAccess(String access) { + this.access = access; + } + + public int getPriority() { + return priority; + } + + public void setPriority(int priority) { + this.priority = priority; + } + + public List getDestinationAddressPrefixes() { + return destinationAddressPrefixes; + } + + public void setDestinationAddressPrefixes(List destinationAddressPrefixes) { + this.destinationAddressPrefixes = destinationAddressPrefixes; + } + + public Set getDestinationApplicationSecurityGroupIds() { + return destinationApplicationSecurityGroupIds; + } + + public void setDestinationApplicationSecurityGroupIds(Set destinationApplicationSecurityGroupIds) { + this.destinationApplicationSecurityGroupIds = destinationApplicationSecurityGroupIds; + } + + public List getDestinationPortRanges() { + return destinationPortRanges; + } + + public void setDestinationPortRanges(List destinationPortRanges) { + this.destinationPortRanges = destinationPortRanges; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public List getSourceAddressPrefixes() { + return sourceAddressPrefixes; + } + + public void setSourceAddressPrefixes(List sourceAddressPrefixes) { + this.sourceAddressPrefixes = sourceAddressPrefixes; + } + + public Set getSourceApplicationSecurityGroupIds() { + return sourceApplicationSecurityGroupIds; + } + + public void setSourceApplicationSecurityGroupIds(Set sourceApplicationSecurityGroupIds) { + this.sourceApplicationSecurityGroupIds = sourceApplicationSecurityGroupIds; + } + + public List getSourcePortRanges() { + return sourcePortRanges; + } + + public void setSourcePortRanges(List sourcePortRanges) { + this.sourcePortRanges = sourcePortRanges; + } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + } + + public void listValue(NetworkSecurityRule networkSecurityRule) { + sourceAddressPrefixes.addAll(networkSecurityRule.sourceAddressPrefixes()); + destinationAddressPrefixes.addAll(networkSecurityRule.destinationAddressPrefixes()); + sourcePortRanges.addAll(networkSecurityRule.sourcePortRanges()); + destinationPortRanges.addAll(networkSecurityRule.destinationPortRanges()); + if (networkSecurityRule.sourceAddressPrefix() != null && !networkSecurityRule.sourceAddressPrefix().isEmpty()) { + sourceAddressPrefixes.add(networkSecurityRule.sourceAddressPrefix()); + } + if (networkSecurityRule.destinationAddressPrefix() != null + && !networkSecurityRule.destinationAddressPrefix().isEmpty()) { + destinationAddressPrefixes.add(networkSecurityRule.destinationAddressPrefix()); + } + if (networkSecurityRule.sourcePortRange() != null && !networkSecurityRule.sourcePortRange().isEmpty()) { + sourcePortRanges.add(networkSecurityRule.sourcePortRange()); + } + if (networkSecurityRule.destinationPortRange() != null + && !networkSecurityRule.destinationPortRange().isEmpty()) { + destinationPortRanges.add(networkSecurityRule.destinationPortRange()); + } + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NSGSubnet.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NSGSubnet.java new file mode 100644 index 000000000..67912fbda --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NSGSubnet.java @@ -0,0 +1,34 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +public class NSGSubnet { + + private String addressPrefix; + private String name; + private String vnet; + + public String getVnet() { + return vnet; + } + + public void setVnet(String vnet) { + this.vnet = vnet; + } + + public String getAddressPrefix() { + return addressPrefix; + } + + public void setAddressPrefix(String addressPrefix) { + this.addressPrefix = addressPrefix; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NamespaceVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NamespaceVH.java new file mode 100644 index 000000000..29da02a72 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NamespaceVH.java @@ -0,0 +1,71 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.Map; + +public class NamespaceVH extends AzureVH { + + private String id; + private String name; + private String type; + private String location; + private Map tags; + private Map properties; + private Map sku; + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getLocation() { + return location; + } + + public Map getTags() { + return tags; + } + + public Map getProperties() { + return properties; + } + + public void setId(String id) { + this.id = id; + } + + public void setName(String name) { + this.name = name; + } + + public void setType(String type) { + this.type = type; + } + + public void setLocation(String location) { + this.location = location; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public void setProperties(Map properties) { + this.properties = properties; + } + + public Map getSku() { + return sku; + } + + public void setSku(Map sku) { + this.sku = sku; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NetworkInterfaceVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NetworkInterfaceVH.java new file mode 100644 index 000000000..068a1d839 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NetworkInterfaceVH.java @@ -0,0 +1,157 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.microsoft.azure.management.network.NicIPConfiguration; + +@JsonSerialize +public class NetworkInterfaceVH extends AzureVH { + + private String internalDnsNameLabel; + private String internalDomainNameSuffix; + private String internalFqdn; + private boolean isAcceleratedNetworkingEnabled; + private boolean isIPForwardingEnabled; + private String key; + private String macAddress; + private String name; + private String networkSecurityGroupId; + private String primaryPrivateIP; + private String type; + private String virtualMachineId; + private List appliedDnsServers; + private List dnsServers; + private List ipConfigurationList; + private Map tags; + + public List getAppliedDnsServers() { + return appliedDnsServers; + } + + public void setAppliedDnsServers(List appliedDnsServers) { + this.appliedDnsServers = appliedDnsServers; + } + + public List getDnsServers() { + return dnsServers; + } + + public void setDnsServers(List dnsServers) { + this.dnsServers = dnsServers; + } + + public String getInternalDnsNameLabel() { + return internalDnsNameLabel; + } + + public void setInternalDnsNameLabel(String internalDnsNameLabel) { + this.internalDnsNameLabel = internalDnsNameLabel; + } + + public String getInternalDomainNameSuffix() { + return internalDomainNameSuffix; + } + + public void setInternalDomainNameSuffix(String internalDomainNameSuffix) { + this.internalDomainNameSuffix = internalDomainNameSuffix; + } + + public String getInternalFqdn() { + return internalFqdn; + } + + public void setInternalFqdn(String internalFqdn) { + this.internalFqdn = internalFqdn; + } + + public boolean isAcceleratedNetworkingEnabled() { + return isAcceleratedNetworkingEnabled; + } + + public void setAcceleratedNetworkingEnabled(boolean isAcceleratedNetworkingEnabled) { + this.isAcceleratedNetworkingEnabled = isAcceleratedNetworkingEnabled; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getMacAddress() { + return macAddress; + } + + public void setMacAddress(String macAddress) { + this.macAddress = macAddress; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getNetworkSecurityGroupId() { + return networkSecurityGroupId; + } + + public void setNetworkSecurityGroupId(String networkSecurityGroupId) { + this.networkSecurityGroupId = networkSecurityGroupId; + } + + public String getPrimaryPrivateIP() { + return primaryPrivateIP; + } + + public void setPrimaryPrivateIP(String primaryPrivateIP) { + this.primaryPrivateIP = primaryPrivateIP; + } + + public Map getTags() { + return tags; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getVirtualMachineId() { + return virtualMachineId; + } + + public void setVirtualMachineId(String virtualMachineId) { + this.virtualMachineId = virtualMachineId; + } + + public List getIpConfigurationList() { + return ipConfigurationList; + } + + public void setIpConfigurationList(List ipConfigurationList) { + this.ipConfigurationList = ipConfigurationList; + } + + public boolean isIPForwardingEnabled() { + return isIPForwardingEnabled; + } + + public void setIPForwardingEnabled(boolean isIPForwardingEnabled) { + this.isIPForwardingEnabled = isIPForwardingEnabled; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NetworkVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NetworkVH.java new file mode 100644 index 000000000..29921eb27 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NetworkVH.java @@ -0,0 +1,104 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.microsoft.azure.management.network.Subnet; + +@JsonSerialize +public class NetworkVH extends AzureVH { + + private String ddosProtectionPlanId; + private int hashCode; + private boolean isDdosProtectionEnabled; + private boolean isVmProtectionEnabled; + private String key; + private String name; + private List addressSpaces; + private List dnsServerIPs; + private Map subnets; + private Map tags; + + public String getDdosProtectionPlanId() { + return ddosProtectionPlanId; + } + + public void setDdosProtectionPlanId(String ddosProtectionPlanId) { + this.ddosProtectionPlanId = ddosProtectionPlanId; + } + + public List getAddressSpaces() { + return addressSpaces; + } + + public void setAddressSpaces(List addressSpaces) { + this.addressSpaces = addressSpaces; + } + + public List getDnsServerIPs() { + return dnsServerIPs; + } + + public void setDnsServerIPs(List dnsServerIPs) { + this.dnsServerIPs = dnsServerIPs; + } + + public int getHashCode() { + return hashCode; + } + + public void setHashCode(int hashCode) { + this.hashCode = hashCode; + } + + public boolean isDdosProtectionEnabled() { + return isDdosProtectionEnabled; + } + + public void setDdosProtectionEnabled(boolean isDdosProtectionEnabled) { + this.isDdosProtectionEnabled = isDdosProtectionEnabled; + } + + public boolean isVmProtectionEnabled() { + return isVmProtectionEnabled; + } + + public void setVmProtectionEnabled(boolean isVmProtectionEnabled) { + this.isVmProtectionEnabled = isVmProtectionEnabled; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + + public Map getSubnets() { + return subnets; + } + + public void setSubnets(Map subnets) { + this.subnets = subnets; + } + + public Map getTags() { + return tags; + } + + public void setTags(Map tags) { + this.tags = tags; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PolicyDefinitionVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PolicyDefinitionVH.java new file mode 100644 index 000000000..f5b6de848 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PolicyDefinitionVH.java @@ -0,0 +1,51 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +public class PolicyDefinitionVH extends AzureVH { + + public String name; + public String description; + public String displayName; + public String policyType; + public String policyRule; + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getDisplayName() { + return displayName; + } + + public String getPolicyType() { + return policyType; + } + + public String getPolicyRule() { + return policyRule; + } + + public void setName(String name) { + this.name = name; + } + + public void setDescription(String description) { + this.description = description; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public void setPolicyType(String policyType) { + this.policyType = policyType; + } + + public void setPolicyRule(String policyRule) { + this.policyRule = policyRule; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PolicyStatesVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PolicyStatesVH.java new file mode 100644 index 000000000..3b00e037a --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PolicyStatesVH.java @@ -0,0 +1,266 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +public class PolicyStatesVH extends AzureVH { + private String timestamp; + private String resourceId; + private String policyAssignmentId; + private String policyDefinitionId; + private String effectiveParameters; + private Boolean isCompliant; + private String subscriptionId; + private String resourceType; + private String resourceLocation; + private String resourceGroup; + private String resourceTags; + private String policyAssignmentName; + private String policyAssignmentOwner; + private String policyAssignmentParameters; + private String policyAssignmentScope; + private String policyDefinitionName; + private String policyDefinitionAction; + private String policyDefinitionCategory; + private String policySetDefinitionId; + private String policySetDefinitionName; + private String policySetDefinitionOwner; + private String policySetDefinitionCategory; + private String policySetDefinitionParameters; + private String managementGroupIds; + private String policyDefinitionReferenceId; + private String policyDescription; + private String policyName; + private String policyType; + private String policyRule; + + public String getTimestamp() { + return timestamp; + } + + public String getResourceId() { + return resourceId; + } + + public String getPolicyAssignmentId() { + return policyAssignmentId; + } + + public String getPolicyDefinitionId() { + return policyDefinitionId; + } + + public String getEffectiveParameters() { + return effectiveParameters; + } + + public String getSubscriptionId() { + return subscriptionId; + } + + public String getResourceType() { + return resourceType; + } + + public String getResourceLocation() { + return resourceLocation; + } + + public String getResourceGroup() { + return resourceGroup; + } + + public String getResourceTags() { + return resourceTags; + } + + public String getPolicyAssignmentName() { + return policyAssignmentName; + } + + public String getPolicyAssignmentOwner() { + return policyAssignmentOwner; + } + + public String getPolicyAssignmentParameters() { + return policyAssignmentParameters; + } + + public String getPolicyAssignmentScope() { + return policyAssignmentScope; + } + + public String getPolicyDefinitionName() { + return policyDefinitionName; + } + + public String getPolicyDefinitionAction() { + return policyDefinitionAction; + } + + public String getPolicyDefinitionCategory() { + return policyDefinitionCategory; + } + + public String getPolicySetDefinitionId() { + return policySetDefinitionId; + } + + public String getPolicySetDefinitionName() { + return policySetDefinitionName; + } + + public String getPolicySetDefinitionOwner() { + return policySetDefinitionOwner; + } + + public String getPolicySetDefinitionCategory() { + return policySetDefinitionCategory; + } + + public String getPolicySetDefinitionParameters() { + return policySetDefinitionParameters; + } + + public String getManagementGroupIds() { + return managementGroupIds; + } + + public String getPolicyDefinitionReferenceId() { + return policyDefinitionReferenceId; + } + + public void setTimestamp(String timestamp) { + this.timestamp = timestamp; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public void setPolicyAssignmentId(String policyAssignmentId) { + this.policyAssignmentId = policyAssignmentId; + } + + public void setPolicyDefinitionId(String policyDefinitionId) { + this.policyDefinitionId = policyDefinitionId; + } + + public void setEffectiveParameters(String effectiveParameters) { + this.effectiveParameters = effectiveParameters; + } + + public Boolean getIsCompliant() { + return isCompliant; + } + + public void setIsCompliant(Boolean isCompliant) { + this.isCompliant = isCompliant; + } + + public void setSubscriptionId(String subscriptionId) { + this.subscriptionId = subscriptionId; + } + + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + public void setResourceLocation(String resourceLocation) { + this.resourceLocation = resourceLocation; + } + + public void setResourceGroup(String resourceGroup) { + this.resourceGroup = resourceGroup; + } + + public void setResourceTags(String resourceTags) { + this.resourceTags = resourceTags; + } + + public void setPolicyAssignmentName(String policyAssignmentName) { + this.policyAssignmentName = policyAssignmentName; + } + + public void setPolicyAssignmentOwner(String policyAssignmentOwner) { + this.policyAssignmentOwner = policyAssignmentOwner; + } + + public void setPolicyAssignmentParameters(String policyAssignmentParameters) { + this.policyAssignmentParameters = policyAssignmentParameters; + } + + public void setPolicyAssignmentScope(String policyAssignmentScope) { + this.policyAssignmentScope = policyAssignmentScope; + } + + public void setPolicyDefinitionName(String policyDefinitionName) { + this.policyDefinitionName = policyDefinitionName; + } + + public void setPolicyDefinitionAction(String policyDefinitionAction) { + this.policyDefinitionAction = policyDefinitionAction; + } + + public void setPolicyDefinitionCategory(String policyDefinitionCategory) { + this.policyDefinitionCategory = policyDefinitionCategory; + } + + public void setPolicySetDefinitionId(String policySetDefinitionId) { + this.policySetDefinitionId = policySetDefinitionId; + } + + public void setPolicySetDefinitionName(String policySetDefinitionName) { + this.policySetDefinitionName = policySetDefinitionName; + } + + public void setPolicySetDefinitionOwner(String policySetDefinitionOwner) { + this.policySetDefinitionOwner = policySetDefinitionOwner; + } + + public void setPolicySetDefinitionCategory(String policySetDefinitionCategory) { + this.policySetDefinitionCategory = policySetDefinitionCategory; + } + + public void setPolicySetDefinitionParameters(String policySetDefinitionParameters) { + this.policySetDefinitionParameters = policySetDefinitionParameters; + } + + public void setManagementGroupIds(String managementGroupIds) { + this.managementGroupIds = managementGroupIds; + } + + public void setPolicyDefinitionReferenceId(String policyDefinitionReferenceId) { + this.policyDefinitionReferenceId = policyDefinitionReferenceId; + } + + public String getPolicyType() { + return policyType; + } + + public String getPolicyRule() { + return policyRule; + } + + public String getPolicyDescription() { + return policyDescription; + } + + public String getPolicyName() { + return policyName; + } + + public void setPolicyDescription(String policyDescription) { + this.policyDescription = policyDescription; + } + + public void setPolicyName(String policyName) { + this.policyName = policyName; + } + + public void setPolicyType(String policyType) { + this.policyType = policyType; + } + + public void setPolicyRule(String policyRule) { + this.policyRule = policyRule; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PostgreSQLServerVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PostgreSQLServerVH.java new file mode 100644 index 000000000..6f9185807 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PostgreSQLServerVH.java @@ -0,0 +1,53 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.Map; + +public class PostgreSQLServerVH extends AzureVH { + + private String name; + private String type; + private String location; + private Map propertiesMap; + private Map skuMap; + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getLocation() { + return location; + } + + public Map getPropertiesMap() { + return propertiesMap; + } + + public Map getSkuMap() { + return skuMap; + } + + public void setName(String name) { + this.name = name; + } + + public void setType(String type) { + this.type = type; + } + + public void setLocation(String location) { + this.location = location; + } + + public void setPropertiesMap(Map propertiesMap) { + this.propertiesMap = propertiesMap; + } + + public void setSkuMap(Map skuMap) { + this.skuMap = skuMap; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PublicIpAddressVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PublicIpAddressVH.java new file mode 100644 index 000000000..d114a6d9b --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PublicIpAddressVH.java @@ -0,0 +1,106 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.Map; + +public class PublicIpAddressVH extends AzureVH { + private String name; + private String fqdn; + private String reverseFqdn; + private String ipAddress; + private String key; + private String regionName; + private String version; + private String type; + private String kind; + private int idleTimeoutInMinutes; + private Map tags; + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getKind() { + return kind; + } + + public Map getTags() { + return tags; + } + + public void setName(String name) { + this.name = name; + } + + public void setType(String type) { + this.type = type; + } + + public void setKind(String kind) { + this.kind = kind; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public String getFqdn() { + return fqdn; + } + + public String getReverseFqdn() { + return reverseFqdn; + } + + public String getIpAddress() { + return ipAddress; + } + + public String getKey() { + return key; + } + + public String getRegionName() { + return regionName; + } + + public String getVersion() { + return version; + } + + public int getIdleTimeoutInMinutes() { + return idleTimeoutInMinutes; + } + + public void setFqdn(String fqdn) { + this.fqdn = fqdn; + } + + public void setReverseFqdn(String reverseFqdn) { + this.reverseFqdn = reverseFqdn; + } + + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + } + + public void setKey(String key) { + this.key = key; + } + + public void setRegionName(String regionName) { + this.regionName = regionName; + } + + public void setVersion(String version) { + this.version = version; + } + + public void setIdleTimeoutInMinutes(int idleTimeoutInMinutes) { + this.idleTimeoutInMinutes = idleTimeoutInMinutes; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RecommendationVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RecommendationVH.java new file mode 100644 index 000000000..283be1689 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RecommendationVH.java @@ -0,0 +1,17 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.Map; + +public class RecommendationVH extends AzureVH{ + + private Map recommendation; + + public Map getRecommendation() { + return recommendation; + } + + public void setRecommendation(Map recommendation) { + this.recommendation = recommendation; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RegAppCertificateVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RegAppCertificateVH.java new file mode 100644 index 000000000..52264db43 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RegAppCertificateVH.java @@ -0,0 +1,76 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +public class RegAppCertificateVH { + private String customKeyIdentifier; + private String endDateTime; + private String keyId; + private String startDateTime; + private String type; + private String usage; + private String key; + private String displayName; + + public String getCustomKeyIdentifier() { + return customKeyIdentifier; + } + + public void setCustomKeyIdentifier(String customKeyIdentifier) { + this.customKeyIdentifier = customKeyIdentifier; + } + + public String getEndDateTime() { + return endDateTime; + } + + public void setEndDateTime(String endDateTime) { + this.endDateTime = endDateTime; + } + + public String getKeyId() { + return keyId; + } + + public void setKeyId(String keyId) { + this.keyId = keyId; + } + + public String getStartDateTime() { + return startDateTime; + } + + public void setStartDateTime(String startDateTime) { + this.startDateTime = startDateTime; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getUsage() { + return usage; + } + + public void setUsage(String usage) { + this.usage = usage; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getDisplayName() { + return displayName; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RegAppSecretVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RegAppSecretVH.java new file mode 100644 index 000000000..d155086ac --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RegAppSecretVH.java @@ -0,0 +1,67 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +public class RegAppSecretVH { + private String customKeyIdentifier; + private String endDateTime; + private String keyId; + private String startDateTime; + private String secretText; + private String hint; + private String displayName; + + public String getCustomKeyIdentifier() { + return customKeyIdentifier; + } + + public void setCustomKeyIdentifier(String customKeyIdentifier) { + this.customKeyIdentifier = customKeyIdentifier; + } + + public String getEndDateTime() { + return endDateTime; + } + + public void setEndDateTime(String endDateTime) { + this.endDateTime = endDateTime; + } + + public String getKeyId() { + return keyId; + } + + public void setKeyId(String keyId) { + this.keyId = keyId; + } + + public String getStartDateTime() { + return startDateTime; + } + + public void setStartDateTime(String startDateTime) { + this.startDateTime = startDateTime; + } + + public String getSecretText() { + return secretText; + } + + public void setSecretText(String secretText) { + this.secretText = secretText; + } + + public String getHint() { + return hint; + } + + public void setHint(String hint) { + this.hint = hint; + } + + public String getDisplayName() { + return displayName; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RegisteredApplicationVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RegisteredApplicationVH.java new file mode 100644 index 000000000..a837cc680 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RegisteredApplicationVH.java @@ -0,0 +1,70 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.List; + +public class RegisteredApplicationVH extends AzureVH { + private String objectId; + private String appId; + private String createdDateTime; + private String displayName; + private String publisherDomain; + private List certificateList; + private List secretList; + + public String getObjectId() { + return objectId; + } + + public void setObjectId(String objectId) { + this.objectId = objectId; + } + + public String getAppId() { + return appId; + } + + public void setAppId(String appId) { + this.appId = appId; + } + + public String getCreatedDateTime() { + return createdDateTime; + } + + public void setCreatedDateTime(String createdDateTime) { + this.createdDateTime = createdDateTime; + } + + public String getDisplayName() { + return displayName; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public String getPublisherDomain() { + return publisherDomain; + } + + public void setPublisherDomain(String publisherDomain) { + this.publisherDomain = publisherDomain; + } + + public List getCertificateList() { + return certificateList; + } + + public void setCertificateList(List certificateList) { + this.certificateList = certificateList; + } + + public List getSecretList() { + return secretList; + } + + public void setSecretList(List secretList) { + this.secretList = secretList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/ResourceGroupVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/ResourceGroupVH.java new file mode 100644 index 000000000..f0f7b8ae2 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/ResourceGroupVH.java @@ -0,0 +1,53 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.Map; + +public class ResourceGroupVH extends AzureVH { + + private String key; + private String type; + private String provisioningState; + private String regionName; + private Map tags; + + public String getKey() { + return key; + } + + public String getType() { + return type; + } + + public String getProvisioningState() { + return provisioningState; + } + + public String getRegionName() { + return regionName; + } + + public Map getTags() { + return tags; + } + + public void setKey(String key) { + this.key = key; + } + + public void setType(String type) { + this.type = type; + } + + public void setProvisioningState(String provisioningState) { + this.provisioningState = provisioningState; + } + + public void setRegionName(String regionName) { + this.regionName = regionName; + } + + public void setTags(Map tags) { + this.tags = tags; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RouteTableSubnet.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RouteTableSubnet.java new file mode 100644 index 000000000..4f7cb2b7b --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RouteTableSubnet.java @@ -0,0 +1,35 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +public class RouteTableSubnet { + + private String addressPrefix; + private String name; + private String vnet; + + public String getVnet() { + return vnet; + } + + public void setVnet(String vnet) { + this.vnet = vnet; + } + + public String getAddressPrefix() { + return addressPrefix; + } + + public void setAddressPrefix(String addressPrefix) { + this.addressPrefix = addressPrefix; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RouteTableVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RouteTableVH.java new file mode 100644 index 000000000..eb10bc3cd --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RouteTableVH.java @@ -0,0 +1,80 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.List; +import java.util.Map; + +public class RouteTableVH extends AzureVH { + private String name; + private String key; + private String regionName; + private String type; + private int hashCode; + private Map tags; + private List subnetList; + private List routeVHlist; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getName() { + return name; + } + + public String getKey() { + return key; + } + + public String getRegionName() { + return regionName; + } + + public int getHashCode() { + return hashCode; + } + + public void setHashCode(int hashCode) { + this.hashCode = hashCode; + } + + public Map getTags() { + return tags; + } + + public void setName(String name) { + this.name = name; + } + + public void setKey(String key) { + this.key = key; + } + + public void setRegionName(String regionName) { + this.regionName = regionName; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public List getSubnetList() { + return subnetList; + } + + public void setSubnetList(List subnetList) { + this.subnetList = subnetList; + } + + public List getRouteVHlist() { + return routeVHlist; + } + + public void setRouteVHlist(List routeVHlist) { + this.routeVHlist = routeVHlist; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RouteVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RouteVH.java new file mode 100644 index 000000000..1f6658b2f --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/RouteVH.java @@ -0,0 +1,32 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +public class RouteVH { + private String name; + private String addressPrefix; + private String nextHop; + + public String getName() { + return name; + } + + public String getAddressPrefix() { + return addressPrefix; + } + + public String getNextHop() { + return nextHop; + } + + public void setName(String name) { + this.name = name; + } + + public void setAddressPrefix(String addressPrefix) { + this.addressPrefix = addressPrefix; + } + + public void setNextHop(String nextHop) { + this.nextHop = nextHop; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SQLDatabaseVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SQLDatabaseVH.java new file mode 100644 index 000000000..a012fb961 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SQLDatabaseVH.java @@ -0,0 +1,138 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@JsonSerialize +public class SQLDatabaseVH extends AzureVH { + private String creationDate; + private UUID currentServiceObjectiveId; + private String collation; + private String databaseId; + private String defaultSecondaryLocation; + private String earliestRestoreDate; + private String edition; + private String elasticPoolName; + private boolean isDataWarehouse; + private String name; + private String status; + private String serverName; + private List> firewallRuleDetails; + private Map tags; + + public UUID getCurrentServiceObjectiveId() { + return currentServiceObjectiveId; + } + + public void setCurrentServiceObjectiveId(UUID currentServiceObjectiveId) { + this.currentServiceObjectiveId = currentServiceObjectiveId; + } + + public String getCollation() { + return collation; + } + + public void setCollation(String collation) { + this.collation = collation; + } + + public String getDatabaseId() { + return databaseId; + } + + public void setDatabaseId(String databaseId) { + this.databaseId = databaseId; + } + + public String getDefaultSecondaryLocation() { + return defaultSecondaryLocation; + } + + public void setDefaultSecondaryLocation(String defaultSecondaryLocation) { + this.defaultSecondaryLocation = defaultSecondaryLocation; + } + + public String getEdition() { + return edition; + } + + public void setEdition(String edition) { + this.edition = edition; + } + + public String getElasticPoolName() { + return elasticPoolName; + } + + public void setElasticPoolName(String elasticPoolName) { + this.elasticPoolName = elasticPoolName; + } + + public boolean isDataWarehouse() { + return isDataWarehouse; + } + + public void setDataWarehouse(boolean isDataWarehouse) { + this.isDataWarehouse = isDataWarehouse; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getCreationDate() { + return creationDate; + } + + public void setCreationDate(String creationDate) { + this.creationDate = creationDate; + } + + public String getEarliestRestoreDate() { + return earliestRestoreDate; + } + + public void setEarliestRestoreDate(String earliestRestoreDate) { + this.earliestRestoreDate = earliestRestoreDate; + } + + public String getServerName() { + return serverName; + } + + public void setServerName(String serverName) { + this.serverName = serverName; + } + + public List> getFirewallRuleDetails() { + return firewallRuleDetails; + } + + public void setFirewallRuleDetails(List> firewallRuleDetails) { + this.firewallRuleDetails = firewallRuleDetails; + } + + public Map getTags() { + return tags; + } + + public void setTags(Map tags) { + this.tags = tags; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SQLServerVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SQLServerVH.java new file mode 100644 index 000000000..078fcf827 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SQLServerVH.java @@ -0,0 +1,122 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@JsonSerialize +public class SQLServerVH extends AzureVH { + + private String kind; + private String name; + private String regionName; + private String state; + private String systemAssignedManagedServiceIdentityPrincipalId; + private String systemAssignedManagedServiceIdentityTenantId; + private Map tags; + private String version; + private String administratorLogin; + List elasticPoolList; + List failoverGroupList; + private List> firewallRuleDetails; + + public String getKind() { + return kind; + } + + public void setKind(String kind) { + this.kind = kind; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getRegionName() { + return regionName; + } + + public void setRegionName(String regionName) { + this.regionName = regionName; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public String getSystemAssignedManagedServiceIdentityPrincipalId() { + return systemAssignedManagedServiceIdentityPrincipalId; + } + + public void setSystemAssignedManagedServiceIdentityPrincipalId( + String systemAssignedManagedServiceIdentityPrincipalId) { + this.systemAssignedManagedServiceIdentityPrincipalId = systemAssignedManagedServiceIdentityPrincipalId; + } + + public String getSystemAssignedManagedServiceIdentityTenantId() { + return systemAssignedManagedServiceIdentityTenantId; + } + + public void setSystemAssignedManagedServiceIdentityTenantId(String systemAssignedManagedServiceIdentityTenantId) { + this.systemAssignedManagedServiceIdentityTenantId = systemAssignedManagedServiceIdentityTenantId; + } + + public Map getTags() { + return tags; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getAdministratorLogin() { + return administratorLogin; + } + + public void setAdministratorLogin(String administratorLogin) { + this.administratorLogin = administratorLogin; + } + + public List> getFirewallRuleDetails() { + return firewallRuleDetails; + } + + public void setFirewallRuleDetails(List> firewallRuleDetails) { + this.firewallRuleDetails = firewallRuleDetails; + } + + public List getElasticPoolList() { + return elasticPoolList; + } + + public void setElasticPoolList(List elasticPoolList) { + this.elasticPoolList = elasticPoolList; + } + + public List getFailoverGroupList() { + return failoverGroupList; + } + + public void setFailoverGroupList(List failoverGroupList) { + this.failoverGroupList = failoverGroupList; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SearchServiceVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SearchServiceVH.java new file mode 100644 index 000000000..babf02d52 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SearchServiceVH.java @@ -0,0 +1,62 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.Map; + +public class SearchServiceVH extends AzureVH { + + private String id; + private String name; + private String type; + private String location; + private Map properties; + private Map sku; + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getLocation() { + return location; + } + + public Map getProperties() { + return properties; + } + + public Map getSku() { + return sku; + } + + public void setId(String id) { + this.id = id; + } + + public void setName(String name) { + this.name = name; + } + + public void setType(String type) { + this.type = type; + } + + public void setLocation(String location) { + this.location = location; + } + + public void setProperties(Map properties) { + this.properties = properties; + } + + public void setSku(Map sku) { + this.sku = sku; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SecurityAlertsVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SecurityAlertsVH.java new file mode 100644 index 000000000..7858e07bd --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SecurityAlertsVH.java @@ -0,0 +1,35 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.HashMap; +import java.util.Map; + +public class SecurityAlertsVH extends AzureVH { + private String name; + private String type; + private HashMap propertiesMap; + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public HashMap getPropertiesMap() { + return propertiesMap; + } + + public void setName(String name) { + this.name = name; + } + + public void setType(String type) { + this.type = type; + } + + public void setPropertiesMap(HashMap propertiesMap) { + this.propertiesMap = propertiesMap; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SecurityGroupVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SecurityGroupVH.java new file mode 100644 index 000000000..c7fe4890c --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SecurityGroupVH.java @@ -0,0 +1,78 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@JsonSerialize +public class SecurityGroupVH extends AzureVH { + + private String key; + private String name; + private Map tags; + private Set networkInterfaceIds; + private List subnetList; + private List inBoundSecurityRules; + private List outBoundSecurityRules; + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Map getTags() { + return tags; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public Set getNetworkInterfaceIds() { + return networkInterfaceIds; + } + + public void setNetworkInterfaceIds(Set networkInterfaceIds) { + this.networkInterfaceIds = networkInterfaceIds; + } + + public List getSubnetList() { + return subnetList; + } + + public void setSubnetList(List subnetList) { + this.subnetList = subnetList; + } + + public List getInBoundSecurityRules() { + return inBoundSecurityRules; + } + + public void setInBoundSecurityRules(List inBoundSecurityRules) { + this.inBoundSecurityRules = inBoundSecurityRules; + } + + public List getOutBoundSecurityRules() { + return outBoundSecurityRules; + } + + public void setOutBoundSecurityRules(List outBoundSecurityRules) { + this.outBoundSecurityRules = outBoundSecurityRules; + } + + + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SitesVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SitesVH.java new file mode 100644 index 000000000..b5e231c1d --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SitesVH.java @@ -0,0 +1,61 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.Map; + +public class SitesVH extends AzureVH { + private String id; + private String etag; + private String location; + private String name; + private String type; + private Map tags; + private Map properties; + + + public String getId() { + return id; + } + public String getEtag() { + return etag; + } + public String getLocation() { + return location; + } + public String getName() { + return name; + } + public String getType() { + return type; + } + + public void setId(String id) { + this.id = id; + } + public void setEtag(String etag) { + this.etag = etag; + } + public void setLocation(String location) { + this.location = location; + } + public void setName(String name) { + this.name = name; + } + public void setType(String type) { + this.type = type; + } + + public Map getProperties() { + return properties; + } + public void setProperties(Map properties) { + this.properties = properties; + } + public Map getTags() { + return tags; + } + public void setTags(Map tags) { + this.tags = tags; + } + + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SnapshotVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SnapshotVH.java new file mode 100644 index 000000000..53b2c9762 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SnapshotVH.java @@ -0,0 +1,61 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.Map; + +public class SnapshotVH extends AzureVH { + private String name; + private String type; + private String key; + private String regionName; + private int sizeInGB; + private Map tags; + + public Map getTags() { + return tags; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getKey() { + return key; + } + + public String getRegionName() { + return regionName; + } + + public int getSizeInGB() { + return sizeInGB; + } + + public void setKey(String key) { + this.key = key; + } + + public void setRegionName(String regionName) { + this.regionName = regionName; + } + + public void setSizeInGB(int sizeInGB) { + this.sizeInGB = sizeInGB; + } + + public void setName(String name) { + this.name = name; + } + + public void setType(String type) { + this.type = type; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/StorageAccountVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/StorageAccountVH.java new file mode 100644 index 000000000..56486fe9d --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/StorageAccountVH.java @@ -0,0 +1,261 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.List; +import java.util.Map; + +import org.joda.time.DateTime; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@JsonSerialize +public class StorageAccountVH extends AzureVH { + + private String resourceGroupName; + private boolean canAccessFromAzureServices; + private boolean isAccessAllowedFromAllNetworks; + private boolean isAzureFilesAadIntegrationEnabled; + private boolean isHnsEnabled; + private String name; + private String regionName; + private String systemAssignedManagedServiceIdentityPrincipalId; + private String systemAssignedManagedServiceIdentityTenantId; + private List endPoints; + private List ipAddressesWithAccess; + private List ipAddressRangesWithAccess; + private List networkSubnetsWithAccess; + private Map tags; + private String kind; + private Map endpointsMap; + + /** + * @return the resourceGroupName + */ + public String getResourceGroupName() { + return resourceGroupName; + } + + /** + * @param resourceGroupName + * the resourceGroupName to set + */ + public void setResourceGroupName(String resourceGroupName) { + this.resourceGroupName = resourceGroupName; + } + + /** + * @return the canAccessFromAzureServices + */ + public boolean isCanAccessFromAzureServices() { + return canAccessFromAzureServices; + } + + /** + * @param canAccessFromAzureServices + * the canAccessFromAzureServices to set + */ + public void setCanAccessFromAzureServices(boolean canAccessFromAzureServices) { + this.canAccessFromAzureServices = canAccessFromAzureServices; + } + + /** + * @return the creationTime + */ + + /** + * @return the endPoints + */ + public List getEndPoints() { + return endPoints; + } + + /** + * @param endPoints + * the endPoints to set + */ + public void setEndPoints(List endPoints) { + this.endPoints = endPoints; + } + + /** + * @return the ipAddressRangesWithAccess + */ + public List getIpAddressRangesWithAccess() { + return ipAddressRangesWithAccess; + } + + /** + * @param ipAddressRangesWithAccess + * the ipAddressRangesWithAccess to set + */ + public void setIpAddressRangesWithAccess(List ipAddressRangesWithAccess) { + this.ipAddressRangesWithAccess = ipAddressRangesWithAccess; + } + + /** + * @return the isAccessAllowedFromAllNetworks + */ + public boolean isAccessAllowedFromAllNetworks() { + return isAccessAllowedFromAllNetworks; + } + + /** + * @param isAccessAllowedFromAllNetworks + * the isAccessAllowedFromAllNetworks to set + */ + public void setAccessAllowedFromAllNetworks(boolean isAccessAllowedFromAllNetworks) { + this.isAccessAllowedFromAllNetworks = isAccessAllowedFromAllNetworks; + } + + /** + * @return the isAzureFilesAadIntegrationEnabled + */ + public boolean isAzureFilesAadIntegrationEnabled() { + return isAzureFilesAadIntegrationEnabled; + } + + /** + * @param isAzureFilesAadIntegrationEnabled + * the isAzureFilesAadIntegrationEnabled to set + */ + public void setAzureFilesAadIntegrationEnabled(boolean isAzureFilesAadIntegrationEnabled) { + this.isAzureFilesAadIntegrationEnabled = isAzureFilesAadIntegrationEnabled; + } + + /** + * @return the isHnsEnabled + */ + public boolean isHnsEnabled() { + return isHnsEnabled; + } + + /** + * @param isHnsEnabled + * the isHnsEnabled to set + */ + public void setHnsEnabled(boolean isHnsEnabled) { + this.isHnsEnabled = isHnsEnabled; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name + * the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the regionName + */ + public String getRegionName() { + return regionName; + } + + /** + * @param regionName + * the regionName to set + */ + public void setRegionName(String regionName) { + this.regionName = regionName; + } + + /** + * @return the networkSubnetsWithAccess + */ + public List getNetworkSubnetsWithAccess() { + return networkSubnetsWithAccess; + } + + /** + * @param networkSubnetsWithAccess + * the networkSubnetsWithAccess to set + */ + public void setNetworkSubnetsWithAccess(List networkSubnetsWithAccess) { + this.networkSubnetsWithAccess = networkSubnetsWithAccess; + } + + /** + * @return the systemAssignedManagedServiceIdentityPrincipalId + */ + public String getSystemAssignedManagedServiceIdentityPrincipalId() { + return systemAssignedManagedServiceIdentityPrincipalId; + } + + /** + * @param systemAssignedManagedServiceIdentityPrincipalId + * the systemAssignedManagedServiceIdentityPrincipalId to set + */ + public void setSystemAssignedManagedServiceIdentityPrincipalId( + String systemAssignedManagedServiceIdentityPrincipalId) { + this.systemAssignedManagedServiceIdentityPrincipalId = systemAssignedManagedServiceIdentityPrincipalId; + } + + /** + * @return the systemAssignedManagedServiceIdentityTenantId + */ + public String getSystemAssignedManagedServiceIdentityTenantId() { + return systemAssignedManagedServiceIdentityTenantId; + } + + /** + * @param systemAssignedManagedServiceIdentityTenantId + * the systemAssignedManagedServiceIdentityTenantId to set + */ + public void setSystemAssignedManagedServiceIdentityTenantId(String systemAssignedManagedServiceIdentityTenantId) { + this.systemAssignedManagedServiceIdentityTenantId = systemAssignedManagedServiceIdentityTenantId; + } + + /** + * @return the tags + */ + public Map getTags() { + return tags; + } + + /** + * @param tags + * the tags to set + */ + public void setTags(Map tags) { + this.tags = tags; + } + + /** + * @return the ipAddressesWithAccess + */ + public List getIpAddressesWithAccess() { + return ipAddressesWithAccess; + } + + /** + * @param ipAddressesWithAccess + * the ipAddressesWithAccess to set + */ + public void setIpAddressesWithAccess(List ipAddressesWithAccess) { + this.ipAddressesWithAccess = ipAddressesWithAccess; + } + + public String getKind() { + return kind; + } + + public void setKind(String kind) { + this.kind = kind; + } + + public Map getEndpointsMap() { + return endpointsMap; + } + + public void setEndpointsMap(Map endpointsMap) { + this.endpointsMap = endpointsMap; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SubnetVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SubnetVH.java new file mode 100644 index 000000000..4a3ea024b --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SubnetVH.java @@ -0,0 +1,90 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.List; +import java.util.Map; + +public class SubnetVH extends AzureVH { + + private String id; + private String etag; + private String name; + private String type; + private List> ipConfigurations; + private String addressPrefix; + private String privateLinkServiceNetworkPolicies; + private String provisioningState; + private String privateEndpointNetworkPolicies; + + public List> getIpConfigurations() { + return ipConfigurations; + } + + public String getAddressPrefix() { + return addressPrefix; + } + + public String getPrivateLinkServiceNetworkPolicies() { + return privateLinkServiceNetworkPolicies; + } + + public String getProvisioningState() { + return provisioningState; + } + + public String getPrivateEndpointNetworkPolicies() { + return privateEndpointNetworkPolicies; + } + + public void setIpConfigurations(List> ipConfigurations) { + this.ipConfigurations = ipConfigurations; + } + + public void setAddressPrefix(String addressPrefix) { + this.addressPrefix = addressPrefix; + } + + public void setPrivateLinkServiceNetworkPolicies(String privateLinkServiceNetworkPolicies) { + this.privateLinkServiceNetworkPolicies = privateLinkServiceNetworkPolicies; + } + + public void setProvisioningState(String provisioningState) { + this.provisioningState = provisioningState; + } + + public void setPrivateEndpointNetworkPolicies(String privateEndpointNetworkPolicies) { + this.privateEndpointNetworkPolicies = privateEndpointNetworkPolicies; + } + + public String getId() { + return id; + } + + public String getEtag() { + return etag; + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public void setId(String id) { + this.id = id; + } + + public void setEtag(String etag) { + this.etag = etag; + } + + public void setName(String name) { + this.name = name; + } + + public void setType(String type) { + this.type = type; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SubscriptionVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SubscriptionVH.java new file mode 100644 index 000000000..34560eec8 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SubscriptionVH.java @@ -0,0 +1,23 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +public class SubscriptionVH { + + @Override + public String toString() { + return "[subscriptionId=" + subscriptionId + ", subscriptionName=" + subscriptionName + "]"; + } + private String subscriptionId; + private String subscriptionName; + public String getSubscriptionId() { + return subscriptionId; + } + public void setSubscriptionId(String subscription) { + this.subscriptionId = subscription; + } + public String getSubscriptionName() { + return subscriptionName; + } + public void setSubscriptionName(String subscriptionName) { + this.subscriptionName = subscriptionName; + } +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VMDiskVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VMDiskVH.java new file mode 100644 index 000000000..29af4022b --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VMDiskVH.java @@ -0,0 +1,41 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +public class VMDiskVH { + String storageAccountType; + String name; + Integer sizeInGB; + String type; + String cachingType; + + public String getType() { + return type; + } + public String getCachingType() { + return cachingType; + } + public void setCachingType(String cachingType) { + this.cachingType = cachingType; + } + public void setType(String type) { + this.type = type; + } + public String getStorageAccountType() { + return storageAccountType; + } + public void setStorageAccountType(String type) { + this.storageAccountType = type; + } + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + public Integer getSizeInGB() { + return sizeInGB; + } + public void setSizeInGB(Integer sizeInGB) { + this.sizeInGB = sizeInGB; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VaultVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VaultVH.java new file mode 100644 index 000000000..ea9f3e987 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VaultVH.java @@ -0,0 +1,115 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.Map; + +public class VaultVH extends AzureVH { + private String id; + private String name; + private String type; + private String location; + private Map tags; + private Map sku; + private boolean enabledForDeployment; + private boolean enabledForDiskEncryption; + private boolean enabledForTemplateDeployment; + private String tenantId; + private String provisioningState; + private String vaultUri; + + public Map getSku() { + return sku; + } + + public boolean isEnabledForDeployment() { + return enabledForDeployment; + } + + public boolean isEnabledForDiskEncryption() { + return enabledForDiskEncryption; + } + + public boolean isEnabledForTemplateDeployment() { + return enabledForTemplateDeployment; + } + + public String getTenantId() { + return tenantId; + } + + public String getProvisioningState() { + return provisioningState; + } + + public String getVaultUri() { + return vaultUri; + } + + public void setSku(Map sku) { + this.sku = sku; + } + + public void setEnabledForDeployment(boolean enabledForDeployment) { + this.enabledForDeployment = enabledForDeployment; + } + + public void setEnabledForDiskEncryption(boolean enabledForDiskEncryption) { + this.enabledForDiskEncryption = enabledForDiskEncryption; + } + + public void setEnabledForTemplateDeployment(boolean enabledForTemplateDeployment) { + this.enabledForTemplateDeployment = enabledForTemplateDeployment; + } + + public void setTenantId(String tenantId) { + this.tenantId = tenantId; + } + + public void setProvisioningState(String provisioningState) { + this.provisioningState = provisioningState; + } + + public void setVaultUri(String vaultUri) { + this.vaultUri = vaultUri; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getLocation() { + return location; + } + + public Map getTags() { + return tags; + } + + public void setId(String id) { + this.id = id; + } + + public void setName(String name) { + this.name = name; + } + + public void setType(String type) { + this.type = type; + } + + public void setLocation(String location) { + this.location = location; + } + + public void setTags(Map tags) { + this.tags = tags; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VirtualMachineVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VirtualMachineVH.java new file mode 100644 index 000000000..f1168079b --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VirtualMachineVH.java @@ -0,0 +1,323 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.microsoft.azure.management.compute.StorageAccountTypes; +import com.microsoft.azure.management.resources.fluentcore.arm.AvailabilityZoneId; + +@JsonSerialize +public class VirtualMachineVH extends AzureVH { + + private String computerName; + private String vmSize; + private Map tags; + private List networkInterfaceIds; + private StorageAccountTypes osDiskStorageAccountType; + private Set availabilityZones; + + + private boolean isManagedDiskEnabled; + private String availabilitySetId; + private String provisioningState; + private String licenseType; + + private List disks; + + private String vmId; + private boolean isBootDiagnosticsEnabled; + private String bootDiagnosticsStorageUri; + private boolean isManagedServiceIdentityEnabled; + private String systemAssignedManagedServiceIdentityTenantId; + private String systemAssignedManagedServiceIdentityPrincipalId; + private Set userAssignedManagedServiceIdentityIds; + private String name; + + private String os; + private String osVersion; + + private String privateIpAddress; + private String publicIpAddress; + + private List> networkSecurityGroups; + + private String vnet; + private String subnet; + private String vnetName; + private String primaryNCIMacAddress; + private String osType; + + public String getOsType() { + return osType; + } + + public void setOsType(String osType) { + this.osType = osType; + } + + public String getPrimaryNCIMacAddress() { + return primaryNCIMacAddress; + } + + public void setPrimaryNCIMacAddress(String primaryNCIMacAddress) { + this.primaryNCIMacAddress = primaryNCIMacAddress; + } + + + public List> getSecondaryNetworks() { + return secondaryNetworks; + } + + public void setSecondaryNetworks(List> secondaryNetworks) { + this.secondaryNetworks = secondaryNetworks; + } + + private String primaryNetworkIntefaceId; + + List> secondaryNetworks; + + public String getVnet() { + return vnet; + } + + public void setVnet(String vnet) { + this.vnet = vnet; + } + + public String getSubnet() { + return subnet; + } + + public void setSubnet(String subnet) { + this.subnet = subnet; + } + + public String getVnetName() { + return vnetName; + } + + public void setVnetName(String vnetName) { + this.vnetName = vnetName; + } + + public String getPrimaryNetworkIntefaceId() { + return primaryNetworkIntefaceId; + } + + public void setPrimaryNetworkIntefaceId(String primaryNetworkIntefaceId) { + this.primaryNetworkIntefaceId = primaryNetworkIntefaceId; + } + + public List> getNetworkSecurityGroups() { + return networkSecurityGroups; + } + + public void setNetworkSecurityGroups(List> networkSecurityGroups) { + this.networkSecurityGroups = networkSecurityGroups; + } + + public String getPrivateIpAddress() { + return privateIpAddress; + } + + public void setPrivateIpAddress(String privateIpAddress) { + this.privateIpAddress = privateIpAddress; + } + + public String getPublicIpAddress() { + return publicIpAddress; + } + + public void setPublicIpAddress(String publicIpAddress) { + this.publicIpAddress = publicIpAddress; + } + + public String getOs() { + return os; + } + + public void setOs(String os) { + this.os = os; + } + + public String getOsVersion() { + return osVersion; + } + + public void setOsVersion(String osVersion) { + this.osVersion = osVersion; + } + + public String getStatus() { + return status; + } + + private String status; + + public List getDisks() { + return disks; + } + + public void setDisks(List disks) { + this.disks = disks; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getComputerName() { + return computerName; + } + + public void setComputerName(String computerName) { + this.computerName = computerName; + } + + public boolean isManagedDiskEnabled() { + return isManagedDiskEnabled; + } + + public void setManagedDiskEnabled(boolean isManagedDiskEnabled) { + this.isManagedDiskEnabled = isManagedDiskEnabled; + } + + + + public String getAvailabilitySetId() { + return availabilitySetId; + } + + public void setAvailabilitySetId(String availabilitySetId) { + this.availabilitySetId = availabilitySetId; + } + + public String getProvisioningState() { + return provisioningState; + } + + public void setProvisioningState(String provisioningState) { + this.provisioningState = provisioningState; + } + + public String getLicenseType() { + return licenseType; + } + + public void setLicenseType(String licenseType) { + this.licenseType = licenseType; + } + + public String getVmId() { + return vmId; + } + + public void setVmId(String vmId) { + this.vmId = vmId; + } + + public boolean isBootDiagnosticsEnabled() { + return isBootDiagnosticsEnabled; + } + + public void setBootDiagnosticsEnabled(boolean isBootDiagnosticsEnabled) { + this.isBootDiagnosticsEnabled = isBootDiagnosticsEnabled; + } + + public String getBootDiagnosticsStorageUri() { + return bootDiagnosticsStorageUri; + } + + public void setBootDiagnosticsStorageUri(String bootDiagnosticsStorageUri) { + this.bootDiagnosticsStorageUri = bootDiagnosticsStorageUri; + } + + public boolean isManagedServiceIdentityEnabled() { + return isManagedServiceIdentityEnabled; + } + + public void setManagedServiceIdentityEnabled(boolean isManagedServiceIdentityEnabled) { + this.isManagedServiceIdentityEnabled = isManagedServiceIdentityEnabled; + } + + public String getSystemAssignedManagedServiceIdentityTenantId() { + return systemAssignedManagedServiceIdentityTenantId; + } + + public void setSystemAssignedManagedServiceIdentityTenantId(String systemAssignedManagedServiceIdentityTenantId) { + this.systemAssignedManagedServiceIdentityTenantId = systemAssignedManagedServiceIdentityTenantId; + } + + public String getSystemAssignedManagedServiceIdentityPrincipalId() { + return systemAssignedManagedServiceIdentityPrincipalId; + } + + public void setSystemAssignedManagedServiceIdentityPrincipalId( + String systemAssignedManagedServiceIdentityPrincipalId) { + this.systemAssignedManagedServiceIdentityPrincipalId = systemAssignedManagedServiceIdentityPrincipalId; + } + + public Set getUserAssignedManagedServiceIdentityIds() { + return userAssignedManagedServiceIdentityIds; + } + + public void setUserAssignedManagedServiceIdentityIds(Set userAssignedManagedServiceIdentityIds) { + this.userAssignedManagedServiceIdentityIds = userAssignedManagedServiceIdentityIds; + } + + + + public String getVmSize() { + return vmSize; + } + + public void setVmSize(String vmSize) { + this.vmSize = vmSize; + } + + public Map getTags() { + return tags; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public List getNetworkInterfaceIds() { + return networkInterfaceIds; + } + + public void setNetworkInterfaceIds(List networkInterfaceIds) { + this.networkInterfaceIds = networkInterfaceIds; + } + + public StorageAccountTypes getOsDiskStorageAccountType() { + return osDiskStorageAccountType; + } + + public void setOsDiskStorageAccountType(StorageAccountTypes osDiskStorageAccountType) { + this.osDiskStorageAccountType = osDiskStorageAccountType; + } + + public Set getAvailabilityZones() { + return availabilityZones; + } + + public void setAvailabilityZones(Set availabilityZones) { + this.availabilityZones = availabilityZones; + } + + public void setStatus(String status) { + this.status = status; + + } + + + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VirtualNetworkRuleVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VirtualNetworkRuleVH.java new file mode 100644 index 000000000..32fdefaf9 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VirtualNetworkRuleVH.java @@ -0,0 +1,23 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +public class VirtualNetworkRuleVH { + private String id; + private boolean ignoreMissingVNetServiceEndpoint; + + public String getId() { + return id; + } + + public boolean isIgnoreMissingVNetServiceEndpoint() { + return ignoreMissingVNetServiceEndpoint; + } + + public void setId(String id) { + this.id = id; + } + + public void setIgnoreMissingVNetServiceEndpoint(boolean ignoreMissingVNetServiceEndpoint) { + this.ignoreMissingVNetServiceEndpoint = ignoreMissingVNetServiceEndpoint; + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/WorkflowVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/WorkflowVH.java new file mode 100644 index 000000000..0e3dd1b63 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/WorkflowVH.java @@ -0,0 +1,51 @@ +package com.tmobile.pacbot.azure.inventory.vo; + +import java.util.Map; + +public class WorkflowVH extends AzureVH{ + + private String id; + private String name; + private String type; + private String location; + private Map tags; + private Map properties; + public String getId() { + return id; + } + public String getName() { + return name; + } + public String getType() { + return type; + } + public String getLocation() { + return location; + } + public Map getTags() { + return tags; + } + public Map getProperties() { + return properties; + } + public void setId(String id) { + this.id = id; + } + public void setName(String name) { + this.name = name; + } + public void setType(String type) { + this.type = type; + } + public void setLocation(String location) { + this.location = location; + } + public void setTags(Map tags) { + this.tags = tags; + } + public void setProperties(Map properties) { + this.properties = properties; + } + + +} From 1a05b71365b0d19de3065870cfc9d1a095d9068b Mon Sep 17 00:00:00 2001 From: johnrexj Date: Fri, 18 Oct 2019 14:54:54 +0530 Subject: [PATCH 008/107] Azure discovery job pom --- jobs/azure-discovery/pom.xml | 93 ++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 jobs/azure-discovery/pom.xml diff --git a/jobs/azure-discovery/pom.xml b/jobs/azure-discovery/pom.xml new file mode 100644 index 000000000..5d9d25ead --- /dev/null +++ b/jobs/azure-discovery/pom.xml @@ -0,0 +1,93 @@ + + + 4.0.0 + com.tmobile.pacbot + azure-discovery + 0.0.1-SNAPSHOT + azure-discovery + + + 1.8 + + + + + org.springframework + spring-context + 4.3.8.RELEASE + + + com.tmobile.cloud + batch-commons + 1.0.0-SNAPSHOT + provided + + + commons-httpclient + commons-httpclient + + + + + com.fasterxml.jackson.core + jackson-databind + 2.9.4 + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.6.1 + + 1.8 + 1.8 + + + + maven-assembly-plugin + + + build-a + + + jar-with-dependencies + + pacbot-azure-discovery + + package + + single + + + + + + org.jacoco + jacoco-maven-plugin + 0.7.6.201602180812 + + + + prepare-agent + + + + report + test + + report + + + + + + + + + From d1078359c9598789b85cd796891a65a0f8d1c440 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Fri, 18 Oct 2019 16:36:46 +0530 Subject: [PATCH 009/107] New upgrae method added for terraform12 upgrade command --- installer/core/terraform/__init__.py | 14 +++++++++++++- installer/custom/commands/reinstall.py | 6 ++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/installer/core/terraform/__init__.py b/installer/core/terraform/__init__.py index 2f51ad382..c4194c476 100644 --- a/installer/core/terraform/__init__.py +++ b/installer/core/terraform/__init__.py @@ -325,7 +325,7 @@ def write_current_status(self, command, status_code, description=""): json.dump(current_status, jsonfile, indent=4) @classmethod - def get_current_status(self): + def get_current_status(cls): """ Write current status for the executed comamnd to status file @@ -339,3 +339,15 @@ def get_current_status(self): status_dict = json.load(jsonfile) return status_dict + + @classmethod + def terrafomr12_upgrade(cls): + """ + Write current status for the executed comamnd to status file + + Returns: + status_dict (dict): Status dict to be written + """ + response = terraform.cmd("0.12upgradde", yes=True) + + return response diff --git a/installer/custom/commands/reinstall.py b/installer/custom/commands/reinstall.py index 3ae853603..7a3e32ae5 100644 --- a/installer/custom/commands/reinstall.py +++ b/installer/custom/commands/reinstall.py @@ -1,5 +1,6 @@ from core.commands import BaseCommand from core.config import Settings +from core.terraform import PyTerraform from core import constants as K from threading import Thread import time @@ -20,7 +21,7 @@ class Reinstall(BaseCommand): def __init__(self, args): Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True) - + args.append((K.CATEGORY_FIELD_NAME, "deploy")) args.append((K.CATEGORY_FIELD_NAME, "upload_tf")) @@ -89,7 +90,7 @@ def re_deploy_pacbot(self, input_instance): """ resources_to_destroy = self.get_resources_to_process(self.destroy_resource_tags_list, input_instance) resources_to_install = self.get_resources_to_process(self.reinstall_resource_tags_list, input_instance) - + try: resources_to_taint = self.get_resources_with_given_tags(input_instance, ["deploy"]) resources_to_taint = [resource for resource in resources_to_taint if resource.PROCESS is True] @@ -145,6 +146,7 @@ def run_tf_apply(self, input_instance, resources_to_destroy, resources_to_instal resources_to_process (list): List of resources to be created/updated terraform_with_targets (boolean): This is True since redeployment is happening """ + PyTerraform.terrafomr12_upgrade() self.install_class( [], input_instance, From 356d55afe4984a07b0346729d3927f197e44a698 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Mon, 21 Oct 2019 13:35:47 +0530 Subject: [PATCH 010/107] terraform 12 upgrade is run genereally --- installer/custom/commands/reinstall.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/custom/commands/reinstall.py b/installer/custom/commands/reinstall.py index 7a3e32ae5..65461bd83 100644 --- a/installer/custom/commands/reinstall.py +++ b/installer/custom/commands/reinstall.py @@ -146,7 +146,7 @@ def run_tf_apply(self, input_instance, resources_to_destroy, resources_to_instal resources_to_process (list): List of resources to be created/updated terraform_with_targets (boolean): This is True since redeployment is happening """ - PyTerraform.terrafomr12_upgrade() + PyTerraform.terrafomr12_upgrade() # This is required only when terraform version 12 is used self.install_class( [], input_instance, From 514191f2585b76475c0c8d6adb8e53cce7c8d8b9 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Mon, 21 Oct 2019 13:46:53 +0530 Subject: [PATCH 011/107] self.args has been removed and added them as part of resources to process --- installer/core/commands/destroy.py | 2 +- installer/core/commands/install.py | 6 +----- installer/core/providers/aws/destroy.py | 3 +-- installer/core/providers/aws/install.py | 3 +-- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/installer/core/commands/destroy.py b/installer/core/commands/destroy.py index aa7389f67..a177bac8d 100644 --- a/installer/core/commands/destroy.py +++ b/installer/core/commands/destroy.py @@ -38,7 +38,7 @@ def execute(self, provider): resources_to_process = self.get_resources_to_process(self.resource_tags_list, input_instance) if resources_to_process: - self.destroy_class(self.args, input_instance).execute( + self.destroy_class(input_instance).execute( resources_to_process, self.terraform_with_targets, self.dry_run diff --git a/installer/core/commands/install.py b/installer/core/commands/install.py index 077f5d7ca..431b29b61 100644 --- a/installer/core/commands/install.py +++ b/installer/core/commands/install.py @@ -9,13 +9,10 @@ class Install(BaseCommand): Base install class which identify actual provide install class and execute installation Attributes: - terraform_with_targets (Boolean): Identify whether complete installation or partial installation is required validation_class (class): Provider validation class for validating inputs (aws validator) input_class (class): Provider input class install_class (class): Provider install class """ - terraform_with_targets = False - def __init__(self, args): """ Constructor method for install @@ -23,7 +20,6 @@ def __init__(self, args): Args: args (List): List of key- value pair of args supplied to the command """ - self.terraform_with_targets = False super().__init__(args) def execute(self, provider): @@ -42,7 +38,7 @@ def execute(self, provider): resources_to_process = self.get_resources_to_process(self.resource_tags_list, input_instance) if resources_to_process: - self.install_class(self.args, input_instance).execute( + self.install_class(input_instance).execute( resources_to_process, self.terraform_with_targets, self.dry_run diff --git a/installer/core/providers/aws/destroy.py b/installer/core/providers/aws/destroy.py index 629c918a6..5f8669baa 100644 --- a/installer/core/providers/aws/destroy.py +++ b/installer/core/providers/aws/destroy.py @@ -29,8 +29,7 @@ class Destroy(BaseAction): exception = None terraform_thread = None - def __init__(self, args, input_obj): - self.args = args + def __init__(self, input_obj): super().__init__(input_obj) def execute(self, resources, terraform_with_targets, dry_run): diff --git a/installer/core/providers/aws/install.py b/installer/core/providers/aws/install.py index 91c29179f..8e01f031f 100644 --- a/installer/core/providers/aws/install.py +++ b/installer/core/providers/aws/install.py @@ -42,8 +42,7 @@ class Install(BaseAction): terraform_outputs = {} terraform_thread = None - def __init__(self, args, input_obj, check_dependent_resources=True): - self.args = args + def __init__(self, input_obj, check_dependent_resources=True): self.check_dependent_resources = check_dependent_resources super().__init__(input_obj) logging.disable(logging.ERROR) # To disable python terraform unwanted warnings From b21b64ca65ac2fed98f578c9a20f34dff67e1d5a Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Mon, 21 Oct 2019 14:24:16 +0530 Subject: [PATCH 012/107] initialised terraform object for tainting resources --- installer/core/terraform/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/installer/core/terraform/__init__.py b/installer/core/terraform/__init__.py index c4194c476..94dc84fbb 100644 --- a/installer/core/terraform/__init__.py +++ b/installer/core/terraform/__init__.py @@ -348,6 +348,9 @@ def terrafomr12_upgrade(cls): Returns: status_dict (dict): Status dict to be written """ + terraform = Terraform( + working_dir=Settings.TERRAFORM_DIR, + ) response = terraform.cmd("0.12upgradde", yes=True) return response From 22da98d8e2e4ec894c36bd6241c3394a67616c88 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Mon, 21 Oct 2019 14:27:59 +0530 Subject: [PATCH 013/107] empty list suplied as args to install constructor has been removed --- installer/custom/commands/reinstall.py | 1 - 1 file changed, 1 deletion(-) diff --git a/installer/custom/commands/reinstall.py b/installer/custom/commands/reinstall.py index 65461bd83..e500155d5 100644 --- a/installer/custom/commands/reinstall.py +++ b/installer/custom/commands/reinstall.py @@ -148,7 +148,6 @@ def run_tf_apply(self, input_instance, resources_to_destroy, resources_to_instal """ PyTerraform.terrafomr12_upgrade() # This is required only when terraform version 12 is used self.install_class( - [], input_instance, check_dependent_resources=False ).execute( From c9f0048cb0eb5dbb0674463fd63b75d24512cf46 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Mon, 21 Oct 2019 14:56:36 +0530 Subject: [PATCH 014/107] variable name corrected --- installer/core/providers/aws/reinstall.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/installer/core/providers/aws/reinstall.py b/installer/core/providers/aws/reinstall.py index 13801557a..04810852d 100644 --- a/installer/core/providers/aws/reinstall.py +++ b/installer/core/providers/aws/reinstall.py @@ -30,12 +30,12 @@ def execute(self, resources_to_destroy, resources_to_install, terraform_with_tar resources (list): Resources to be installed terraform_with_targets (boolean): If partial install is to be done (if --tags is supplied) dry_run (boolean): Decides whether original install should be done - """ + """ self.generate_terraform_files(resources_to_install, terraform_with_targets) self.run_tf_execution_and_status_threads(resources_to_destroy, resources_to_install, terraform_with_targets, dry_run) if not self.executed_with_error: - self.render_resource_outputs(resources) + self.render_resource_outputs(resources_to_install) else: raise self.exception From d1bb5736f355d9bb3da56cca83a10403c4c7c372 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Mon, 21 Oct 2019 15:25:20 +0530 Subject: [PATCH 015/107] Taint resources commented out and upgrade command removed self.args --- installer/custom/commands/reinstall.py | 12 ++++++------ installer/custom/commands/upgrade.py | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/installer/custom/commands/reinstall.py b/installer/custom/commands/reinstall.py index e500155d5..49b830157 100644 --- a/installer/custom/commands/reinstall.py +++ b/installer/custom/commands/reinstall.py @@ -91,12 +91,12 @@ def re_deploy_pacbot(self, input_instance): resources_to_destroy = self.get_resources_to_process(self.destroy_resource_tags_list, input_instance) resources_to_install = self.get_resources_to_process(self.reinstall_resource_tags_list, input_instance) - try: - resources_to_taint = self.get_resources_with_given_tags(input_instance, ["deploy"]) - resources_to_taint = [resource for resource in resources_to_taint if resource.PROCESS is True] - response = PyTerraform().terraform_taint(resources_to_taint) # If tainted or destroyed already then skip it - except Exception as e: - pass + # try: + # resources_to_taint = self.get_resources_with_given_tags(input_instance, ["deploy"]) + # resources_to_taint = [resource for resource in resources_to_taint if resource.PROCESS is True] + # response = PyTerraform().terraform_taint(resources_to_taint) # If tainted or destroyed already then skip it + # except Exception as e: + # pass terraform_with_targets = False if self.need_complete_install else True resources_to_install = self.get_complete_resources(input_instance) if self.need_complete_install else resources_to_install diff --git a/installer/custom/commands/upgrade.py b/installer/custom/commands/upgrade.py index fa6743122..4248f772d 100644 --- a/installer/custom/commands/upgrade.py +++ b/installer/custom/commands/upgrade.py @@ -83,7 +83,6 @@ def upgrade_pacbot(self, input_instance): self.run_pre_deployment_process(resources_to_process) self.install_class( - self.args, input_instance, check_dependent_resources=False ).execute( From 0351889dcd64502672c130c16db519e6bea4c5e6 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Mon, 21 Oct 2019 17:12:29 +0530 Subject: [PATCH 016/107] Terraform files extension is changed from .tf to .tf.sjon to be compatible with all tf versions in apply method --- installer/core/terraform/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/installer/core/terraform/__init__.py b/installer/core/terraform/__init__.py index 94dc84fbb..9e520bd48 100644 --- a/installer/core/terraform/__init__.py +++ b/installer/core/terraform/__init__.py @@ -86,6 +86,8 @@ def terraform_apply(self, resources=None): if exists_teraform_lock(): raise Exception(K.ANOTHER_PROCESS_RUNNING) + self.change_tf_extension_to_tf_json() + CMD = Settings.get('running_command', "Terraform Apply") terraform = Terraform( working_dir=Settings.TERRAFORM_DIR, @@ -354,3 +356,10 @@ def terrafomr12_upgrade(cls): response = terraform.cmd("0.12upgradde", yes=True) return response + + @classmethod + def change_tf_extension_to_tf_json(cls): + working_dir = Settings.TERRAFORM_DIR + + for file in [f for f in os.listdir(working_dir) if item.endswith(".tf")]: + os.renmae(f, "%s.json" % f) From daff33e2acd27c5949e2feafd1568e56bff93810 Mon Sep 17 00:00:00 2001 From: johnrexj Date: Mon, 21 Oct 2019 19:24:11 +0530 Subject: [PATCH 017/107] data shipper changes for azure --- .../tmobile/cso/pacman/datashipper/es/ESManager.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/es/ESManager.java b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/es/ESManager.java index 105fbaf78..9cb6155b1 100644 --- a/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/es/ESManager.java +++ b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/es/ESManager.java @@ -399,7 +399,7 @@ public static void configureIndexAndTypes(String ds, List> e Iterator it = types.iterator(); while (it.hasNext()) { String _type = it.next(); - String indexName = ds + "_" + _type; + String indexName = ds+ "_" + _type; if (!indexExists(indexName)) { StringBuilder payLoad = new StringBuilder(_payLoad); payLoad.append("\"" + _type + "\":{},\"issue_" + _type + "\": { \"_parent\": {\"type\": \"" + _type @@ -410,7 +410,6 @@ public static void configureIndexAndTypes(String ds, List> e payLoad.append("}}"); try { invokeAPI("PUT", indexName, payLoad.toString()); - invokeAPI("PUT", "/" + indexName + "/_alias/" + ds, null); } catch (IOException e) { LOGGER.error("Error in configureIndexAndTypes",e); Map errorMap = new HashMap<>(); @@ -420,9 +419,14 @@ public static void configureIndexAndTypes(String ds, List> e errorList.add(errorMap); } } + try { + invokeAPI("PUT", "/" + indexName + "/_alias/" + ds, null); + invokeAPI("PUT", "/" + indexName + "/_alias/" + "ds-all", null); + } catch (IOException e) { + + } } - - } + } /** * Gets the existing info. From d19e04ed987fcc5159be4fb9269c7da4b65aabda Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Fri, 25 Oct 2019 14:19:42 +0530 Subject: [PATCH 018/107] Terraform extension change is applied before install/destroy when move from old terraform to new terraform version --- installer/core/log.py | 10 ++++++---- installer/core/terraform/__init__.py | 2 -- installer/resources/pacbot_app/build_ui_and_api.py | 10 ++++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/installer/core/log.py b/installer/core/log.py index 44f00876b..27b355dc0 100644 --- a/installer/core/log.py +++ b/installer/core/log.py @@ -99,7 +99,7 @@ def write_terraform_plan_log(self, response): self.write_debug_log(K.TERRAFORM_PLAN_COMPLETED) - def write_terraform_apply_log_header(self): + def write_terraform_apply_log_header(self, header=None): """ Write terraform apply command response to install log @@ -111,9 +111,10 @@ def write_terraform_apply_log_header(self): logfile.write("\n*** Terraform Apply Started") logfile.write("\nDateTime: %s\n" % datetime.now().strftime('%Y-%m-%d %H:%M:%S')) logfile.write("*" * 100) - self.write_debug_log(K.TERRAFORM_APPLY_STARTED) + header = header if header else K.TERRAFORM_APPLY_STARTED + self.write_debug_log(header) - def write_terraform_destroy_log_header(self): + def write_terraform_destroy_log_header(self, header=None): """ Write terraform destroy command response to destroy log @@ -125,7 +126,8 @@ def write_terraform_destroy_log_header(self): logfile.write("\n*** Terraform Destroy Started ***") logfile.write("\nDateTime: %s\n" % datetime.now().strftime('%Y-%m-%d %H:%M:%S')) logfile.write("*" * 100) - self.write_debug_log(K.TERRAFORM_DESTROY_STARTED) + header = header if header else K.TERRAFORM_DESTROY_STARTED + self.write_debug_log(header) def _write_header(self, head_msg=None): """ diff --git a/installer/core/terraform/__init__.py b/installer/core/terraform/__init__.py index 9e520bd48..d087bbae7 100644 --- a/installer/core/terraform/__init__.py +++ b/installer/core/terraform/__init__.py @@ -86,8 +86,6 @@ def terraform_apply(self, resources=None): if exists_teraform_lock(): raise Exception(K.ANOTHER_PROCESS_RUNNING) - self.change_tf_extension_to_tf_json() - CMD = Settings.get('running_command', "Terraform Apply") terraform = Terraform( working_dir=Settings.TERRAFORM_DIR, diff --git a/installer/resources/pacbot_app/build_ui_and_api.py b/installer/resources/pacbot_app/build_ui_and_api.py index a9c28907f..2c4bf7b1f 100644 --- a/installer/resources/pacbot_app/build_ui_and_api.py +++ b/installer/resources/pacbot_app/build_ui_and_api.py @@ -2,6 +2,7 @@ from resources.s3.bucket import BucketStorage from resources.pacbot_app.alb import ApplicationLoadBalancer from core.terraform.utils import get_terraform_scripts_dir, get_terraform_provider_file +from core.terraform import PyTerraform from core.config import Settings import os @@ -46,3 +47,12 @@ def _create_dir_to_store_build_ap(self): raise Exception("Not able to create directory to store API Jars and UI code") return upload_dir + + + def pre_terraform_destroy(self): + # To support latest terraform version + PyTerraform.change_tf_extension_to_tf_json() + + def pre_generate_terraform(self): + # To support latest terraform version + PyTerraform.change_tf_extension_to_tf_json() From 1f22367179f85466acf57f0184978808858645d6 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Fri, 25 Oct 2019 14:23:42 +0530 Subject: [PATCH 019/107] Destroy heading changed during redeploy --- installer/core/constants.py | 1 + installer/core/providers/aws/reinstall.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/installer/core/constants.py b/installer/core/constants.py index 25e17e2bb..642a4f22f 100644 --- a/installer/core/constants.py +++ b/installer/core/constants.py @@ -88,6 +88,7 @@ TERRAFORM_OUTPUT_STORED = "Terraform output is stored" TERRAFORM_DESTROY_STARTED = "Terraform destroy started" +TERRAFORM_REDEPLOY_DESTROY_STARTED = "Terraform destroy started as part of Redeploy" TERRAFORM_DESTROY_RUNNING = "Destroying resources" TERRAFORM_DESTROY_ERROR = "Terraform destroy encountered an error" TERRAFORM_DESTROY_COMPLETED = "Terraform destroy executed successfully!!! Please check destroy log for more details" diff --git a/installer/core/providers/aws/reinstall.py b/installer/core/providers/aws/reinstall.py index 04810852d..4c060877d 100644 --- a/installer/core/providers/aws/reinstall.py +++ b/installer/core/providers/aws/reinstall.py @@ -99,6 +99,6 @@ def show_progress_status_all(self, resources, terraform_with_targets, dry_run): def render_terraform_destroy_progress(self): """Show the status of terraform init command execution""" start_time = datetime.now() - self.show_step_heading(K.TERRAFORM_DESTROY_STARTED, write_log=False) + self.show_step_heading(K.TERRAFORM_REDEPLOY_DESTROY_STARTED, write_log=False) while self.destroy is False and self.terraform_thread.isAlive(): self.show_progress_message(K.TERRAFORM_DESTROY_STARTED, 0.5) From 6b98caddb0499004dfdb02b03ec257e8c7b5e1b0 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Fri, 25 Oct 2019 14:31:10 +0530 Subject: [PATCH 020/107] Pep8 standard modifications --- installer/core/providers/aws/reinstall.py | 1 - installer/core/terraform/utils.py | 1 - installer/custom/commands/reinstall.py | 1 - installer/resources/pacbot_app/build_ui_and_api.py | 1 - installer/settings/default.local.py | 2 +- 5 files changed, 1 insertion(+), 5 deletions(-) diff --git a/installer/core/providers/aws/reinstall.py b/installer/core/providers/aws/reinstall.py index 4c060877d..9c4e11f61 100644 --- a/installer/core/providers/aws/reinstall.py +++ b/installer/core/providers/aws/reinstall.py @@ -83,7 +83,6 @@ def re_create_resources(self, resources_to_destroy, resources_to_install, terraf self._cleanup_installation_process(dry_run) - def show_progress_status_all(self, resources, terraform_with_targets, dry_run): """ Show the status of installation continously in this thread diff --git a/installer/core/terraform/utils.py b/installer/core/terraform/utils.py index be6f1b817..0320603e1 100644 --- a/installer/core/terraform/utils.py +++ b/installer/core/terraform/utils.py @@ -163,7 +163,6 @@ def get_resource_created_status_op_file(resource_id): return _get_resource_status_file_name(resource_id, '1') - def get_type_corrected_tags(tags): """ Get tags type corrected since earlier version used list and now changed to dict for terraform compatibility diff --git a/installer/custom/commands/reinstall.py b/installer/custom/commands/reinstall.py index 49b830157..fe35eb3ef 100644 --- a/installer/custom/commands/reinstall.py +++ b/installer/custom/commands/reinstall.py @@ -22,7 +22,6 @@ def __init__(self, args): Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True) - args.append((K.CATEGORY_FIELD_NAME, "deploy")) args.append((K.CATEGORY_FIELD_NAME, "upload_tf")) self.destroy_resource_tags_list = [v for (k, v) in args if k == self.category_field_name] diff --git a/installer/resources/pacbot_app/build_ui_and_api.py b/installer/resources/pacbot_app/build_ui_and_api.py index 2c4bf7b1f..196ce0575 100644 --- a/installer/resources/pacbot_app/build_ui_and_api.py +++ b/installer/resources/pacbot_app/build_ui_and_api.py @@ -48,7 +48,6 @@ def _create_dir_to_store_build_ap(self): return upload_dir - def pre_terraform_destroy(self): # To support latest terraform version PyTerraform.change_tf_extension_to_tf_json() diff --git a/installer/settings/default.local.py b/installer/settings/default.local.py index a742c27bc..7af122139 100644 --- a/installer/settings/default.local.py +++ b/installer/settings/default.local.py @@ -51,4 +51,4 @@ # This settings enable Vulnerability feature and servie ENABLE_VULNERABILITY_FEATURE = False QUALYS_API_URL = "" # Qualys API Url without trailing slash -QUALYS_INFO = "" #Base64 encoded user:password of qualys +QUALYS_INFO = "" # Base64 encoded user:password of qualys From 83f8e0b8f11f09f7b3a34e9257b944ee663ec8dd Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Fri, 25 Oct 2019 15:49:24 +0530 Subject: [PATCH 021/107] Typo corrected --- installer/core/terraform/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/core/terraform/__init__.py b/installer/core/terraform/__init__.py index d087bbae7..1b5edc61f 100644 --- a/installer/core/terraform/__init__.py +++ b/installer/core/terraform/__init__.py @@ -359,5 +359,5 @@ def terrafomr12_upgrade(cls): def change_tf_extension_to_tf_json(cls): working_dir = Settings.TERRAFORM_DIR - for file in [f for f in os.listdir(working_dir) if item.endswith(".tf")]: + for file in [f for f in os.listdir(working_dir) if f.endswith(".tf")]: os.renmae(f, "%s.json" % f) From 2bb5ce291c9059e59d2e315039354ad4258a6350 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Fri, 25 Oct 2019 15:50:38 +0530 Subject: [PATCH 022/107] Type corrected --- installer/core/terraform/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/core/terraform/__init__.py b/installer/core/terraform/__init__.py index 1b5edc61f..e96726c13 100644 --- a/installer/core/terraform/__init__.py +++ b/installer/core/terraform/__init__.py @@ -360,4 +360,4 @@ def change_tf_extension_to_tf_json(cls): working_dir = Settings.TERRAFORM_DIR for file in [f for f in os.listdir(working_dir) if f.endswith(".tf")]: - os.renmae(f, "%s.json" % f) + os.rename(f, "%s.json" % f) From df8393f43379d3d21c927a58577f4c7352ed7648 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Fri, 25 Oct 2019 15:51:49 +0530 Subject: [PATCH 023/107] Pep8 standard modifications --- installer/core/terraform/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/installer/core/terraform/__init__.py b/installer/core/terraform/__init__.py index e96726c13..a9a464361 100644 --- a/installer/core/terraform/__init__.py +++ b/installer/core/terraform/__init__.py @@ -360,4 +360,5 @@ def change_tf_extension_to_tf_json(cls): working_dir = Settings.TERRAFORM_DIR for file in [f for f in os.listdir(working_dir) if f.endswith(".tf")]: - os.rename(f, "%s.json" % f) + file_path = os.path.join(working_dir, file) + os.rename(file_path, "%s.json" % file_path) From 12164712eaa5bd75a4690948aca53edbfcd368f1 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Fri, 25 Oct 2019 16:17:52 +0530 Subject: [PATCH 024/107] Security group is added as tag for redeploy to generate terraform files --- installer/core/providers/aws/reinstall.py | 1 + installer/custom/commands/reinstall.py | 1 + 2 files changed, 2 insertions(+) diff --git a/installer/core/providers/aws/reinstall.py b/installer/core/providers/aws/reinstall.py index 9c4e11f61..4a1366528 100644 --- a/installer/core/providers/aws/reinstall.py +++ b/installer/core/providers/aws/reinstall.py @@ -101,3 +101,4 @@ def render_terraform_destroy_progress(self): self.show_step_heading(K.TERRAFORM_REDEPLOY_DESTROY_STARTED, write_log=False) while self.destroy is False and self.terraform_thread.isAlive(): self.show_progress_message(K.TERRAFORM_DESTROY_STARTED, 0.5) + print("\n") diff --git a/installer/custom/commands/reinstall.py b/installer/custom/commands/reinstall.py index fe35eb3ef..986a1a1de 100644 --- a/installer/custom/commands/reinstall.py +++ b/installer/custom/commands/reinstall.py @@ -34,6 +34,7 @@ def __init__(self, args): args.append((K.CATEGORY_FIELD_NAME, "submit-job")) args.append((K.CATEGORY_FIELD_NAME, "rule-engine-job")) args.append((K.CATEGORY_FIELD_NAME, "upload_tf")) + args.append((K.CATEGORY_FIELD_NAME, "security")) self.reinstall_resource_tags_list = [v for (k, v) in args if k == self.category_field_name] self.need_complete_install = self._need_complete_installation() From 84699c1606c7e12d744581b646fc8f2693d6ea5d Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Fri, 25 Oct 2019 16:26:14 +0530 Subject: [PATCH 025/107] All tf files are generated before redeploy --- installer/custom/commands/reinstall.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/installer/custom/commands/reinstall.py b/installer/custom/commands/reinstall.py index 986a1a1de..dabb137aa 100644 --- a/installer/custom/commands/reinstall.py +++ b/installer/custom/commands/reinstall.py @@ -34,7 +34,6 @@ def __init__(self, args): args.append((K.CATEGORY_FIELD_NAME, "submit-job")) args.append((K.CATEGORY_FIELD_NAME, "rule-engine-job")) args.append((K.CATEGORY_FIELD_NAME, "upload_tf")) - args.append((K.CATEGORY_FIELD_NAME, "security")) self.reinstall_resource_tags_list = [v for (k, v) in args if k == self.category_field_name] self.need_complete_install = self._need_complete_installation() @@ -146,11 +145,16 @@ def run_tf_apply(self, input_instance, resources_to_destroy, resources_to_instal resources_to_process (list): List of resources to be created/updated terraform_with_targets (boolean): This is True since redeployment is happening """ - PyTerraform.terrafomr12_upgrade() # This is required only when terraform version 12 is used - self.install_class( + installer = self.install_class( input_instance, check_dependent_resources=False - ).execute( + ) + + all_resources = self.get_complete_resources(input_instance) + installer.generate_terraform_files(all_resources, False) # To make all tf file in compatible with all terraform versions + + PyTerraform.terrafomr12_upgrade() # This is required only when terraform version 12 is used + installer.execute( resources_to_destroy, resources_to_install, terraform_with_targets, From 1a22209469b3552ee0f15463b3a0622e12a3de4e Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Fri, 25 Oct 2019 16:32:00 +0530 Subject: [PATCH 026/107] Terraform file generation before redeploy is done using custom method --- installer/core/providers/aws/reinstall.py | 1 - installer/custom/commands/reinstall.py | 12 ++++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/installer/core/providers/aws/reinstall.py b/installer/core/providers/aws/reinstall.py index 4a1366528..9c4e11f61 100644 --- a/installer/core/providers/aws/reinstall.py +++ b/installer/core/providers/aws/reinstall.py @@ -101,4 +101,3 @@ def render_terraform_destroy_progress(self): self.show_step_heading(K.TERRAFORM_REDEPLOY_DESTROY_STARTED, write_log=False) while self.destroy is False and self.terraform_thread.isAlive(): self.show_progress_message(K.TERRAFORM_DESTROY_STARTED, 0.5) - print("\n") diff --git a/installer/custom/commands/reinstall.py b/installer/custom/commands/reinstall.py index dabb137aa..d0274ccaf 100644 --- a/installer/custom/commands/reinstall.py +++ b/installer/custom/commands/reinstall.py @@ -136,6 +136,12 @@ def inactivate_required_services_for_redeploy(self, resources_to_destroy, resour """ pass + def generate_terraform_files_and_upgrade_state(self): + all_resources = self.get_complete_resources(input_instance) + for resource in all_resources: + resource.generate_terraform() + PyTerraform.terrafomr12_upgrade() # This is required only when terraform version 12 is used + def run_tf_apply(self, input_instance, resources_to_destroy, resources_to_install, terraform_with_targets): """ Execute the installation of resources by invoking the execute method of provider class @@ -145,15 +151,13 @@ def run_tf_apply(self, input_instance, resources_to_destroy, resources_to_instal resources_to_process (list): List of resources to be created/updated terraform_with_targets (boolean): This is True since redeployment is happening """ + self.generate_terraform_files_and_upgrade_state() + installer = self.install_class( input_instance, check_dependent_resources=False ) - all_resources = self.get_complete_resources(input_instance) - installer.generate_terraform_files(all_resources, False) # To make all tf file in compatible with all terraform versions - - PyTerraform.terrafomr12_upgrade() # This is required only when terraform version 12 is used installer.execute( resources_to_destroy, resources_to_install, From a5ed731ce3a6610d09c95b043b0caaea009dcb6f Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Fri, 25 Oct 2019 17:49:23 +0530 Subject: [PATCH 027/107] input parameter is passed to state update method --- installer/custom/commands/reinstall.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/installer/custom/commands/reinstall.py b/installer/custom/commands/reinstall.py index d0274ccaf..4a4418212 100644 --- a/installer/custom/commands/reinstall.py +++ b/installer/custom/commands/reinstall.py @@ -136,7 +136,7 @@ def inactivate_required_services_for_redeploy(self, resources_to_destroy, resour """ pass - def generate_terraform_files_and_upgrade_state(self): + def generate_terraform_files_and_upgrade_state(self, input_instance): all_resources = self.get_complete_resources(input_instance) for resource in all_resources: resource.generate_terraform() @@ -151,7 +151,7 @@ def run_tf_apply(self, input_instance, resources_to_destroy, resources_to_instal resources_to_process (list): List of resources to be created/updated terraform_with_targets (boolean): This is True since redeployment is happening """ - self.generate_terraform_files_and_upgrade_state() + self.generate_terraform_files_and_upgrade_state(input_instance) installer = self.install_class( input_instance, From 75bd8fc7cad7ca6e30e404d9df963387627523fd Mon Sep 17 00:00:00 2001 From: Kanchana Date: Fri, 25 Oct 2019 18:42:51 +0530 Subject: [PATCH 028/107] 3 azure tagging rules ported --- .../files/rule_engine_cloudwatch_rules.json | 66 +++++++++++++++++++ installer/resources/pacbot_app/files/DB.sql | 5 +- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json b/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json index 557210723..2366ad25f 100644 --- a/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json +++ b/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json @@ -2462,6 +2462,72 @@ "modifiedDate": "2019-09-18", "severity": "high", "category": "security" + }, + { + "ruleId": "PacMan_TaggingRule_version-1_VirtualmachineTaggingRule_virtualmachine", + "ruleUUID": "azure_virtualmachine_should_be_tagged_with_mandatory_tags", + "policyId": "PacMan_TaggingRule_version-1", + "ruleName": "VirtualmachineTaggingRule", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "VirtualmachineTaggingRule", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-missing-mandatory-tags\",\"key\":\"ruleKey\"},{\"encrypt\":false,\"value\":\",\",\"key\":\"splitterChar\"},{\"encrypt\":false,\"value\":\"Application,Environment,Stack,Role\",\"key\":\"mandatoryTags\"},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"tagging\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_TaggingRule_version-1_VirtualmachineTaggingRule_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"VirtualmachineTaggingRule\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_TaggingRule_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_virtualmachine_should_be_tagged_with_mandatory_tags\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_virtualmachine_should_be_tagged_with_mandatory_tags", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Virtualmachine should be tagged with mandatory tags", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "tagging" + }, + { + "ruleId": "PacMan_TaggingRule_version-1_SqlserverTaggingRule_sqlserver", + "ruleUUID": "azure_sqlserver_should_be_tagged_with_mandatory_tags", + "policyId": "PacMan_TaggingRule_version-1", + "ruleName": "SqlserverTaggingRule", + "targetType": "sqlserver", + "assetGroup": "azure", + "alexaKeyword": "SqlserverTaggingRule", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-missing-mandatory-tags\",\"key\":\"ruleKey\"},{\"encrypt\":false,\"value\":\",\",\"key\":\"splitterChar\"},{\"encrypt\":false,\"value\":\"Application,Environment,Stack,Role\",\"key\":\"mandatoryTags\"},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"tagging\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_TaggingRule_version-1_SqlserverTaggingRule_sqlserver\",\"autofix\":false,\"alexaKeyword\":\"SqlserverTaggingRule\",\"ruleRestUrl\":\"\",\"targetType\":\"sqlserver\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_TaggingRule_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_sqlserver_should_be_tagged_with_mandatory_tags\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_sqlserver_should_be_tagged_with_mandatory_tags", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Sqlserver should be tagged with mandatory tags", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "tagging" + }, + { + "ruleId": "PacMan_TaggingRule_version-1_SqldatabaseTaggingRule_sqldatabase", + "ruleUUID": "azure_sqldatabase_should_be_tagged_with_mandatory_tags", + "policyId": "PacMan_TaggingRule_version-1", + "ruleName": "SqldatabaseserverTaggingRule", + "targetType": "sqldatabase", + "assetGroup": "azure", + "alexaKeyword": "SqldatabaseserverTaggingRule", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-missing-mandatory-tags\",\"key\":\"ruleKey\"},{\"encrypt\":false,\"value\":\",\",\"key\":\"splitterChar\"},{\"encrypt\":false,\"value\":\"Application,Environment,Stack,Role\",\"key\":\"mandatoryTags\"},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"tagging\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_TaggingRule_version-1_SqldatabaseTaggingRule_sqldatabase\",\"autofix\":false,\"alexaKeyword\":\"SqldatabaseTaggingRule\",\"ruleRestUrl\":\"\",\"targetType\":\"sqldatabase\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_TaggingRule_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_sqldatabase_should_be_tagged_with_mandatory_tags\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_sqldatabase_should_be_tagged_with_mandatory_tags", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Sqldatabase should be tagged with mandatory tags", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "tagging" } ] diff --git a/installer/resources/pacbot_app/files/DB.sql b/installer/resources/pacbot_app/files/DB.sql index 4916b0d17..4da0ebc0d 100644 --- a/installer/resources/pacbot_app/files/DB.sql +++ b/installer/resources/pacbot_app/files/DB.sql @@ -1345,7 +1345,9 @@ INSERT IGNORE INTO cf_RuleInstance (`ruleId`,`ruleUUID`,`policyId`,`ruleName`,`t INSERT IGNORE INTO cf_RuleInstance (`ruleId`,`ruleUUID`,`policyId`,`ruleName`,`targetType`,`assetGroup`,`alexaKeyword`,`ruleParams`,`ruleFrequency`,`ruleExecutable`,`ruleRestUrl`,`ruleType`,`ruleArn`,`status`,`userId`,`displayName`,`createdDate`,`modifiedDate`,`severity`,`category`) VALUES ('PacMan_Ec2PublicAccessPortWithS5Vulnerability_version-1_Ec2PublicAccessPortWithS5Vulnerability_ec2','aws_ec2_pub_vuln_s5_rule','PacMan_Ec2PublicAccessPortWithS5Vulnerability_version-1','Ec2PublicAccessPortWithS5Vuln','ec2','aws','Ec2PublicAccessPortWithS5Vuln','{"params":[{"encrypt":false,"value":"check-for-ec2-public-access-port-with-s5-vulnerabilities","key":"ruleKey"},{"encrypt":false,"value":"S5","key":"severityVulnValue"},{"encrypt":false,"value":"PacMan_EC2WithPublicIPAccess_version-1_Ec2WithPublicAccess_ec2","key":"ec2PortRuleId"},{"key":"esEc2WithVulnInfoForS5Url","value":"/aws_ec2/vulninfo/_search","isValueNew":true,"encrypt":false},{"key":"esEc2PubAccessPortUrl","value":"/aws/issue_ec2/_search","isValueNew":true,"encrypt":false},{"key":"esAppElbWithInstanceUrl","value":"/aws_appelb/appelb_instances/_search","isValueNew":true,"encrypt":false},{"key":"esClassicElbWithInstanceUrl","value":"/aws_classicelb/classicelb_instances/_search","isValueNew":true,"encrypt":false},{"key":"esAppElbPubAccessPortUrl","value":"/aws_appelb/issue_appelb/_search","isValueNew":true,"encrypt":false},{"key":"esClassicElbPubAccessPortUrl","value":"/aws_classicelb/issue_classicelb/_search","isValueNew":true,"encrypt":false},{"key":"appElbPortRuleId","value":"PacMan_ElbWithPublicAccess_version-1_ApplicationElbWithPublicAccess_appelb","isValueNew":true,"encrypt":false},{"key":"classicElbPortRuleId","value":"PacMan_ElbWithPublicAccess_version-1_ClassicElbWithPublicAccess_classicelb","isValueNew":true,"encrypt":false},{"encrypt":false,"value":"critical","key":"severity"},{"encrypt":false,"value":"security","key":"ruleCategory"}],"environmentVariables":[],"ruleId":"PacMan_Ec2PublicAccessPortWithS5Vulnerability_version-1_Ec2PublicAccessPortWithS5Vulnerability_ec2","autofix":false,"alexaKeyword":"Ec2PublicAccessPortWithS5Vulnerability","ruleRestUrl":"","targetType":"ec2","pac_ds":"aws","policyId":"PacMan_Ec2PublicAccessPortWithS5Vulnerability_version-1","assetGroup":"aws","ruleUUID":"aws_ec2_pub_vuln_s5_rule","ruleType":"ManageRule"}','0 0 ? * MON *','','','Manage Rule',concat('arn:aws:events:',@region,':',@account,':rule/aws_ec2_pub_vuln_s5_rule'),'ENABLED','ASGC','An Ec2 instance with remotely exploitable vulnerability (S5) should not be open to internet','2019-08-05','2019-08-05','high','governance'); INSERT IGNORE INTO cf_RuleInstance (`ruleId`,`ruleUUID`,`policyId`,`ruleName`,`targetType`,`assetGroup`,`alexaKeyword`,`ruleParams`,`ruleFrequency`,`ruleExecutable`,`ruleRestUrl`,`ruleType`,`ruleArn`,`status`,`userId`,`displayName`,`createdDate`,`modifiedDate`,`severity`,`category`) VALUES ('PacMan_Ec2InstanceScannedByQualys_version-1_Ec2-instance-scanned-by-qualys-API_ec2','aws_ec2_qualys_scanned_rule','PacMan_Ec2InstanceScannedByQualys_version-1','Ec2InstanceScannedByQualysAPI','ec2','aws','Ec2InstanceScannedByQualysAPI','{"params":[{"encrypt":false,"value":"30","key":"target"},{"key":"esQualysUrl","value":"/aws_ec2/qualysinfo/_search","isValueNew":true,"encrypt":false},{"key":"discoveredDaysRange","value":"7","isValueNew":true,"encrypt":false},{"key":"ruleKey","value":"check-for-resource-scanned-by-qualys","isValueNew":true,"encrypt":false},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"security","key":"ruleCategory"}],"environmentVariables":[],"ruleId":"PacMan_Ec2InstanceScannedByQualys_version-1_Ec2-instance-scanned-by-qualys-API_ec2","autofix":false,"alexaKeyword":"Ec2InstanceScannedByQualysAPI","ruleRestUrl":"","targetType":"ec2","pac_ds":"aws","policyId":"PacMan_Ec2InstanceScannedByQualys_version-1","assetGroup":"aws","ruleUUID":"aws_ec2_qualys_scanned_rule","ruleType":"ManageRule"}','0 0 ? * MON *','','','Manage Rule',concat('arn:aws:events:',@region,':',@account,':rule/aws_ec2_qualys_scanned_rule'),'ENABLED','ASGC','Every EC2 instance should be scanned by Qualys vulnerability assessment tool atleast once a month','2019-09-18','2019-09-18','high','security'); - +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_VirtualmachineTaggingRule_virtualmachine','azure_virtualmachine_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','VirtualmachineTaggingRule','virtualmachine','azure','VirtualmachineTaggingRule','{"params":[{"encrypt":false,"value":"check-for-missing-mandatory-tags","key":"ruleKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_VirtualmachineTaggingRule_virtualmachine","autofix":false,"alexaKeyword":"VirtualmachineTaggingRule","ruleRestUrl":"","targetType":"virtualmachine","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","assetGroup":"azure","ruleUUID":"azure_virtualmachine_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_virtualmachine_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Virtualmachine should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_SqlserverTaggingRule_sqlserver','azure_sqlserver_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','SqlserverTaggingRule','sqlserver','azure','SqlserverTaggingRule','{"params":[{"encrypt":false,"value":"check-for-missing-mandatory-tags","key":"ruleKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_SqlserverTaggingRule_sqlserver","autofix":false,"alexaKeyword":"SqlserverTaggingRule","ruleRestUrl":"","targetType":"sqlserver","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","assetGroup":"azure","ruleUUID":"azure_sqlserver_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_sqlserver_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Sqlserver should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_SqldatabaseTaggingRule_sqldatabase','azure_sqldatabase_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','SqldatabaseTaggingRule','sqldatabase','azure','SqldatabaseTaggingRule','{"params":[{"encrypt":false,"value":"check-for-missing-mandatory-tags","key":"ruleKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_SqldatabaseTaggingRule_sqldatabase","autofix":false,"alexaKeyword":"SqldatabaseTaggingRule","ruleRestUrl":"","targetType":"sqldatabase","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","assetGroup":"azure","ruleUUID":"azure_sqldatabase_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_sqldatabase_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Sqldatabase should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); /* Omni Seach Configuration */ @@ -2523,6 +2525,7 @@ UPDATE `pacmandata`.`pac_config_key_metadata` SET `description` = 'Vulnerability UPDATE `pacmandata`.`pac_config_key_metadata` SET `description` = 'Vulnerability application resource details both' WHERE `cfkey` = 'vulnerability.application.resourcedetailsboth'; UPDATE `pacmandata`.`pac_config_key_metadata` SET `description` = 'Vulnerability severity summary' WHERE `cfkey` = 'vulnerability.summary.severity'; UPDATE `pacmandata`.`pac_config_key_metadata` SET `description` = 'Vulnerability types' WHERE `cfkey` = 'vulnerability.types'; +UPDATE `cf_Policy` SET policyDesc = 'All cloud assets should be tagged with following mandatory tags. Application, Environment, Role and Stack. Assets without these mandatory tags will be marked as non-complaint. Below is an example for the tag value pairs.\n\nTag name: Application\nExample value: Rebellion\n\nNotes\nThis value for the application tag should be the approved application name give for the project during the cloud on-boarding process. Unknown applications will be marked for review and possible termination.\n\nTag name: Environment\nExample value: Production or Non Production or Non Production::qat1 or Non Production::dit1 (Refer Naming guide)\n\nNotes\nThe value for environment should distinguish the asset as a Production or Non Production class. You can further qualify Non Production assets using the :: separator. Look at the examples 3 and 4.\n\nTag name: Stack\nExample Value: Apache Httpd\n\nTag name: Role\nExample value: Webserver\n\n \nEach asset should at least have these 4 mandatory tags. You can have additional tags as well' WHERE policyId = 'PacMan_TaggingRule_version-1'; DELETE FROM `pac_config_properties` WHERE cfkey='features.vulnerability.enabled'; INSERT IGNORE INTO pac_config_properties(`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('features.vulnerability.enabled',concat(@VULNERABILITY_FEATURE_ENABLED,''),'api','prd','latest',NULL,NULL,NULL,NULL); From d733e73e758b46b96b6068c7660ec4fcccc292c8 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Mon, 28 Oct 2019 07:00:07 +0530 Subject: [PATCH 029/107] Reinstall command is made same as redeployement for legacy. --- installer/custom/commands/redeploy.py | 151 ++++++++---------- .../commands/{reinstall.py => redeployv1.py} | 150 ++++++++++------- 2 files changed, 151 insertions(+), 150 deletions(-) rename installer/custom/commands/{reinstall.py => redeployv1.py} (52%) diff --git a/installer/custom/commands/redeploy.py b/installer/custom/commands/redeploy.py index acefa3d9d..1755bdfc7 100644 --- a/installer/custom/commands/redeploy.py +++ b/installer/custom/commands/redeploy.py @@ -7,26 +7,36 @@ from core.providers.aws.boto3 import elb from core.terraform import PyTerraform from core.providers.aws.boto3.ecs import stop_all_tasks_in_a_cluster, deregister_task_definition +from core.commands import BaseCommand +from core.config import Settings +from core.terraform import PyTerraform +from core import constants as K from threading import Thread import time import importlib import sys -import inspect import os class Redeploy(BaseCommand): """ - This calss is defined to redeploy PacBot which is already installed by Installer command + This calss is defined to reinstall PacBot which is already installed by Redeploy command Attributes: validation_class (class): This validate the input and resources input_class (class): Main class to read input from user install_class (class): Provider based install class need_complete_install (boolean): True if complete installation is required else False - + dry_run (boolean): Need actual insalltion or not """ def __init__(self, args): + + Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True) + + args.append((K.CATEGORY_FIELD_NAME, "deploy")) + args.append((K.CATEGORY_FIELD_NAME, "upload_tf")) + self.destroy_resource_tags_list = [v for (k, v) in args if k == self.category_field_name] + args.append((K.CATEGORY_FIELD_NAME, "deploy")) args.append((K.CATEGORY_FIELD_NAME, "roles")) args.append((K.CATEGORY_FIELD_NAME, "all_read_role")) @@ -35,18 +45,22 @@ def __init__(self, args): args.append((K.CATEGORY_FIELD_NAME, "submit-job")) args.append((K.CATEGORY_FIELD_NAME, "rule-engine-job")) args.append((K.CATEGORY_FIELD_NAME, "upload_tf")) + self.reinstall_resource_tags_list = [v for (k, v) in args if k == self.category_field_name] self.need_complete_install = self._need_complete_installation() - Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True) - super().__init__(args) + + self.dry_run = True if any([x[1] for x in args if x[0] == "dry-run"]) else self.dry_run def _need_complete_installation(self): + """ + Checj whether the redeploy need complete reinstallation. + """ need_complete_install = False redshift_cluster_file_tf = os.path.join(Settings.TERRAFORM_DIR, "datastore_redshift_RedshiftCluster.tf") redshift_cluster_file_tf_json = os.path.join(Settings.TERRAFORM_DIR, "datastore_redshift_RedshiftCluster.tf.json") - if os.path.exists(redshift_cluster_file) or os.path.exists(redshift_cluster_file_tf_json): + if os.path.exists(redshift_cluster_file_tf) or os.path.exists(redshift_cluster_file_tf_json): need_complete_install = True return need_complete_install @@ -78,7 +92,7 @@ def initialize_install_classes(self, provider): self.input_class = getattr(importlib.import_module( provider.provider_module + '.input'), 'SystemInstallInput') self.install_class = getattr(importlib.import_module( - provider.provider_module + '.install'), 'Install') + provider.provider_module + '.reinstall'), 'ReInstall') def re_deploy_pacbot(self, input_instance): """ @@ -87,93 +101,32 @@ def re_deploy_pacbot(self, input_instance): Args: input_instance (Input object): User input values """ - resources_to_process = self.get_resources_to_process(self.resource_tags_list, input_instance) - try: - resources_to_taint = self.get_resources_with_given_tags(input_instance, ["deploy"]) - resources_to_taint = [resource for resource in resources_to_taint if resource.PROCESS is True] - response = PyTerraform().terraform_taint(resources_to_taint) # If tainted or destroyed already then skip it - except Exception as e: - pass + resources_to_destroy = self.get_resources_to_process(self.destroy_resource_tags_list, input_instance) + resources_to_install = self.get_resources_to_process(self.reinstall_resource_tags_list, input_instance) terraform_with_targets = False if self.need_complete_install else True - resources_to_process = self.get_complete_resources(input_instance) if self.need_complete_install else resources_to_process - - self.run_pre_deployment_process(resources_to_process) - self.run_real_deployment(input_instance, resources_to_process, terraform_with_targets) - - def run_pre_deployment_process(self, resources_to_process): - """ - Before redeploy get started do predeployment activities + resources_to_install = self.get_complete_resources(input_instance) if self.need_complete_install else resources_to_install - Args: - resources_to_process (list): List of resources to be created/updated - """ - if not self.dry_run: - elb.delete_all_listeners_of_alb( - ApplicationLoadBalancer.get_input_attr('name'), - Settings.AWS_AUTH_CRED) - - tg_resources = self._get_resources_of_a_given_class_type(resources_to_process, ALBTargetGroupResource) - tg_names = [resource.get_input_attr('name') for resource in tg_resources] - elb.delete_alltarget_groups( - tg_names, - Settings.AWS_AUTH_CRED) - - def inactivate_required_services_for_redeploy(self, resources_to_process, dry_run): - """ - Before redeploy get started or on redeploy happens stop the tasks and deregister task definition + # self.run_pre_deployment_process(resources_to_process) + self.run_real_deployment(input_instance, resources_to_destroy, resources_to_install, terraform_with_targets) - Args: - resources_to_process (list): List of resources to be created/updated - only_tasks (boolean): This flasg decides whther to deregister task definition or not - """ - if dry_run: - return - - for resource in resources_to_process: - if self.terraform_thread.isAlive(): - resource_base_classes = inspect.getmro(resource.__class__) - - if ECSTaskDefinitionResource in resource_base_classes: - try: - deregister_task_definition( - resource.get_input_attr('family'), - Settings.AWS_AUTH_CRED, - ) - except: - pass - elif ECSClusterResource in resource_base_classes: - cluster_name = resource.get_input_attr('name') - else: - return - - for i in range(3): - if self.terraform_thread.isAlive(): - try: - stop_all_tasks_in_a_cluster( - cluster_name, - Settings.AWS_ACCESS_KEY, - Settings.AWS_SECRET_KEY, - Settings.AWS_REGION - ) - except: - pass - time.sleep(20) - else: - return - - def run_real_deployment(self, input_instance, resources_to_process, terraform_with_targets): + def run_real_deployment(self, input_instance, resources_to_destroy, resources_to_install, terraform_with_targets): """ Main thread method which invokes the 2 thread: one for actual execution and another for displaying status Args: input_instance (Input obj): Input object with values read from user - resources_to_process (list): List of resources to be created/updated + resources_to_destroy (list): List of resources to be destroyed for recreation + resources_to_install (list): List of resources to be recreated terraform_with_targets (boolean): This is True since redeployment is happening """ - self.terraform_thread = Thread(target=self.run_tf_apply, args=(input_instance, list(resources_to_process), terraform_with_targets)) + self.terraform_thread = Thread( + target=self.run_reinstallation, + args=(input_instance, list(resources_to_destroy), list(resources_to_install), terraform_with_targets)) # Dt-run variable is passed as it is rquired otherwise argument parsing issue will occur - stop_related_task_thread = Thread(target=self.inactivate_required_services_for_redeploy, args=(list(resources_to_process), self.dry_run)) + stop_related_task_thread = Thread( + target=self.inactivate_required_services_for_redeploy, + args=(list(resources_to_destroy), list(resources_to_install), self.dry_run)) self.terraform_thread.start() stop_related_task_thread.start() @@ -181,21 +134,43 @@ def run_real_deployment(self, input_instance, resources_to_process, terraform_wi self.terraform_thread.join() stop_related_task_thread.join() - def run_tf_apply(self, input_instance, resources_to_process, terraform_with_targets): + def inactivate_required_services_for_redeploy(self, resources_to_destroy, resources_to_install, dry_run): + """ + This is a place holder to run some script parallely if there is anything to do + + Args: + resources_to_destroy (list): List of resources to be destroyed for recreation + resources_to_install (list): List of resources to be recreated + only_tasks (boolean): This flasg decides whther to deregister task definition or not + """ + pass + + def generate_terraform_files_and_upgrade_state(self, input_instance): + all_resources = self.get_complete_resources(input_instance) + for resource in all_resources: + resource.generate_terraform() + PyTerraform.terrafomr12_upgrade() # This is required only when terraform version 12 is used + + def run_reinstallation(self, input_instance, resources_to_destroy, resources_to_install, terraform_with_targets): """ Execute the installation of resources by invoking the execute method of provider class Args: input_instance (Input obj): Input object with values read from user - resources_to_process (list): List of resources to be created/updated + resources_to_destroy (list): List of resources to be destroyed for recreation + resources_to_install (list): List of resources to be recreated terraform_with_targets (boolean): This is True since redeployment is happening """ - self.install_class( - self.args, + self.generate_terraform_files_and_upgrade_state(input_instance) + + installer = self.install_class( input_instance, check_dependent_resources=False - ).execute( - resources_to_process, + ) + + installer.execute( + resources_to_destroy, + resources_to_install, terraform_with_targets, self.dry_run ) diff --git a/installer/custom/commands/reinstall.py b/installer/custom/commands/redeployv1.py similarity index 52% rename from installer/custom/commands/reinstall.py rename to installer/custom/commands/redeployv1.py index 4a4418212..c21441bad 100644 --- a/installer/custom/commands/reinstall.py +++ b/installer/custom/commands/redeployv1.py @@ -1,31 +1,23 @@ -from core.commands import BaseCommand -from core.config import Settings -from core.terraform import PyTerraform -from core import constants as K from threading import Thread import time import importlib import sys +import inspect import os -class Reinstall(BaseCommand): +class RedeployV1(BaseCommand): """ - This calss is defined to reinstall PacBot which is already installed by Installer command + This calss is Older version defined to redeploy PacBot which is already installed by Installer command Attributes: validation_class (class): This validate the input and resources input_class (class): Main class to read input from user install_class (class): Provider based install class + need_complete_install (boolean): True if complete installation is required else False + """ def __init__(self, args): - - Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True) - - args.append((K.CATEGORY_FIELD_NAME, "deploy")) - args.append((K.CATEGORY_FIELD_NAME, "upload_tf")) - self.destroy_resource_tags_list = [v for (k, v) in args if k == self.category_field_name] - args.append((K.CATEGORY_FIELD_NAME, "deploy")) args.append((K.CATEGORY_FIELD_NAME, "roles")) args.append((K.CATEGORY_FIELD_NAME, "all_read_role")) @@ -34,11 +26,10 @@ def __init__(self, args): args.append((K.CATEGORY_FIELD_NAME, "submit-job")) args.append((K.CATEGORY_FIELD_NAME, "rule-engine-job")) args.append((K.CATEGORY_FIELD_NAME, "upload_tf")) - self.reinstall_resource_tags_list = [v for (k, v) in args if k == self.category_field_name] self.need_complete_install = self._need_complete_installation() - - self.dry_run = True if any([x[1] for x in args if x[0] == "dry-run"]) else self.dry_run + Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True) + super().__init__(args) def _need_complete_installation(self): need_complete_install = False @@ -46,7 +37,7 @@ def _need_complete_installation(self): redshift_cluster_file_tf = os.path.join(Settings.TERRAFORM_DIR, "datastore_redshift_RedshiftCluster.tf") redshift_cluster_file_tf_json = os.path.join(Settings.TERRAFORM_DIR, "datastore_redshift_RedshiftCluster.tf.json") - if os.path.exists(redshift_cluster_file_tf) or os.path.exists(redshift_cluster_file_tf_json): + if os.path.exists(redshift_cluster_file) or os.path.exists(redshift_cluster_file_tf_json): need_complete_install = True return need_complete_install @@ -78,7 +69,7 @@ def initialize_install_classes(self, provider): self.input_class = getattr(importlib.import_module( provider.provider_module + '.input'), 'SystemInstallInput') self.install_class = getattr(importlib.import_module( - provider.provider_module + '.reinstall'), 'ReInstall') + provider.provider_module + '.install'), 'Install') def re_deploy_pacbot(self, input_instance): """ @@ -87,23 +78,82 @@ def re_deploy_pacbot(self, input_instance): Args: input_instance (Input object): User input values """ - resources_to_destroy = self.get_resources_to_process(self.destroy_resource_tags_list, input_instance) - resources_to_install = self.get_resources_to_process(self.reinstall_resource_tags_list, input_instance) - - # try: - # resources_to_taint = self.get_resources_with_given_tags(input_instance, ["deploy"]) - # resources_to_taint = [resource for resource in resources_to_taint if resource.PROCESS is True] - # response = PyTerraform().terraform_taint(resources_to_taint) # If tainted or destroyed already then skip it - # except Exception as e: - # pass + resources_to_process = self.get_resources_to_process(self.resource_tags_list, input_instance) + try: + resources_to_taint = self.get_resources_with_given_tags(input_instance, ["deploy"]) + resources_to_taint = [resource for resource in resources_to_taint if resource.PROCESS is True] + response = PyTerraform().terraform_taint(resources_to_taint) # If tainted or destroyed already then skip it + except Exception as e: + pass terraform_with_targets = False if self.need_complete_install else True - resources_to_install = self.get_complete_resources(input_instance) if self.need_complete_install else resources_to_install + resources_to_process = self.get_complete_resources(input_instance) if self.need_complete_install else resources_to_process + + self.run_pre_deployment_process(resources_to_process) + self.run_real_deployment(input_instance, resources_to_process, terraform_with_targets) + + def run_pre_deployment_process(self, resources_to_process): + """ + Before redeploy get started do predeployment activities - # self.run_pre_deployment_process(resources_to_process) - self.run_real_deployment(input_instance, resources_to_destroy, resources_to_install, terraform_with_targets) + Args: + resources_to_process (list): List of resources to be created/updated + """ + if not self.dry_run: + elb.delete_all_listeners_of_alb( + ApplicationLoadBalancer.get_input_attr('name'), + Settings.AWS_AUTH_CRED) + + tg_resources = self._get_resources_of_a_given_class_type(resources_to_process, ALBTargetGroupResource) + tg_names = [resource.get_input_attr('name') for resource in tg_resources] + elb.delete_alltarget_groups( + tg_names, + Settings.AWS_AUTH_CRED) + + def inactivate_required_services_for_redeploy(self, resources_to_process, dry_run): + """ + Before redeploy get started or on redeploy happens stop the tasks and deregister task definition - def run_real_deployment(self, input_instance, resources_to_destroy, resources_to_install, terraform_with_targets): + Args: + resources_to_process (list): List of resources to be created/updated + only_tasks (boolean): This flasg decides whther to deregister task definition or not + """ + if dry_run: + return + + for resource in resources_to_process: + if self.terraform_thread.isAlive(): + resource_base_classes = inspect.getmro(resource.__class__) + + if ECSTaskDefinitionResource in resource_base_classes: + try: + deregister_task_definition( + resource.get_input_attr('family'), + Settings.AWS_AUTH_CRED, + ) + except: + pass + elif ECSClusterResource in resource_base_classes: + cluster_name = resource.get_input_attr('name') + else: + return + + for i in range(3): + if self.terraform_thread.isAlive(): + try: + stop_all_tasks_in_a_cluster( + cluster_name, + Settings.AWS_ACCESS_KEY, + Settings.AWS_SECRET_KEY, + Settings.AWS_REGION + ) + except: + pass + time.sleep(20) + else: + return + + def run_real_deployment(self, input_instance, resources_to_process, terraform_with_targets): """ Main thread method which invokes the 2 thread: one for actual execution and another for displaying status @@ -112,13 +162,9 @@ def run_real_deployment(self, input_instance, resources_to_destroy, resources_to resources_to_process (list): List of resources to be created/updated terraform_with_targets (boolean): This is True since redeployment is happening """ - self.terraform_thread = Thread( - target=self.run_tf_apply, - args=(input_instance, list(resources_to_destroy), list(resources_to_install), terraform_with_targets)) + self.terraform_thread = Thread(target=self.run_tf_apply, args=(input_instance, list(resources_to_process), terraform_with_targets)) # Dt-run variable is passed as it is rquired otherwise argument parsing issue will occur - stop_related_task_thread = Thread( - target=self.inactivate_required_services_for_redeploy, - args=(list(resources_to_destroy), list(resources_to_install), self.dry_run)) + stop_related_task_thread = Thread(target=self.inactivate_required_services_for_redeploy, args=(list(resources_to_process), self.dry_run)) self.terraform_thread.start() stop_related_task_thread.start() @@ -126,23 +172,7 @@ def run_real_deployment(self, input_instance, resources_to_destroy, resources_to self.terraform_thread.join() stop_related_task_thread.join() - def inactivate_required_services_for_redeploy(self, resources_to_destroy, resources_to_install, dry_run): - """ - Before redeploy get started or on redeploy happens stop the tasks and deregister task definition - - Args: - resources_to_process (list): List of resources to be created/updated - only_tasks (boolean): This flasg decides whther to deregister task definition or not - """ - pass - - def generate_terraform_files_and_upgrade_state(self, input_instance): - all_resources = self.get_complete_resources(input_instance) - for resource in all_resources: - resource.generate_terraform() - PyTerraform.terrafomr12_upgrade() # This is required only when terraform version 12 is used - - def run_tf_apply(self, input_instance, resources_to_destroy, resources_to_install, terraform_with_targets): + def run_tf_apply(self, input_instance, resources_to_process, terraform_with_targets): """ Execute the installation of resources by invoking the execute method of provider class @@ -151,16 +181,12 @@ def run_tf_apply(self, input_instance, resources_to_destroy, resources_to_instal resources_to_process (list): List of resources to be created/updated terraform_with_targets (boolean): This is True since redeployment is happening """ - self.generate_terraform_files_and_upgrade_state(input_instance) - - installer = self.install_class( + self.install_class( + self.args, input_instance, check_dependent_resources=False - ) - - installer.execute( - resources_to_destroy, - resources_to_install, + ).execute( + resources_to_process, terraform_with_targets, self.dry_run ) From aa703294158aac21e61995180160f376787dd5ac Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Mon, 28 Oct 2019 15:24:02 +0530 Subject: [PATCH 030/107] Silent install option added to install pacbot without reading inputs from console --- installer/core/command.py | 6 +++- installer/core/commands/__init__.py | 4 ++- installer/core/constants.py | 5 +++ installer/core/providers/aws/input.py | 31 +++++++++++++----- installer/provison.sh | 45 +++++++++++++++++++++++++++ installer/settings/default.local.py | 5 ++- 6 files changed, 85 insertions(+), 11 deletions(-) diff --git a/installer/core/command.py b/installer/core/command.py index 4667cef5c..2a725575b 100644 --- a/installer/core/command.py +++ b/installer/core/command.py @@ -17,7 +17,11 @@ class Command: optional_args (list): Optional arguments passed to CLI """ base_dir = 'commands' - valid_arg_keys = ["--" + K.CATEGORY_FIELD_NAME, "--dry-run"] + valid_arg_keys = [ + "--" + K.CATEGORY_FIELD_NAME, + "--dry-run", + "--silent" + ] def __init__(self): self.commands_dir_path = self.get_core_commands_dir_path() diff --git a/installer/core/commands/__init__.py b/installer/core/commands/__init__.py index 90f329bdb..250fd6b99 100644 --- a/installer/core/commands/__init__.py +++ b/installer/core/commands/__init__.py @@ -23,6 +23,7 @@ class BaseCommand(metaclass=ABCMeta): category_field_name = K.CATEGORY_FIELD_NAME terraform_with_targets = False dry_run = False + silent_install = False def __init__(self, args): """ @@ -37,6 +38,7 @@ def __init__(self, args): self.terraform_with_targets = True self.dry_run = True if any([x[1] for x in args if x[0] == "dry-run"]) else self.dry_run + self.silent_install = True if any([x[1] for x in args if x[0] == "silent"]) else self.silent_install def get_complete_resources(self, input_instance): """ @@ -140,7 +142,7 @@ def read_input(self): Returns: input_instancce (object): Provider Input instance """ - input_instancce = self.input_class() + input_instancce = self.input_class(self.silent_install) input_instancce.read_input() return input_instancce diff --git a/installer/core/constants.py b/installer/core/constants.py index 642a4f22f..e9c22cca2 100644 --- a/installer/core/constants.py +++ b/installer/core/constants.py @@ -32,6 +32,11 @@ AWS_WITH_EC2_ROLE = "3. Using IAM role attached to the this instance" AWS_CHOOSE_AUTH_OPTION = "Type 1 or 2 or 3 to continue to create services in AWS: " AWS_INCORRECT_MECHANISM = "Entered an incorrect value!!!" +AWS_AUTH_MECHANISM_NOT_SUPPLIED = "Please add value 1 or 2 or 3 for AWS_AUTH_MECHANISM in settings/local.py" +AWS_ACCESS_KEY_NOT_SUPPLIED ="Please enter value for AWS_ACCESS_KEY in settings/local.py file" +AWS_SECRET_KEY_NOT_SUPPLIED ="Please enter value for AWS_SECRET_KEY in settings/local.py file" +AWS_REGION_NOT_SUPPLIED ="Please enter value for AWS_REGION in settings/local.py file" +AWS_ASSUME_ROLE_NOT_SUPPLIED ="Please enter value for AWS_ASSUME_ROLE_ARN in settings/local.py file" AWS_ACCESS_KEY_INPUT = "Please enter AWS access key: " AWS_SECRET_KEY_INPUT = "Please enter AWS secret key: " diff --git a/installer/core/providers/aws/input.py b/installer/core/providers/aws/input.py index e847d7d35..083e456f4 100644 --- a/installer/core/providers/aws/input.py +++ b/installer/core/providers/aws/input.py @@ -11,6 +11,9 @@ class SystemInput(MsgMixin, metaclass=ABCMeta): """Base input class for installation/destruction/status commands. This class reads required input from user for the process to start""" AWS_AUTH_CRED = {} + def __init__(slef, silent_install = False): + self.silent_install = silent_install + def read_input(self): """Read required inputs from user for the process to start""" self.show_step_heading(K.INPUT_READING_STARTED) @@ -32,6 +35,13 @@ def read_input(self): self.show_step_finish(K.INPUT_READING_COMPLETED) def read_aws_auth_mechanism(self): + if self.silent_install: + auth_mechanism = getattr(Settings, 'AWS_ACCESS_KEY', None) + if auth_mechanism in [1, 2, 3]: + return auth_mechanism + + raise Exception(K.AWS_AUTH_MECHANISM_NOT_SUPPLIED) + while True: self.show_inner_inline_message("\n\t%s" % K.AWS_AUTH_MECHANISM) self.show_inner_inline_message("\n\t%s" % K.AWS_WITH_KEYS) @@ -48,7 +58,11 @@ def read_aws_auth_mechanism(self): def read_aws_access_key(self): """Read AWS access key from user if it is not already set in settings""" settings_access_key = getattr(Settings, 'AWS_ACCESS_KEY', None) + if settings_access_key is None or settings_access_key == '': + if self.silent_install: + raise Exception(K.AWS_ACCESS_KEY_NOT_SUPPLIED) + aws_access_key = input("\n\t%s" % K.AWS_ACCESS_KEY_INPUT) if len(aws_access_key) < 20: self.show_step_inner_error("\n\t" + K.INVALID_KEY) @@ -62,8 +76,10 @@ def read_aws_secret_key(self): """Read AWS secret key from user if it is not already set in settings""" settings_secret_key = getattr(Settings, 'AWS_SECRET_KEY', None) if settings_secret_key is None or settings_secret_key == '': - aws_secret_key = input("\n\t%s" % K.AWS_SECRET_KEY_INPUT) + if self.silent_install: + raise Exception(K.AWS_SECRET_KEY_NOT_SUPPLIED) + aws_secret_key = input("\n\t%s" % K.AWS_SECRET_KEY_INPUT) if len(aws_secret_key) < 25: self.show_step_inner_error("\n\t" + K.INVALID_KEY) raise Exception(K.INVALID_KEY) @@ -76,6 +92,9 @@ def read_aws_assume_role_arn(self): """Read AWS secret key from user if it is not already set in settings""" settings_assume_role_arn = getattr(Settings, 'AWS_ASSUME_ROLE_ARN', None) if settings_assume_role_arn is None or settings_assume_role_arn == '': + if self.silent_install: + raise Exception(K.AWS_ASSUME_ROLE_NOT_SUPPLIED) + assume_role_arn = input("\n\t%s" % K.AWS_ASSUME_ROLE_INPUT) else: assume_role_arn = settings_assume_role_arn @@ -86,6 +105,9 @@ def read_aws_region(self): """Read AWS region from user if it is not already set in settings""" settings_region = getattr(Settings, 'AWS_REGION', None) if settings_region is None or settings_region == '': + if self.silent_install: + raise Exception(K.AWS_REGION_NOT_SUPPLIED) + aws_region = input("\n\t%s" % K.AWS_REGION_INPUT) else: aws_region = settings_region @@ -115,13 +137,6 @@ class SystemDestroyInput(SystemInput): def read_input(self): super().read_input() - # for item in Settings.get('INSTALL_INPUTS_REQUIRED', []): - # key_val = input("\n\t%s" % item['input_msg']) - # if item['required']: - # if key_val.strip() == "": - # raise Exception("Value required for %s" % item['input_key']) - # Settings.set(item['input_key'], key_val) - # setattr(self, item['input_key'], key_val) class SystemStatusInput(SystemInput): diff --git a/installer/provison.sh b/installer/provison.sh index 0bd221ff7..b7ed9150e 100644 --- a/installer/provison.sh +++ b/installer/provison.sh @@ -58,3 +58,48 @@ virtualenv ~/envs/pacbot_env --python=python3 source ~/envs/pacbot_env/bin/activate echo source ~/envs/pacbot_env/bin/activate >> ~/.bashrc pip install -r requirements.txt + + +--------------------------------- +-----Ubuntu----- +--------------------------------- + +sudo apt -y update +sudo add-apt-repository ppa:openjdk-r/ppa +sudo apt-get update +sudo apt install -y openjdk-8-jdk +sudo update-java-alternatives --set openjdk-8-jdk + +sudo apt install -y maven +sudo apt install -y docker +sudo apt install -y docker.io +sudo systemctl start docker +sudo apt install -y python3 +sudo apt install -y python3-venv +sudo apt install -y mysql-client + +sudo apt -y install unzip +wget https://releases.hashicorp.com/terraform/0.11.10/terraform_0.11.10_linux_amd64.zip +unzip terraform_0.11.10_linux_amd64.zip +sudo mv terraform /usr/bin + +echo alias cdd=\"cd $(pwd)\" >> ~/.bashrc +echo alias cdt=\"cd $(pwd)/data/terraform\" >> ~/.bashrc +echo alias cdl=\"cd $(pwd)/log\" >> ~/.bashrc +source ~/.bashrc + +sudo apt install -y curl +curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash - +sudo apt -y update +sudo apt install -y nodejs +sudo apt install -y npm +sudo npm install -g yarn +sudo npm install -g @angular/cli + +## Install virtualenv +mkdir ~/envs/ +python3 -m venv ~/envs/pacbot_env +source ~/envs/pacbot_env/bin/activate +echo source ~/envs/pacbot_env/bin/activate >> ~/.bashrc +pip install -r requirements.txt + diff --git a/installer/settings/default.local.py b/installer/settings/default.local.py index 7af122139..d7bcacdb3 100644 --- a/installer/settings/default.local.py +++ b/installer/settings/default.local.py @@ -43,10 +43,13 @@ USER_EMAIL_ID = "" # System reads below data from user if not updated here +AWS_AUTH_MECHANISM = None # Value should be numeric 1 or 2 or 3. I. If kept like this input is read from +# if AWS_AUTH_MECHANISM == 1 AWS_ACCESS_KEY = "" AWS_SECRET_KEY = "" AWS_REGION = "" - +# If AWS_AUTH_MECHANISM == 2, AWS_ASSUME_ROLE_ARN is required +AWS_ASSUME_ROLE_ARN = "" # This settings enable Vulnerability feature and servie ENABLE_VULNERABILITY_FEATURE = False From 8bf25c6cf780bdf3d92c07c00d1aabb25bd1038d Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Mon, 28 Oct 2019 16:21:52 +0530 Subject: [PATCH 031/107] Default values are added if aws cli is not configured with values --- installer/core/providers/aws/boto3/sts.py | 5 +++-- installer/core/providers/aws/input.py | 17 ++++++++++++----- installer/files/scripts/utils.py | 13 +++++++------ 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/installer/core/providers/aws/boto3/sts.py b/installer/core/providers/aws/boto3/sts.py index 6df3ac397..bff9456ac 100644 --- a/installer/core/providers/aws/boto3/sts.py +++ b/installer/core/providers/aws/boto3/sts.py @@ -16,9 +16,10 @@ def get_sts_client(aws_auth_cred): return prepare_aws_client_with_given_cred('sts', aws_auth_cred) -def generate_temp_credentials(assume_role_arn): +def generate_temp_credentials(assume_role_arn, region_name): response = boto3.client( - 'sts' + "sts", + region_name=region_name ).assume_role( RoleArn=assume_role_arn, RoleSessionName=str(uuid.uuid4()) diff --git a/installer/core/providers/aws/input.py b/installer/core/providers/aws/input.py index 083e456f4..6808110c5 100644 --- a/installer/core/providers/aws/input.py +++ b/installer/core/providers/aws/input.py @@ -11,7 +11,7 @@ class SystemInput(MsgMixin, metaclass=ABCMeta): """Base input class for installation/destruction/status commands. This class reads required input from user for the process to start""" AWS_AUTH_CRED = {} - def __init__(slef, silent_install = False): + def __init__(self, silent_install = False): self.silent_install = silent_install def read_input(self): @@ -19,15 +19,18 @@ def read_input(self): self.show_step_heading(K.INPUT_READING_STARTED) self.AWS_AUTH_CRED['aws_auth_option'] = self.read_aws_auth_mechanism() + self.AWS_AUTH_CRED['aws_region'] = self.read_aws_region() if self.AWS_AUTH_CRED['aws_auth_option'] == 1: self.AWS_AUTH_CRED['aws_access_key'] = self.read_aws_access_key() self.AWS_AUTH_CRED['aws_secret_key'] = self.read_aws_secret_key() elif self.AWS_AUTH_CRED['aws_auth_option'] == 2: self.AWS_AUTH_CRED['assume_role_arn'] = self.read_aws_assume_role_arn() - self.AWS_AUTH_CRED['tmp_credentials'] = generate_temp_credentials(self.AWS_AUTH_CRED['assume_role_arn']) + self.AWS_AUTH_CRED['tmp_credentials'] = generate_temp_credentials( + self.AWS_AUTH_CRED['assume_role_arn'], + self.AWS_AUTH_CRED['aws_region'] + ) - self.AWS_AUTH_CRED['aws_region'] = self.read_aws_region() Settings.set('AWS_AUTH_CRED', self.AWS_AUTH_CRED) @@ -36,10 +39,10 @@ def read_input(self): def read_aws_auth_mechanism(self): if self.silent_install: - auth_mechanism = getattr(Settings, 'AWS_ACCESS_KEY', None) + auth_mechanism = getattr(Settings, 'AWS_AUTH_MECHANISM', None) if auth_mechanism in [1, 2, 3]: return auth_mechanism - + self.show_step_inner_error(K.AWS_AUTH_MECHANISM_NOT_SUPPLIED) raise Exception(K.AWS_AUTH_MECHANISM_NOT_SUPPLIED) while True: @@ -61,6 +64,7 @@ def read_aws_access_key(self): if settings_access_key is None or settings_access_key == '': if self.silent_install: + self.show_step_inner_error(K.AWS_ACCESS_KEY_NOT_SUPPLIED) raise Exception(K.AWS_ACCESS_KEY_NOT_SUPPLIED) aws_access_key = input("\n\t%s" % K.AWS_ACCESS_KEY_INPUT) @@ -77,6 +81,7 @@ def read_aws_secret_key(self): settings_secret_key = getattr(Settings, 'AWS_SECRET_KEY', None) if settings_secret_key is None or settings_secret_key == '': if self.silent_install: + self.show_step_inner_error(K.AWS_SECRET_KEY_NOT_SUPPLIED) raise Exception(K.AWS_SECRET_KEY_NOT_SUPPLIED) aws_secret_key = input("\n\t%s" % K.AWS_SECRET_KEY_INPUT) @@ -93,6 +98,7 @@ def read_aws_assume_role_arn(self): settings_assume_role_arn = getattr(Settings, 'AWS_ASSUME_ROLE_ARN', None) if settings_assume_role_arn is None or settings_assume_role_arn == '': if self.silent_install: + self.show_step_inner_error(K.AWS_ASSUME_ROLE_NOT_SUPPLIED) raise Exception(K.AWS_ASSUME_ROLE_NOT_SUPPLIED) assume_role_arn = input("\n\t%s" % K.AWS_ASSUME_ROLE_INPUT) @@ -106,6 +112,7 @@ def read_aws_region(self): settings_region = getattr(Settings, 'AWS_REGION', None) if settings_region is None or settings_region == '': if self.silent_install: + self.show_step_inner_error(K.AWS_REGION_NOT_SUPPLIED) raise Exception(K.AWS_REGION_NOT_SUPPLIED) aws_region = input("\n\t%s" % K.AWS_REGION_INPUT) diff --git a/installer/files/scripts/utils.py b/installer/files/scripts/utils.py index c5e71e8fe..1aab5cf0b 100644 --- a/installer/files/scripts/utils.py +++ b/installer/files/scripts/utils.py @@ -24,9 +24,10 @@ def get_provider_details(provider, provider_json_file): return aws_provider['provider']['aws'] -def generate_temp_credentials(assume_role_arn): +def generate_temp_credentials(assume_role_arn, region_name): response = boto3.client( - 'sts' + 'sts', + region_name=region_name ).assume_role( RoleArn=assume_role_arn, RoleSessionName=str(uuid.uuid4()) @@ -37,34 +38,34 @@ def generate_temp_credentials(assume_role_arn): def prepare_aws_client_with_given_aws_details(service_name, aws_details): auth_data = {} + auth_data['region_name'] = aws_details['region'] if 'access_key' in aws_details: auth_data['aws_access_key_id'] = aws_details['access_key'] auth_data['aws_secret_access_key'] = aws_details['secret_key'] elif 'assume_role' in aws_details: - temp_cred = generate_temp_credentials(aws_details['assume_role']['role_arn']) + temp_cred = generate_temp_credentials(aws_details['assume_role']['role_arn'], auth_data['region_name']) auth_data['aws_access_key_id'] = temp_cred['AccessKeyId'] auth_data['aws_secret_access_key'] = temp_cred['SecretAccessKey'] auth_data['aws_session_token'] = temp_cred['SessionToken'] - auth_data['region_name'] = aws_details['region'] return boto3.client(service_name, **auth_data) def prepare_aws_resource_with_given_aws_details(service_name, aws_details): auth_data = {} + auth_data['region_name'] = aws_details['region'] if 'access_key' in aws_details: auth_data['aws_access_key_id'] = aws_details['access_key'] auth_data['aws_secret_access_key'] = aws_details['secret_key'] elif 'assume_role' in aws_details: - temp_cred = generate_temp_credentials(aws_details['assume_role']['role_arn']) + temp_cred = generate_temp_credentials(aws_details['assume_role']['role_arn'], auth_data['region_name']) auth_data['aws_access_key_id'] = temp_cred['AccessKeyId'] auth_data['aws_secret_access_key'] = temp_cred['SecretAccessKey'] auth_data['aws_session_token'] = temp_cred['SessionToken'] - auth_data['region_name'] = aws_details['region'] return boto3.resource(service_name, **auth_data) From fbb455144b3a7853c819ba9d73853e6ae021edcf Mon Sep 17 00:00:00 2001 From: Kanchana Date: Thu, 31 Oct 2019 17:07:26 +0530 Subject: [PATCH 032/107] Comapliance changes for azure --- .../controller/ComplianceController.java | 2 +- .../api/compliance/domain/AssetCountDTO.java | 167 ++- .../domain/ExemptedAssetByPolicy.java | 30 + .../domain/ExemptedAssetByPolicyData.java | 33 + .../repository/ComplianceRepository.java | 50 +- .../repository/ComplianceRepositoryImpl.java | 398 ++++-- .../compliance/service/ComplianceService.java | 618 ++++---- .../service/ComplianceServiceImpl.java | 549 +++++--- .../service/IssueTrendServiceImpl.java | 2 +- .../service/ProjectionServiceImpl.java | 1251 +++++++++-------- .../service/TaggingServiceImpl.java | 2 +- .../controller/ComplianceControllerTest.java | 4 +- .../service/ComplianceServiceImplTest.java | 18 +- .../service/IssueTrendServiceImplTest.java | 2 +- .../service/ProjectionServiceImplTest.java | 6 +- .../service/TaggingServiceImplTest.java | 2 +- 16 files changed, 1760 insertions(+), 1374 deletions(-) create mode 100644 api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/domain/ExemptedAssetByPolicy.java create mode 100644 api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/domain/ExemptedAssetByPolicyData.java diff --git a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/controller/ComplianceController.java b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/controller/ComplianceController.java index 5d36208a4..3e0c47e69 100644 --- a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/controller/ComplianceController.java +++ b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/controller/ComplianceController.java @@ -224,7 +224,7 @@ public ResponseEntity getPatching(@RequestParam("ag") String assetGroup) } OutputDTO output = null; try { - output = new OutputDTO(complianceService.getPatching(assetGroup, null)); + output = new OutputDTO(complianceService.getPatching(assetGroup, null,null)); } catch (ServiceException e) { return complianceService.formatException(e); } diff --git a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/domain/AssetCountDTO.java b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/domain/AssetCountDTO.java index 34e8945d2..cbc2e365c 100644 --- a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/domain/AssetCountDTO.java +++ b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/domain/AssetCountDTO.java @@ -1,78 +1,89 @@ -/******************************************************************************* - * Copyright 2018 T Mobile, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy - * of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - ******************************************************************************/ -/** - Copyright (C) 2017 T Mobile Inc - All Rights Reserve - Purpose: - Author :santoshi - Modified Date: Nov 5, 2017 - - **/ -package com.tmobile.pacman.api.compliance.domain; -/** - * The Class AssetCountDTO. - */ -public class AssetCountDTO { - - /** The name. */ - private String name; - - /** The type. */ - private String type; - - /** - * Gets the type. - * - * @return the type - */ - public String getType() { - return type; - } - - /** - * Sets the type. - * - * @param type the new type - */ - public void setType(String type) { - this.type = type; - } - - /** - * Gets the name. - * - * @return the name - */ - public String getName() { - return name; - } - - /** - * Sets the name. - * - * @param name the new name - */ - public void setName(String name) { - this.name = name; - } - - /* (non-Javadoc) - * @see java.lang.Object#toString() - */ - @Override - public String toString() { - return "ClassPojo [name = " + name + ",type = " + type + "]"; - } -} +/******************************************************************************* + * Copyright 2018 T Mobile, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +/** + Copyright (C) 2017 T Mobile Inc - All Rights Reserve + Purpose: + Author :SGorle + Modified Date: Nov 5, 2017 + + **/ +package com.tmobile.pacman.api.compliance.domain; +/** + * The Class AssetCountDTO. + */ +public class AssetCountDTO { + + /** The name. */ + private String name; + + /** The type. */ + private String type; + + /** The provider **/ + private String provider; + + /** + * Gets the type. + * + * @return the type + */ + public String getType() { + return type; + } + + /** + * Sets the type. + * + * @param type the new type + */ + public void setType(String type) { + this.type = type; + } + + /** + * Gets the name. + * + * @return the name + */ + public String getName() { + return name; + } + + /** + * Sets the name. + * + * @param name the new name + */ + public void setName(String name) { + this.name = name; + } + + public String getProvider() { + return provider; + } + + public void setProvider(String provider) { + this.provider = provider; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "ClassPojo [name = " + name + ",type = " + type + ",provider = " + provider + "]"; + } +} diff --git a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/domain/ExemptedAssetByPolicy.java b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/domain/ExemptedAssetByPolicy.java new file mode 100644 index 000000000..ab2d1de7b --- /dev/null +++ b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/domain/ExemptedAssetByPolicy.java @@ -0,0 +1,30 @@ +package com.tmobile.pacman.api.compliance.domain; + +public class ExemptedAssetByPolicy { + + private String message; + + private ExemptedAssetByPolicyData data; + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public ExemptedAssetByPolicyData getData() { + return data; + } + + public void setData(ExemptedAssetByPolicyData data) { + this.data = data; + } + + @Override + public String toString() { + return "ExemptedAssetByPolicy [message=" + message + ", data=" + data + + "]"; + } +} diff --git a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/domain/ExemptedAssetByPolicyData.java b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/domain/ExemptedAssetByPolicyData.java new file mode 100644 index 000000000..d1fe634dc --- /dev/null +++ b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/domain/ExemptedAssetByPolicyData.java @@ -0,0 +1,33 @@ +package com.tmobile.pacman.api.compliance.domain; + +import java.util.List; +import java.util.Map; + +public class ExemptedAssetByPolicyData { + + private String totalExempted; + + private List> exempted; + + public String getTotalExempted() { + return totalExempted; + } + + public void setTotalExempted(String totalExempted) { + this.totalExempted = totalExempted; + } + + public List> getExempted() { + return exempted; + } + + public void setExempted(List> exempted) { + this.exempted = exempted; + } + + @Override + public String toString() { + return "ExemptedAssetByPolicyData [totalExempted=" + totalExempted + + ", exempted=" + exempted + "]"; + } +} diff --git a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/repository/ComplianceRepository.java b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/repository/ComplianceRepository.java index bc08d1298..dfe9fef5b 100644 --- a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/repository/ComplianceRepository.java +++ b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/repository/ComplianceRepository.java @@ -435,9 +435,11 @@ public Map getAllApplicationsAssetCountForTargetType( * * @param assetGroup the asset group * @param domain the domain + * @param application the application + * @param type the type * @return Map */ - public Map getTotalAssetCount(String assetGroup, String domain); + public Map getTotalAssetCount(String assetGroup, String domain, String application,String type); /** * Gets true if it updates the kernel version for the given instanceId @@ -479,7 +481,7 @@ public Map getRuleCategoryWeightagefromDB(String domain) * @return Map * @throws DataException the data exception */ - public Map getTaggingByAG(String assetGroup,String ttypes) + public Map getTaggingByAG(String assetGroup,String ttypes,String application) throws DataException; /** @@ -522,17 +524,19 @@ public Map getPatchableAssetsByApplication(String assetGroup, public List> getRuleIdWithDisplayNameWithRuleCategoryQuery( String targetTypes, String ruleCategory) throws DataException; + /** - * This method applicable for ec2 and onpremserver target types. If method - * receives,asset group and targettype(ec2/onpremserver) as request - * parameters, then it gives the asset count of that target type. + * Gets the patchabe assets count. * * @param assetGroup the asset group * @param targetType the target type - * @return Long + * @param application the application + * @param environment the environment + * @param searchText the search text + * @return the patchabe assets count * @throws DataException the data exception */ - public Long getPatchabeAssetsCount(String assetGroup, String targetType) + public Long getPatchabeAssetsCount(String assetGroup, String targetType,String application,String environment,String searchText) throws DataException; /** @@ -543,10 +547,11 @@ public Long getPatchabeAssetsCount(String assetGroup, String targetType) * * @param assetGroup the asset group * @param targetType the target type + * @param application * @return Long * @throws DataException the data exception */ - public Long getUnpatchedAssetsCount(String assetGroup, String targetType) + public Long getUnpatchedAssetsCount(String assetGroup, String targetType, String application) throws DataException; /** @@ -591,7 +596,7 @@ public String fetchSystemConfiguration(final String keyname) * @return the instance count for qualys * @throws DataException the data exception */ - public Long getInstanceCountForQualys(String assetGroup,String apiType,String application,String enivironment) + public Long getInstanceCountForQualys(String assetGroup,String apiType,String application,String enivironment,String resourceType) throws DataException; /** @@ -604,7 +609,7 @@ public Long getInstanceCountForQualys(String assetGroup,String apiType,String ap * @return the instance count for qualys by apps or env * @throws DataException the data exception */ - public Map getInstanceCountForQualysByAppsOrEnv(String assetGroup,String apiType,String application,String enivironment) + public Map getInstanceCountForQualysByAppsOrEnv(String assetGroup,String apiType,String application,String enivironment,String targetType) throws DataException; /** @@ -634,4 +639,29 @@ public Map getInstanceCountForQualysByAppsOrEnv(String assetGroup, * @return the total asset count by environment */ public Map getTotalAssetCountByEnvironment(String assetGroup, String application,String targetType); + + /** + * Gets the datasource for the target type. + * + * @param assetGroup + * the asset group + * @param domain + * the domain + * @param targetType + * the targetType + * @return String + */ + public List> getDataSourceForTargetTypeForAG(String assetGroup, String domain, String targetType); + + /** + * Gets the exempted assets count by rule. + * + * @param assetGroup the asset group + * @return the exempted assets count by rule + * @throws DataException the data exception + */ + public Map getExemptedAssetsCountByRule(String assetGroup, String application,String type) + throws DataException; + + } diff --git a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/repository/ComplianceRepositoryImpl.java b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/repository/ComplianceRepositoryImpl.java index c9deeffa9..3cc80b447 100644 --- a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/repository/ComplianceRepositoryImpl.java +++ b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/repository/ComplianceRepositoryImpl.java @@ -87,6 +87,8 @@ import com.tmobile.pacman.api.compliance.domain.AssetCountData; import com.tmobile.pacman.api.compliance.domain.AssetCountEnvCount; import com.tmobile.pacman.api.compliance.domain.Compare; +import com.tmobile.pacman.api.compliance.domain.ExemptedAssetByPolicy; +import com.tmobile.pacman.api.compliance.domain.ExemptedAssetByPolicyData; import com.tmobile.pacman.api.compliance.domain.IssueExceptionResponse; import com.tmobile.pacman.api.compliance.domain.IssueResponse; import com.tmobile.pacman.api.compliance.domain.IssuesException; @@ -443,7 +445,7 @@ public Map getTagging(String assetGroup, String targetType) throws ruleIdWithTargetTypeQuery = "SELECT A.targetType FROM cf_RuleInstance A, cf_Policy B WHERE A.policyId = B.policyId AND A.status = 'ENABLED' AND B.policyId = 'PacMan_TaggingRule_version-1'"; ruleIdwithTargetType = rdsepository.getDataFromPacman(ruleIdWithTargetTypeQuery); if (Strings.isNullOrEmpty(targetType)) { - assetCount = assetServiceClient.getTotalAssetsCount(assetGroup, targetType, null); + assetCount = assetServiceClient.getTotalAssetsCount(assetGroup, targetType, null,null); data = assetCount.getData(); assetcountCount = data.getAssetcount(); @@ -591,7 +593,7 @@ public List> getRecommendations(String assetGroup, String ta */ public Long getTotalAssetCountForAnytargetType(String assetGroup, String targetType) { - AssetCount totalAssets = assetServiceClient.getTotalAssetsCount(assetGroup, targetType, null); + AssetCount totalAssets = assetServiceClient.getTotalAssetsCount(assetGroup, targetType, null,null); AssetCountData data = totalAssets.getData(); AssetCountByAppEnvDTO[] assetcount = data.getAssetcount(); Long totalAssetsCount = 0l; @@ -603,24 +605,6 @@ public Long getTotalAssetCountForAnytargetType(String assetGroup, String targetT return totalAssetsCount; } - /* - * (non-Javadoc) - * - * @see com.tmobile.pacman.api.compliance.repository.ComplianceRepository# - * getTotalAssetCount(java.lang.String, java.lang.String) - */ - public Map getTotalAssetCount(String assetGroup, String domain) { - - AssetCount totalAssets = assetServiceClient.getTotalAssetsCount(assetGroup, null, domain); - AssetCountData data = totalAssets.getData(); - AssetCountByAppEnvDTO[] assetcount = data.getAssetcount(); - Map assetCountByType = new HashMap<>(); - for (AssetCountByAppEnvDTO assetCount_Count : assetcount) { - assetCountByType.put(assetCount_Count.getType(), Long.parseLong(assetCount_Count.getCount())); - } - return assetCountByType; - } - /** * Gets the resource details from ES. * @@ -1676,54 +1660,59 @@ public Map getRuleCategoryWeightagefromDB(String domain) throws } /* - * (non-Javadoc) - * - * @see com.tmobile.pacman.api.compliance.repository.ComplianceRepository# - * getTaggingByAG(java.lang.String) - */ - @SuppressWarnings("rawtypes") - public Map getTaggingByAG(String assetGroup,String targetTypes) throws DataException { - List targetTypeList = Arrays.asList(targetTypes.split("\\s*,\\s*")); - - Gson gson = new GsonBuilder().create(); - String responseDetails = null; - StringBuilder urlToQueryBuffer = new StringBuilder(esUrl).append("/").append(assetGroup).append("/") - .append(SEARCH); - StringBuilder requestBody = null; - List tagsList = new ArrayList<>(Arrays.asList(mandatoryTags.split(","))); - - String body = "{\"size\":0,\"query\":{\"bool\":{\"must\":[{\"term\":{\"type.keyword\":{\"value\":\"issue\"}}},{\"term\":{\"policyId.keyword\":{\"value\":\"PacMan_TaggingRule_version-1\"}}},{\"term\":{\"issueStatus.keyword\":{\"value\":\"open\"}}}"; - - body = body + "]"; - if (!tagsList.isEmpty()) { - body = body + ",\"should\":["; - - for (String tag : tagsList) { - body = body + "{\"match_phrase_prefix\":{\"missingTags\":\"" + tag + "\"}},"; - } - body = body.substring(0, body.length() - 1); - body = body + "]"; - body = body + ",\"minimum_should_match\":1"; - } - body = body + "}},\"aggs\":{\"name\":{\"terms\":{\"field\":\"targetType.keyword\",\"size\":"+targetTypeList.size()+"}}}}"; - requestBody = new StringBuilder(body); - try { - responseDetails = PacHttpUtils.doHttpPost(urlToQueryBuffer.toString(), requestBody.toString()); - } catch (Exception e) { - throw new DataException(e); - } - Map response = (Map) gson.fromJson(responseDetails, Map.class); - Map aggregations = (Map) response.get(AGGREGATIONS); - Map name = (Map) aggregations.get("name"); - List> buckets = (List>) name.get(BUCKETS); - - return buckets - .parallelStream() - .filter(buket -> buket.get("doc_count") != null) - .collect( - Collectors.toMap(buket -> buket.get("key").toString(), buket -> buket.get("doc_count"), ( - oldValue, newValue) -> newValue)); - } + * (non-Javadoc) + * + * @see com.tmobile.pacman.api.compliance.repository.ComplianceRepository# + * getTaggingByAG(java.lang.String) + */ + @SuppressWarnings("rawtypes") + public Map getTaggingByAG(String assetGroup, String targetTypes, String application) + throws DataException { + List targetTypeList = Arrays.asList(targetTypes.split("\\s*,\\s*")); + + Gson gson = new GsonBuilder().create(); + String responseDetails = null; + StringBuilder urlToQueryBuffer = new StringBuilder(esUrl).append("/").append(assetGroup).append("/") + .append(SEARCH); + StringBuilder requestBody = null; + List tagsList = new ArrayList<>(Arrays.asList(mandatoryTags.split(","))); + + String body = "{\"size\":0,\"query\":{\"bool\":{\"must\":[{\"term\":{\"type.keyword\":{\"value\":\"issue\"}}},{\"term\":{\"policyId.keyword\":{\"value\":\"PacMan_TaggingRule_version-1\"}}},{\"term\":{\"issueStatus.keyword\":{\"value\":\"open\"}}}"; + + // Added resourceType to the Query + String targetTypesTerms = targetTypes.replaceAll("'", "\""); + body = body + ",{\"terms\":{\"targetType.keyword\":[" + targetTypesTerms + "]}}"; + if (application != null) { + body = body + ",{\"match\":{\"tags.Application.keyword\":\"" + application + "\"}}"; + } + body = body + "]"; + if (!tagsList.isEmpty()) { + body = body + ",\"should\":["; + + for (String tag : tagsList) { + body = body + "{\"match_phrase_prefix\":{\"missingTags\":\"" + tag + "\"}},"; + } + body = body.substring(0, body.length() - 1); + body = body + "]"; + body = body + ",\"minimum_should_match\":1"; + } + body = body + "}},\"aggs\":{\"name\":{\"terms\":{\"field\":\"targetType.keyword\",\"size\":" + + targetTypeList.size() + "}}}}"; + requestBody = new StringBuilder(body); + try { + responseDetails = PacHttpUtils.doHttpPost(urlToQueryBuffer.toString(), requestBody.toString()); + } catch (Exception e) { + throw new DataException(e); + } + Map response = (Map) gson.fromJson(responseDetails, Map.class); + Map aggregations = (Map) response.get(AGGREGATIONS); + Map name = (Map) aggregations.get("name"); + List> buckets = (List>) name.get(BUCKETS); + + return buckets.parallelStream().filter(buket -> buket.get("doc_count") != null) + .collect(Collectors.toMap(buket -> buket.get("key").toString(), buket -> buket.get("doc_count"), + (oldValue, newValue) -> newValue)); + } /* * (non-Javadoc) @@ -1857,26 +1846,30 @@ public Long getPatchabeAssetsCount(String assetGroup, String targetType) throws } } - /* - * (non-Javadoc) - * - * @see com.tmobile.pacman.api.compliance.repository.ComplianceRepository# - * getUnpatchedAssetsCount(java.lang.String, java.lang.String) - */ - public Long getUnpatchedAssetsCount(String assetGroup, String targetType) throws DataException { - String ruleId = null; - if (EC2.equalsIgnoreCase(targetType)) { - ruleId = EC2_KERNEL_COMPLIANCE_RULE; - } - Map mustFilter = formatUnpatchedMustFilter(targetType, ruleId); - String type = ISSUE_UNDERSCORE + targetType; - try { - return elasticSearchRepository.getTotalDocumentCountForIndexAndType(assetGroup, type, mustFilter, null, - null, null, null); - } catch (Exception e) { - throw new DataException("" + e); - } - } + /* + * (non-Javadoc) + * + * @see com.tmobile.pacman.api.compliance.repository.ComplianceRepository# + * getUnpatchedAssetsCount(java.lang.String, java.lang.String) + */ + public Long getUnpatchedAssetsCount(String assetGroup, String targetType, String application) throws DataException { + String policyId = null; + if (EC2.equalsIgnoreCase(targetType) || VIRTUALMACHINE.equalsIgnoreCase(targetType)) { + policyId = CLOUD_KERNEL_COMPLIANCE_POLICY; + } + + Map mustFilter = formatUnpatchedMustFilter(targetType, policyId); + if (StringUtils.isNotBlank(application)) { + mustFilter.put(TAGS_APPS, application); + } + String type = ISSUE_UNDERSCORE + targetType; + try { + return elasticSearchRepository.getTotalDocumentCountForIndexAndType(assetGroup, type, mustFilter, null, + null, null, null); + } catch (Exception e) { + throw new DataException("" + e); + } + } /* * (non-Javadoc) @@ -1950,56 +1943,85 @@ public Map getRuleCategoryPercentage(Map ruleCateg return ruleCategoryPercentage; } - private JsonObject getResopnse(String assetGroup,String apiType,String application,String environment) throws DataException{ - StringBuilder urlToQuery = formatURL(assetGroup, EC2,apiType); - String responseJson = ""; - try { - responseJson = PacHttpUtils.doHttpPost(urlToQuery.toString(), - getQueryForQualys(apiType,application,environment).toString()); - } catch (Exception e) { - logger.error(e.toString()); - throw new DataException(e.getMessage()); - } - JsonParser jsonParser = new JsonParser(); - return (JsonObject) jsonParser.parse(responseJson); - } - - public Long getInstanceCountForQualys(String assetGroup,String apiType,String application,String environment) - throws DataException { - return getResopnse(assetGroup, apiType, application, environment).get(COUNT).getAsLong(); - } - - public Map getInstanceCountForQualysByAppsOrEnv(String assetGroup,String apiType,String application,String environment ) - throws DataException { - Map assetWithTagsMap = new HashMap<>(); - JsonObject resultJson = getResopnse(assetGroup, apiType, application, environment); - - JsonObject aggs = (JsonObject) resultJson.get(AGGREGATIONS); - JsonObject name = (JsonObject) aggs.get("NAME"); - JsonArray buckets = name.get(BUCKETS).getAsJsonArray(); - // convert Json Array to Map object - for (JsonElement bucket : buckets) { - assetWithTagsMap.put(bucket.getAsJsonObject().get("key").getAsString(), bucket.getAsJsonObject() - .get(DOC_COUNT).getAsLong()); - } - - return assetWithTagsMap; - } - - private StringBuilder getQueryForQualys(String apiType,String application,String environment){ - StringBuilder requestBody = new StringBuilder( - "{\"query\":{\"bool\":{\"must\":[{\"match\":{\"latest\":\"true\"}},{\"match\":{\"statename\":\"running\"}}],\"should\":[{\"script\":{\"script\":\"LocalDate.parse(doc['firstdiscoveredon.keyword'].value.substring(0,10)).isBefore(LocalDate.from(Instant.ofEpochMilli(new Date().getTime()).atZone(ZoneId.systemDefault())).minusDays(7))\"}},{\"has_child\":{\"type\":\"qualysinfo\",\"query\":{\"match\":{\"latest\":\"true\"}}}}],\"minimum_should_match\":1}}"); - if ("noncompliancepolicy".equals(apiType)) { - requestBody - .append("}"); - }else if ("policydetailsbyapplication".equals(apiType)) { - requestBody - .append(",\"aggs\":{\"NAME\":{\"terms\":{\"field\":\"tags.Application.keyword\",\"size\":10000}}}}"); - } else if ("policydetailsbyenvironment".equals(apiType)) { - requestBody = new StringBuilder("{\"query\":{\"bool\":{\"must\":[{\"match\":{\"latest\":\"true\"}},{\"match\":{\"statename\":\"running\"}},{\"match\":{\"tags.Application.keyword\":\""+application+"\"}},{\"match\":{\"tags.Environment.keyword\":\""+environment+"\"}}],\"should\":[{\"script\":{\"script\":\"LocalDate.parse(doc['firstdiscoveredon.keyword'].value.substring(0,10)).isBefore(LocalDate.from(Instant.ofEpochMilli(new Date().getTime()).atZone(ZoneId.systemDefault())).minusDays(7))\"}},{\"has_child\":{\"type\":\"qualysinfo\",\"query\":{\"match\":{\"latest\":\"true\"}}}}],\"minimum_should_match\":1}}}"); - } - return requestBody; - } + private JsonObject getResopnse(String assetGroup, String apiType, String application, String environment, + String resourceType) throws DataException { + StringBuilder urlToQuery = formatURL(assetGroup, resourceType, apiType); + String responseJson = ""; + try { + responseJson = PacHttpUtils.doHttpPost(urlToQuery.toString(), + getQueryForQualys(apiType, application, environment, resourceType).toString()); + } catch (Exception e) { + logger.error(e.toString()); + throw new DataException(e.getMessage()); + } + JsonParser jsonParser = new JsonParser(); + return (JsonObject) jsonParser.parse(responseJson); + } + + + public Long getInstanceCountForQualys(String assetGroup, String apiType, String application, String environment, + String resourceType) throws DataException { + return getResopnse(assetGroup, apiType, application, environment, resourceType).get(COUNT).getAsLong(); + } + + public Map getInstanceCountForQualysByAppsOrEnv(String assetGroup, String apiType, String application, + String environment, String resourceType) throws DataException { + Map assetWithTagsMap = new HashMap<>(); + JsonObject resultJson = getResopnse(assetGroup, apiType, application, environment, resourceType); + + JsonObject aggs = (JsonObject) resultJson.get(AGGREGATIONS); + JsonObject name = (JsonObject) aggs.get("NAME"); + JsonArray buckets = name.get(BUCKETS).getAsJsonArray(); + // convert Json Array to Map object + for (JsonElement bucket : buckets) { + assetWithTagsMap.put(bucket.getAsJsonObject().get("key").getAsString(), + bucket.getAsJsonObject().get(DOC_COUNT).getAsLong()); + } + + return assetWithTagsMap; + } + + private StringBuilder getQueryForQualys(String apiType, String application, String environment, + String resourceType) { + StringBuilder requestBody = new StringBuilder(); + + if (EC2.equals(resourceType)) { + requestBody = new StringBuilder( + "{\"query\":{\"bool\":{\"must\":[{\"match\":{\"latest\":\"true\"}},{\"match\":{\"statename.keyword\":\"running\"}}"); + } else if (VIRTUALMACHINE.equals(resourceType)) { + requestBody = new StringBuilder( + "{\"query\":{\"bool\":{\"must\":[{\"match\":{\"latest\":\"true\"}},{\"match\":{\"status.keyword\":\"running\"}}"); + } + if (StringUtils.isNotBlank(application)) { + requestBody.append(",{\"match\":{\"tags.Application.keyword\":\"" + application + "\"}}"); + } + if (StringUtils.isNotBlank(environment)) { + requestBody.append(",{\"match\":{\"tags.Environment.keyword\":\"" + environment + "\"}}"); + } + requestBody.append( + "],\"should\":[{\"script\":{\"script\":\"LocalDate.parse(doc['firstdiscoveredon.keyword'].value.substring(0,10)).isBefore(LocalDate.from(Instant.ofEpochMilli(new Date().getTime()).atZone(ZoneId.systemDefault())).minusDays(7))\"}},{\"has_child\":{\"type\":\"qualysinfo\",\"query\":{\"match\":{\"latest\":\"true\"}}}}],\"minimum_should_match\":1}}"); + if ("noncompliancepolicy".equals(apiType)) { + + requestBody.append("}"); + } else if ("policydetailsbyapplication".equals(apiType)) { + requestBody.append( + ",\"aggs\":{\"NAME\":{\"terms\":{\"field\":\"tags.Application.keyword\",\"size\":10000}}}}"); + } else if ("policydetailsbyenvironment".equals(apiType)) { + + if (EC2.equals(resourceType)) { + requestBody = new StringBuilder( + "{\"query\":{\"bool\":{\"must\":[{\"match\":{\"latest\":\"true\"}},{\"match\":{\"statename.keyword\":\"running\"}},{\"match\":{\"tags.Application.keyword\":\"" + + application + "\"}},{\"match\":{\"tags.Environment.keyword\":\"" + environment + + "\"}}],\"should\":[{\"script\":{\"script\":\"LocalDate.parse(doc['firstdiscoveredon.keyword'].value.substring(0,10)).isBefore(LocalDate.from(Instant.ofEpochMilli(new Date().getTime()).atZone(ZoneId.systemDefault())).minusDays(7))\"}},{\"has_child\":{\"type\":\"qualysinfo\",\"query\":{\"match\":{\"latest\":\"true\"}}}}],\"minimum_should_match\":1}}}"); + } else if (VIRTUALMACHINE.equals(resourceType)) { + requestBody = new StringBuilder( + "{\"query\":{\"bool\":{\"must\":[{\"match\":{\"latest\":\"true\"}},{\"match\":{\"status.keyword\":\"running\"}},{\"match\":{\"tags.Application.keyword\":\"" + + application + "\"}},{\"match\":{\"tags.Environment.keyword\":\"" + environment + + "\"}}],\"should\":[{\"script\":{\"script\":\"LocalDate.parse(doc['firstdiscoveredon.keyword'].value.substring(0,10)).isBefore(LocalDate.from(Instant.ofEpochMilli(new Date().getTime()).atZone(ZoneId.systemDefault())).minusDays(7))\"}},{\"has_child\":{\"type\":\"qualysinfo\",\"query\":{\"match\":{\"latest\":\"true\"}}}}],\"minimum_should_match\":1}}}"); + } + } + return requestBody; + } private StringBuilder formatURL(String assetGroup, String resourcetype,String apiType) { StringBuilder urlToQuery = new StringBuilder(esUrl).append("/").append( @@ -2422,4 +2444,104 @@ public Map getTotalAssetCountByEnvironment(String assetGroup, Strin } return assetCountByEnv; } + + /** + * Function for getting dataSource and target type of an asset group and domain + * + * @see com.tmobile.pacman.api.compliance.repository.ComplianceRepository# + * getDataSourceForTargetTypeForAG(java.lang.String, java.lang.String) + */ + public List> getDataSourceForTargetTypeForAG(String assetGroup, String domain, + String targetType) { + + List> dataSourceForTargetType = new ArrayList>(); + AssetApi assetApi = assetServiceClient.getTargetTypeList(assetGroup, domain); + AssetApiData data = assetApi.getData(); + AssetCountDTO[] targetTypes = data.getTargettypes(); + for (AssetCountDTO name : targetTypes) { + Map datasourceTargetType = new HashMap(); + if (!Strings.isNullOrEmpty(name.getType())) { + datasourceTargetType.put(TYPE, name.getType()); + datasourceTargetType.put(PROVIDER, name.getProvider()); + if (targetType == null) { + dataSourceForTargetType.add(datasourceTargetType); + } else { + if (datasourceTargetType.get(TYPE).equals(targetType)) { + dataSourceForTargetType.add(datasourceTargetType); + } + } + } + } + return dataSourceForTargetType; + } + + /* + * (non-Javadoc) + * + * @see com.tmobile.pacman.api.compliance.repository.ComplianceRepository# + * getTotalAssetCount(java.lang.String, java.lang.String) + */ + public Map getTotalAssetCount(String assetGroup, String domain, String application, String type) { + AssetCount totalAssets = assetServiceClient.getTotalAssetsCount(assetGroup, type, domain, application); + AssetCountData data = totalAssets.getData(); + AssetCountByAppEnvDTO[] assetcount = data.getAssetcount(); + Map assetCountByType = new HashMap<>(); + for (AssetCountByAppEnvDTO assetCount_Count : assetcount) { + assetCountByType.put(assetCount_Count.getType(), Long.parseLong(assetCount_Count.getCount())); + } + return assetCountByType; + } + + @Override + public Map getExemptedAssetsCountByRule(String assetGroup, String application, String type) + throws DataException { + + Map exemptedAssetsCount = new HashMap<>(); + ExemptedAssetByPolicy exemptedAssetByPolicy = assetServiceClient.getTotalAssetsExemptedByPolicy(assetGroup, + application, type, null); + ExemptedAssetByPolicyData data = exemptedAssetByPolicy.getData(); + for (Map exempted : data.getExempted()) { + exemptedAssetsCount.put(exempted.get("ruleid").toString(), + Integer.parseInt(exempted.get(COUNT).toString())); + } + return exemptedAssetsCount; + } + + /* + * (non-Javadoc) + * + * @see com.tmobile.pacman.api.compliance.repository.ComplianceRepository# + * getPatchabeAssetsCount(java.lang.String, java.lang.String) + */ + public Long getPatchabeAssetsCount(String assetGroup, String targetType, String application, String environment, + String searchText) throws DataException { + Map mustFilter = new HashMap<>(); + Map mustNotFilter = null; + + if (!StringUtils.isEmpty(application)) { + mustFilter.put(CommonUtils.convertAttributetoKeyword(TAGS_APPLICATION), application); + } + if (!StringUtils.isEmpty(environment)) { + mustFilter.put(CommonUtils.convertAttributetoKeyword(TAGS_ENVIRONMENT), environment); + } + + mustFilter.put(LATEST, true); + if (EC2.equalsIgnoreCase(targetType)) { + mustFilter.put(CommonUtils.convertAttributetoKeyword(STATE_NAME), RUNNING); + mustNotFilter = new HashMap<>(); + mustNotFilter.put(CommonUtils.convertAttributetoKeyword(PLATFORM), WINDOWS); + } else if (VIRTUALMACHINE.equalsIgnoreCase(targetType)) { + mustFilter.put(CommonUtils.convertAttributetoKeyword("status"), RUNNING); + mustNotFilter = new HashMap<>(); + mustNotFilter.put(CommonUtils.convertAttributetoKeyword("osType"), AZURE_WINDOWS); + } + try { + return elasticSearchRepository.getTotalDocumentCountForIndexAndType(assetGroup, targetType, mustFilter, + mustNotFilter, null, searchText, null); + } catch (Exception e) { + throw new DataException(e); + } + } + + } diff --git a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/ComplianceService.java b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/ComplianceService.java index 54fc7d066..be7410170 100644 --- a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/ComplianceService.java +++ b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/ComplianceService.java @@ -1,310 +1,308 @@ -/******************************************************************************* - * Copyright 2018 T Mobile, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy - * of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - ******************************************************************************/ -package com.tmobile.pacman.api.compliance.service; - -import java.util.List; -import java.util.Map; - -import org.springframework.http.ResponseEntity; - -import com.tmobile.pacman.api.commons.exception.ServiceException; -import com.tmobile.pacman.api.compliance.domain.IssueExceptionResponse; -import com.tmobile.pacman.api.compliance.domain.IssueResponse; -import com.tmobile.pacman.api.compliance.domain.IssuesException; -import com.tmobile.pacman.api.compliance.domain.KernelVersion; -import com.tmobile.pacman.api.compliance.domain.PolicyViolationDetails; -import com.tmobile.pacman.api.compliance.domain.Request; -import com.tmobile.pacman.api.compliance.domain.ResponseWithOrder; -import com.tmobile.pacman.api.compliance.domain.RuleDetails; - -// TODO: Auto-generated Javadoc -/** - * The Interface ComplianceService. - */ -public interface ComplianceService { - - /** - * Gets the issues details based on name of the domain passed. - * - * @param request the request - * @return ResponseWithOrder - * @throws ServiceException the service exception - */ - public ResponseWithOrder getIssues(Request request) throws ServiceException; - - /** - * Gets Issue count based on name of the asset group/ruleId/domain passed. - * - * @param assetGroup the asset group - * @param ruleId the rule id - * @param domain the domain - * @return long - * @throws ServiceException the service exception - */ - public long getIssuesCount(String assetGroup, String ruleId, String domain) throws ServiceException; - - /** - * Gets Compliance distribution by rule category and severity. - * - * @param assetGroup the asset group - * @param domain the domain - * @return Map - * @throws ServiceException the service exception - */ - public Map getDistribution(String assetGroup, String domain) throws ServiceException; - - /** - * Gets Tagging compliance details based on name of name of the asset group/tagettype passed. - * - * @param assetGroup the asset group - * @param targetType the target type - * @return Map - * @throws ServiceException the service exception - */ - - public Map getTagging(String assetGroup, String targetType) throws ServiceException; - - /** - * Gets the count of expiredCertificates with in 60days and - * totalCertificates for given assetGroup. - * - * @param assetGroup the asset group - * @return Map - * @throws ServiceException the service exception - */ - public Map getCertificates(String assetGroup) throws ServiceException; - - /** - * Gets the patching. - * - * @param assetGroup - * name of the asset group - * @param targetType - * the target type - * @return Method description: asssetGroup is mandatory. Method returns - * count of totalPached/toalUnpatched/TotalInstances for given - * assetGroup. - * @throws ServiceException - * the service exception - */ - public Map getPatching(String assetGroup, String targetType) throws ServiceException; - - /** - * If method receives - * assetGroup as request parameter, method returns list of all the issue - * counts which are related to recommendations rules from the ES for the - * given assetGroup with all the targetTypes.If method receives both - * assetGroup and targetType as request parameter,method returns list of all - * the issue counts which are related to recommendations rules from the ES - * for the given targetType & assetGroup. - * - * @param assetGroup the asset group - * @param targetType the target type - * @return List> - * @throws ServiceException the service exception - */ - public List> getRecommendations(String assetGroup, String targetType) throws ServiceException; - - /** - * Gets list of issue audit log details for the size you have given. - * - * @param annotationId the annotation id - * @param targetType the target type - * @param from the from - * @param size the size - * @param searchText the search text - * @return ResponseWithOrder - * @throws ServiceException the service exception - */ - public ResponseWithOrder getIssueAuditLog(String annotationId, String targetType, int from, int size, - String searchText) throws ServiceException; - - /** - * Gets the resource details. - * - * @param assetGroup the asset group - * @param resourceId the resource id - * @return List> - * @throws ServiceException the service exception - */ - public List> getResourceDetails(String assetGroup, String resourceId) throws ServiceException; - - /** - * Returns true if its successfully closes all the issues in ES - * for that ruleId else false. - * - * @param ruleDetails the rule details - * @return Map - */ - - public Map closeIssuesByRule(RuleDetails ruleDetails); - - /** - * Gets the list of all the rules compliance mapped to that domain. - * - * @param request the request - * @return ResponseWithOrder - * @throws ServiceException the service exception - */ - public ResponseWithOrder getRulecompliance(Request request) throws ServiceException; - - /** - * Gets the rule details by application.SearchText is used to match any text - * you are looking for. - * - * @param assetGroup the asset group - * @param ruleId the rule id - * @param searchText the search text - * @return List> - * @throws ServiceException the service exception - */ - public List> getRuleDetailsbyApplication(String assetGroup, String ruleId, String searchText) - throws ServiceException; - - /** - * Gets the rule details by environment.SearchText is used to match any - * text you are looking for. - * - * @param assetGroup the asset group - * @param ruleId the rule id - * @param application the application - * @param searchText the search text - * @return List> - * @throws ServiceException the service exception - */ - public List> getRuleDetailsbyEnvironment(String assetGroup, String ruleId, String application, - String searchText) throws ServiceException; - - /** - * Gets the rule description and other details. - * - * @param ruleId the rule id - * @return Map - * @throws ServiceException the service exception - */ - public Map getRuleDescription(String ruleId) throws ServiceException; - - /** - * Gets the kernel version of an instance id from DB where the kernel version updated by web service. - * - * @param instanceId the instance id - * @return Map - * @throws ServiceException the service exception - */ - public Map getKernelComplianceByInstanceIdFromDb(String instanceId) throws ServiceException; - - /** - * Returns true if it updates the - * kernel version for the given instanceId successfully. - * - * @param kernelVersion the kernel version - * @return Map - */ - public Map updateKernelVersion(final KernelVersion kernelVersion); - - /** - * Gets the overall compliance by domain.Over all compliance is calculated by its severity and rule category weightages. - * - * @param assetGroup the asset group - * @param domain the domain - * @return Map - * @throws ServiceException the service exception - */ - public Map getOverallComplianceByDomain(String assetGroup, String domain) throws ServiceException; - - /** - * Gets the list of targetTypes for given asset group and domain - * based on project target types configurations. - * - * @param assetgroup the assetgroup - * @param domain the domain - * @return List - * @throws ServiceException the service exception - */ - public List getResourceType(String assetgroup, String domain)throws ServiceException; - - /** - * Gets the rule severity and category details. - * - * @param ruleDetails the rule details - * @return List> - * @throws ServiceException the service exception - */ - public List> getRuleSevCatDetails(List> ruleDetails) throws ServiceException; - - /** - * Gets the policy violation details by issue id. - * - * @param assetgroup the assetgroup - * @param issueId the issue id - * @return PolicyViolationDetails - * @throws ServiceException the service exception - */ - public PolicyViolationDetails getPolicyViolationDetailsByIssueId(String assetgroup, String issueId) - throws ServiceException; - - /** - * Adds the issue exception. - * - * @param issueException the issue exception - * @return Boolean - * @throws ServiceException the service exception - */ - public Boolean addIssueException(IssueResponse issueException) throws ServiceException; - - /** - * Revoke issue exception. - * - * @param issueId the issue id - * @return boolean - * @throws ServiceException the service exception - */ - public Boolean revokeIssueException(String issueId) throws ServiceException; - - /** - * Generic method to throw the service exception. - * - * @param e the e - * @return ResponseEntity - */ - public ResponseEntity formatException(ServiceException e); - - /** - * method to get current kernel versions. - * - * @return Map - */ - public Map getCurrentKernelVersions(); - - /** - * Adds the multiple issue exception. - * - * @param issuesException the issues exception - * @return the issue exception response - * @throws ServiceException the service exception - */ - public IssueExceptionResponse addMultipleIssueException(IssuesException issuesException) throws ServiceException; - - /** - * Revoke multiple issue exception. - * - * @param issueIds the issue ids - * @return the issue exception response - * @throws ServiceException the service exception - */ - public IssueExceptionResponse revokeMultipleIssueException(List issueIds) throws ServiceException; - -} +/******************************************************************************* + * Copyright 2018 T Mobile, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +package com.tmobile.pacman.api.compliance.service; + +import java.util.List; +import java.util.Map; + +import org.springframework.http.ResponseEntity; + +import com.tmobile.pacman.api.commons.exception.ServiceException; +import com.tmobile.pacman.api.compliance.domain.IssueExceptionResponse; +import com.tmobile.pacman.api.compliance.domain.IssueResponse; +import com.tmobile.pacman.api.compliance.domain.IssuesException; +import com.tmobile.pacman.api.compliance.domain.KernelVersion; +import com.tmobile.pacman.api.compliance.domain.PolicyViolationDetails; +import com.tmobile.pacman.api.compliance.domain.Request; +import com.tmobile.pacman.api.compliance.domain.ResponseWithOrder; +import com.tmobile.pacman.api.compliance.domain.RuleDetails; + +// TODO: Auto-generated Javadoc +/** + * The Interface ComplianceService. + */ +public interface ComplianceService { + + /** + * Gets the issues details based on name of the domain passed. + * + * @param request the request + * @return ResponseWithOrder + * @throws ServiceException the service exception + */ + public ResponseWithOrder getIssues(Request request) throws ServiceException; + + /** + * Gets Issue count based on name of the asset group/ruleId/domain passed. + * + * @param assetGroup the asset group + * @param ruleId the rule id + * @param domain the domain + * @return long + * @throws ServiceException the service exception + */ + public long getIssuesCount(String assetGroup, String ruleId, String domain) throws ServiceException; + + /** + * Gets Compliance distribution by rule category and severity. + * + * @param assetGroup the asset group + * @param domain the domain + * @return Map + * @throws ServiceException the service exception + */ + public Map getDistribution(String assetGroup, String domain) throws ServiceException; + + /** + * Gets Tagging compliance details based on name of name of the asset group/tagettype passed. + * + * @param assetGroup the asset group + * @param targetType the target type + * @return Map + * @throws ServiceException the service exception + */ + + public Map getTagging(String assetGroup, String targetType) throws ServiceException; + + /** + * Gets the count of expiredCertificates with in 60days and + * totalCertificates for given assetGroup. + * + * @param assetGroup the asset group + * @return Map + * @throws ServiceException the service exception + */ + public Map getCertificates(String assetGroup) throws ServiceException; + + /** + * Gets the patching. + * + * @param assetGroup name of the asset group + * @param targetType the target type + * @param application the application + * @return Method description: asssetGroup is mandatory. Method returns + * count of totalPached/toalUnpatched/TotalInstances for given + * assetGroup. + * @throws ServiceException the service exception + */ + public Map getPatching(String assetGroup, String targetType, String application) throws ServiceException; + + /** + * If method receives + * assetGroup as request parameter, method returns list of all the issue + * counts which are related to recommendations rules from the ES for the + * given assetGroup with all the targetTypes.If method receives both + * assetGroup and targetType as request parameter,method returns list of all + * the issue counts which are related to recommendations rules from the ES + * for the given targetType & assetGroup. + * + * @param assetGroup the asset group + * @param targetType the target type + * @return List> + * @throws ServiceException the service exception + */ + public List> getRecommendations(String assetGroup, String targetType) throws ServiceException; + + /** + * Gets list of issue audit log details for the size you have given. + * + * @param annotationId the annotation id + * @param targetType the target type + * @param from the from + * @param size the size + * @param searchText the search text + * @return ResponseWithOrder + * @throws ServiceException the service exception + */ + public ResponseWithOrder getIssueAuditLog(String annotationId, String targetType, int from, int size, + String searchText) throws ServiceException; + + /** + * Gets the resource details. + * + * @param assetGroup the asset group + * @param resourceId the resource id + * @return List> + * @throws ServiceException the service exception + */ + public List> getResourceDetails(String assetGroup, String resourceId) throws ServiceException; + + /** + * Returns true if its successfully closes all the issues in ES + * for that ruleId else false. + * + * @param ruleDetails the rule details + * @return Map + */ + + public Map closeIssuesByRule(RuleDetails ruleDetails); + + /** + * Gets the list of all the rules compliance mapped to that domain. + * + * @param request the request + * @return ResponseWithOrder + * @throws ServiceException the service exception + */ + public ResponseWithOrder getRulecompliance(Request request) throws ServiceException; + + /** + * Gets the rule details by application.SearchText is used to match any text + * you are looking for. + * + * @param assetGroup the asset group + * @param ruleId the rule id + * @param searchText the search text + * @return List> + * @throws ServiceException the service exception + */ + public List> getRuleDetailsbyApplication(String assetGroup, String ruleId, String searchText) + throws ServiceException; + + /** + * Gets the rule details by environment.SearchText is used to match any + * text you are looking for. + * + * @param assetGroup the asset group + * @param ruleId the rule id + * @param application the application + * @param searchText the search text + * @return List> + * @throws ServiceException the service exception + */ + public List> getRuleDetailsbyEnvironment(String assetGroup, String ruleId, String application, + String searchText) throws ServiceException; + + /** + * Gets the rule description and other details. + * + * @param ruleId the rule id + * @return Map + * @throws ServiceException the service exception + */ + public Map getRuleDescription(String ruleId) throws ServiceException; + + /** + * Gets the kernel version of an instance id from DB where the kernel version updated by web service. + * + * @param instanceId the instance id + * @return Map + * @throws ServiceException the service exception + */ + public Map getKernelComplianceByInstanceIdFromDb(String instanceId) throws ServiceException; + + /** + * Returns true if it updates the + * kernel version for the given instanceId successfully. + * + * @param kernelVersion the kernel version + * @return Map + */ + public Map updateKernelVersion(final KernelVersion kernelVersion); + + /** + * Gets the overall compliance by domain.Over all compliance is calculated by its severity and rule category weightages. + * + * @param assetGroup the asset group + * @param domain the domain + * @return Map + * @throws ServiceException the service exception + */ + public Map getOverallComplianceByDomain(String assetGroup, String domain) throws ServiceException; + + /** + * Gets the list of targetTypes for given asset group and domain + * based on project target types configurations. + * + * @param assetgroup the assetgroup + * @param domain the domain + * @return List + * @throws ServiceException the service exception + */ + public List getResourceType(String assetgroup, String domain)throws ServiceException; + + /** + * Gets the rule severity and category details. + * + * @param ruleDetails the rule details + * @return List> + * @throws ServiceException the service exception + */ + public List> getRuleSevCatDetails(List> ruleDetails) throws ServiceException; + + /** + * Gets the policy violation details by issue id. + * + * @param assetgroup the assetgroup + * @param issueId the issue id + * @return PolicyViolationDetails + * @throws ServiceException the service exception + */ + public PolicyViolationDetails getPolicyViolationDetailsByIssueId(String assetgroup, String issueId) + throws ServiceException; + + /** + * Adds the issue exception. + * + * @param issueException the issue exception + * @return Boolean + * @throws ServiceException the service exception + */ + public Boolean addIssueException(IssueResponse issueException) throws ServiceException; + + /** + * Revoke issue exception. + * + * @param issueId the issue id + * @return boolean + * @throws ServiceException the service exception + */ + public Boolean revokeIssueException(String issueId) throws ServiceException; + + /** + * Generic method to throw the service exception. + * + * @param e the e + * @return ResponseEntity + */ + public ResponseEntity formatException(ServiceException e); + + /** + * method to get current kernel versions. + * + * @return Map + */ + public Map getCurrentKernelVersions(); + + /** + * Adds the multiple issue exception. + * + * @param issuesException the issues exception + * @return the issue exception response + * @throws ServiceException the service exception + */ + public IssueExceptionResponse addMultipleIssueException(IssuesException issuesException) throws ServiceException; + + /** + * Revoke multiple issue exception. + * + * @param issueIds the issue ids + * @return the issue exception response + * @throws ServiceException the service exception + */ + public IssueExceptionResponse revokeMultipleIssueException(List issueIds) throws ServiceException; + +} diff --git a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/ComplianceServiceImpl.java b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/ComplianceServiceImpl.java index 66d7b57e1..edeb9f742 100644 --- a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/ComplianceServiceImpl.java +++ b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/ComplianceServiceImpl.java @@ -27,8 +27,12 @@ import java.util.Set; import java.util.StringTokenizer; import java.util.TreeMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.stream.Collectors; +import javax.annotation.PostConstruct; + import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,6 +55,7 @@ import com.tmobile.pacman.api.commons.exception.ServiceException; import com.tmobile.pacman.api.commons.repo.ElasticSearchRepository; import com.tmobile.pacman.api.commons.utils.CommonUtils; +import com.tmobile.pacman.api.commons.utils.PacHttpUtils; import com.tmobile.pacman.api.commons.utils.ResponseUtils; import com.tmobile.pacman.api.compliance.client.AuthServiceClient; import com.tmobile.pacman.api.compliance.domain.AssetCountDTO; @@ -111,6 +116,32 @@ public class ComplianceServiceImpl implements ComplianceService, Constants { @Value("${features.vulnerability.enabled:false}") private boolean qualysEnabled; + + /** The es host. */ + @Value("${elastic-search.host}") + private String esHost; + + /** The es port. */ + @Value("${elastic-search.port}") + private int esPort; + + /** The critical issue default time interval for calculating delta. */ +// @Value("${critical.issues.defaulttime}") + private String defaultTime = "24hrs"; + + /** The Constant PROTOCOL. */ + static final String PROTOCOL = "http"; + + /** The es url. */ + private String esUrl; + + /** + * Inits the. + */ + @PostConstruct + void init() { + esUrl = PROTOCOL + "://" + esHost + ":" + esPort; + } /** * {@inheritDoc} @@ -198,7 +229,8 @@ public Map getCertificates(String assetGroup) throws ServiceExcept * {@inheritDoc} */ @Override - public Map getPatching(String assetGroup, String targetType) throws ServiceException { + public Map getPatching(String assetGroup, String targetType, String application) throws ServiceException { + logger.info("input value for getPatching are {} {} {}",assetGroup,targetType,application); Long totalPatched; Long totalUnpatched = 0l; Long totalAssets = 0l; @@ -215,14 +247,19 @@ public Map getPatching(String assetGroup, String targetType) throw } for (AssetCountDTO targettype : targetTypes) { String type = targettype.getType(); - if (EC2.equalsIgnoreCase(type)) { - totalAssets += repository.getPatchabeAssetsCount(assetGroup, targettype.getType()); - totalUnpatched += repository.getUnpatchedAssetsCount(assetGroup, targettype.getType()); + if (EC2.equalsIgnoreCase(type) || VIRTUALMACHINE.equalsIgnoreCase(type)) { + totalAssets += repository.getPatchabeAssetsCount(assetGroup, targettype.getType(),application,null,null); + totalUnpatched += repository.getUnpatchedAssetsCount(assetGroup, targettype.getType(),application); } } } catch (DataException e) { + logger.error("Error @ getPatching ", e); throw new ServiceException(e); } + if(totalUnpatched > totalAssets){ + totalUnpatched = totalAssets; + } + totalPatched = totalAssets - totalUnpatched; if (totalAssets > 0) { patchingPercentage = (totalPatched * HUNDRED) / totalAssets; @@ -304,6 +341,7 @@ public Boolean addIssueException(final IssueResponse issueException) throws Serv @SuppressWarnings("rawtypes") public ResponseWithOrder getRulecompliance(Request request) throws ServiceException { // Ignoring input as we need to return all. + logger.debug("getRulecompliance invoked with {}",request); int size = 0; int from = 0; String assetGroup = request.getAg(); @@ -317,60 +355,133 @@ public ResponseWithOrder getRulecompliance(Request request) throws ServiceExcept List> openIssuesByRuleListFinal; ResponseWithOrder response = null; String rule = null; - String ttypes = repository.getTargetTypeForAG(assetGroup, filters.get(DOMAIN)); - + String ttypes = ""; + String resourceTypeFilter = null; + if(filters.containsKey(Constants.RESOURCE_TYPE) && StringUtils.isNotBlank(filters.get(Constants.RESOURCE_TYPE))) { + ttypes = "'"+filters.get(Constants.RESOURCE_TYPE).trim()+"'"; + resourceTypeFilter = filters.get(Constants.RESOURCE_TYPE).trim(); + }else if(!Strings.isNullOrEmpty(filters.get(CommonUtils.convertAttributetoKeyword(TARGET_TYPE)))) { + ttypes = "'"+filters.get(CommonUtils.convertAttributetoKeyword(TARGET_TYPE)).trim()+"'"; + resourceTypeFilter = filters.get(CommonUtils.convertAttributetoKeyword(TARGET_TYPE)).trim(); + } else { + ttypes = repository.getTargetTypeForAG(assetGroup, filters.get(DOMAIN)); + } + logger.debug("Types in scope for invocation {}",ttypes); + final List > dataSourceTargetType = repository.getDataSourceForTargetTypeForAG(assetGroup, filters.get(DOMAIN), resourceTypeFilter); + String application ; + if(filters.containsKey(Constants.APPS)) { + application = filters.get(Constants.APPS); + }else { + application = null; + } + if (!Strings.isNullOrEmpty(ttypes)) { try { - List> ruleIdwithName = repository.getRuleIdWithDisplayNameWithRuleCategoryQuery( - ttypes, ruleCategory); - List> ruleIdwithsScanDate = repository.getRulesLastScanDate(); - Map totalassetCount = repository.getTotalAssetCount(assetGroup, filters.get(DOMAIN)); - + List> rules = new ArrayList<>(); + /*--For filters we need to take rule Id's which match the filter condition--*/ if (!Strings.isNullOrEmpty(filters.get(RULEID_KEYWORD))) { rule = rule + "," + "'" + filters.get(RULEID_KEYWORD) + "'"; - ruleIdwithName = repository.getRuleIdDetails(rule); + rules = repository.getRuleIdDetails(rule); + if(!rules.isEmpty()) + resourceTypeFilter = rules.get(0).get(TARGET_TYPE).toString(); } else { - if (!Strings.isNullOrEmpty(filters.get(CommonUtils.convertAttributetoKeyword(TARGET_TYPE)))) { - ruleIdwithName = repository.getRuleIDsForTargetType(filters.get(CommonUtils - .convertAttributetoKeyword(TARGET_TYPE))); - } + rules = repository.getRuleIdWithDisplayNameWithRuleCategoryQuery( + ttypes, ruleCategory); } - - if (!ruleIdwithName.isEmpty()) { + + logger.debug("Rules in scope {}",rules); + + if (!rules.isEmpty()) { // Make map of rule severity,category - List> ruleSevCatDetails = getRuleSevCatDetails(ruleIdwithName); + List> ruleSevCatDetails = getRuleSevCatDetails(rules); Map ruleCatDetails = ruleSevCatDetails.parallelStream().collect( Collectors.toMap(c -> c.get(RULEID).toString(), c -> c.get(RULE_CATEGORY), (oldvalue, newValue) -> newValue)); Map ruleSevDetails = ruleSevCatDetails.parallelStream().collect( Collectors.toMap(c -> c.get(RULEID).toString(), c -> c.get(SEVERITY), (oldvalue, newValue) -> newValue)); - - Map untagMap = repository.getTaggingByAG(assetGroup,ttypes); - Map totalAwsUnTagMap = repository.getTaggingByAG(AWS,ttypes); - final Map openIssuesByRuleByAG = repository.getNonCompliancePolicyByEsWithAssetGroup( - assetGroup, null, filters, from, size, ttypes); - final Map openIssuesByRuleByAGAWS = repository - .getNonCompliancePolicyByEsWithAssetGroup(AWS, null, filters, from, size, ttypes); - ruleIdwithName - .forEach(ruleIdDetails -> { + + Map ruleAutoFixDetails = ruleSevCatDetails.parallelStream().collect( + Collectors.toMap(c -> c.get(RULEID).toString(), c -> c.get("autofix"), (oldvalue, + newValue) -> newValue)); + + ExecutorService executor = Executors.newCachedThreadPool(); + + + Map totalassetCount = new HashMap<>(); + + totalassetCount.putAll(repository.getTotalAssetCount(assetGroup, filters.get(DOMAIN), application,resourceTypeFilter)); // Can't execute in thread as security context is not passed in feign. + + List> ruleIdwithsScanDate = new ArrayList<>(); + executor.execute(()->{ + try { + ruleIdwithsScanDate.addAll(repository.getRulesLastScanDate()); + } catch (DataException e) { + logger.error("Error fetching rule Last scan date",e); + } + + }); + + Map exemptedAssetsCount = new HashMap<>(); + // executor.execute(()->{ + try { + if(filters.containsKey(Constants.RESOURCE_TYPE)) {// Currently exempted info is only used when resorucetype is passed. Temporary perf fix + exemptedAssetsCount.putAll(repository.getExemptedAssetsCountByRule(assetGroup,application,filters.get(Constants.RESOURCE_TYPE))); + } + } catch (DataException e) { + logger.error("Error fetching exempted asset count",e); + } + + + // }); + + Map untagMap = new HashMap<>(); + + List> rulesTemp = rules; + String ttypesTemp = ttypes; + executor.execute(()->{ + + boolean tagginPolicyExists = rulesTemp.stream().filter(ruleObj-> ruleObj.get(RULEID).toString().contains(TAGGIG_POLICY)).findAny().isPresent(); + + if(tagginPolicyExists) + try { + untagMap.putAll(repository.getTaggingByAG(assetGroup,ttypesTemp,application)); + } catch (DataException e) { + logger.error("Error fetching tagging information ",e); + } + }); + final Map openIssuesByRuleByAG = new HashMap<>(); + executor.execute(()->{ + try { + openIssuesByRuleByAG.putAll(repository.getNonCompliancePolicyByEsWithAssetGroup( + assetGroup, null, filters, from, size, ttypesTemp)); + } catch (DataException e) { + logger.error("Error fetching rule issue aggregations ",e); + + } + + }); + + executor.shutdown(); + + while(!executor.isTerminated()) { + + + } + + rules.forEach(ruleIdDetails -> { Map ruleIdwithsScanDateMap = new HashMap<>(); LinkedHashMap openIssuesByRule = new LinkedHashMap<>(); Long assetCount = 0l; Long issuecountPerRuleAG = 0l; - Long issuecountPerRuleAGAWS = 0l; double compliancePercentage; - double contributionPercentage; + double contributionPercentage = 0; String resourceType = null; String ruleId = null; - long totaluntagged = 0l; - long totalTagged = 0; - double compliance = 0; - long totalAwsUntagged = 0l; - + if (!ruleIdwithsScanDate.isEmpty()) { ruleIdwithsScanDateMap = ruleIdwithsScanDate.stream().collect( Collectors.toMap(s -> (String) s.get(RULEID), @@ -381,119 +492,94 @@ public ResponseWithOrder getRulecompliance(Request request) throws ServiceExcept resourceType = ruleIdDetails.get(TARGET_TYPE).toString(); assetCount = (null != totalassetCount.get(resourceType)) ? totalassetCount .get(resourceType) : 0l; - if (ruleId.equalsIgnoreCase(EC2_KERNEL_COMPLIANCE_RULE)) { - Map cloudPatching = null; - try { - cloudPatching = getPatching(assetGroup, EC2); - } catch (ServiceException e) { - logger.error(e.getMessage()); - } - Long awsUnpatchedInstances = cloudPatching.get(UNPATCHED_INSTANCES); - compliancePercentage = cloudPatching.get(PATCHING_PERCENTAGE); - contributionPercentage = Math.floor(cloudPatching.get(UNPATCHED_INSTANCES) - * HUNDRED / awsUnpatchedInstances); - assetCount = cloudPatching.get(TOTAL_INSTANCES); - issuecountPerRuleAG = cloudPatching.get(UNPATCHED_INSTANCES); - } else if (ruleId.equalsIgnoreCase(ONPREM_KERNEL_COMPLIANCE_RULE)) { - Map onpremPatching = null; - try { - onpremPatching = getPatching(assetGroup, ONPREMSERVER); - } catch (ServiceException e) { - logger.error(e.getMessage()); - } - - Long onpremUnpatchedInstances = onpremPatching.get(UNPATCHED_INSTANCE); - compliancePercentage = onpremPatching.get(PATCHING_PERCENTAGE); - contributionPercentage = Math.floor(onpremPatching.get(UNPATCHED_INSTANCE) - * HUNDRED / onpremUnpatchedInstances); - assetCount = onpremPatching.get(TOTAL_INSTANCES); - issuecountPerRuleAG = onpremPatching.get(UNPATCHED_INSTANCES); + if (null != openIssuesByRuleByAG.get(ruleId)) { + issuecountPerRuleAG = (null != openIssuesByRuleByAG.get(ruleId)) ? openIssuesByRuleByAG + .get(ruleId) : 0l; + + } + if (ruleId.contains(CLOUD_KERNEL_COMPLIANCE_POLICY)|| ruleId.equalsIgnoreCase(ONPREM_KERNEL_COMPLIANCE_RULE)) { + + try { + assetCount = repository.getPatchabeAssetsCount(assetGroup, resourceType,application,null,null); + issuecountPerRuleAG = repository.getUnpatchedAssetsCount(assetGroup, resourceType,application); + } catch (DataException e) { + logger.error("Error fetching patching info",e); + } + } else if (ruleId.contains(TAGGIG_POLICY)) { - + issuecountPerRuleAG = 0l; if (untagMap.get(resourceType) != null) { String totaluntaggedStr = untagMap.get(resourceType).toString() .substring(0, untagMap.get(resourceType).toString().length() - TWO); - totaluntagged = Long.parseLong(totaluntaggedStr); - } - if (totalAwsUnTagMap.get(resourceType) != null) { - String totalAWSuntaggedStr = totalAwsUnTagMap - .get(resourceType) - .toString() - .substring(0, - totalAwsUnTagMap.get(resourceType).toString().length() - TWO); - totalAwsUntagged = Long.parseLong(totalAWSuntaggedStr); - } - - if (totaluntagged > assetCount) { - totaluntagged = assetCount; - } - totalTagged = assetCount - totaluntagged; - if (assetCount > 0) { - compliance = (totalTagged * HUNDRED / assetCount); - compliance = Math.floor(compliance); - } else { - compliance = HUNDRED; - } - if (compliance > HUNDRED) { - compliance = HUNDRED; + issuecountPerRuleAG = Long.parseLong(totaluntaggedStr); } - - compliancePercentage = compliance; - if (totalAwsUntagged > 0) { - contributionPercentage = Math.floor(totaluntagged * HUNDRED / totalAwsUntagged); - } else { - contributionPercentage = 0; - } - issuecountPerRuleAG = totaluntagged; } else { - if((ruleId.equalsIgnoreCase(CLOUD_QUALYS_RULE) && qualysEnabled) || ruleId.equalsIgnoreCase(SSM_AGENT_RULE)){ + if((ruleId.contains(CLOUD_QUALYS_POLICY) && qualysEnabled) || ruleId.equalsIgnoreCase(SSM_AGENT_RULE)){ //qualys coverage require only running instances + logger.info("qualys coverage require only running instances {}",ruleId); try { - assetCount = repository.getInstanceCountForQualys(assetGroup,"noncompliancepolicy","", ""); + if(StringUtils.isNotBlank(filters.get(Constants.APPS))) { + assetCount = repository.getInstanceCountForQualys(assetGroup,"noncompliancepolicy",filters.get(Constants.APPS), "",resourceType); + } else { + assetCount = repository.getInstanceCountForQualys(assetGroup,"noncompliancepolicy","", "",resourceType); + } + } catch (DataException e) { - logger.error("error",e.getMessage()); + logger.error("Error fetching qualys data",e); } - }else{ - assetCount = (null != totalassetCount.get(resourceType)) ? totalassetCount - .get(resourceType) : 0l; - } - - if (null != openIssuesByRuleByAG.get(ruleId)) { - issuecountPerRuleAG = (null != openIssuesByRuleByAG.get(ruleId)) ? openIssuesByRuleByAG - .get(ruleId) : 0l; - issuecountPerRuleAGAWS = (null != openIssuesByRuleByAGAWS.get(ruleId)) ? openIssuesByRuleByAGAWS - .get(ruleId) : 0l; - } - if (issuecountPerRuleAGAWS <= 0) { - compliancePercentage = HUNDRED; - contributionPercentage = 0; - - } else { - - if (issuecountPerRuleAG > assetCount) { - issuecountPerRuleAG = assetCount; - } - compliancePercentage = Math - .floor(((assetCount - issuecountPerRuleAG) * HUNDRED) / assetCount); - contributionPercentage = Math.floor(issuecountPerRuleAG * HUNDRED - / issuecountPerRuleAGAWS); } + + } + if (issuecountPerRuleAG > assetCount) { + issuecountPerRuleAG = assetCount; + } + Long passed = assetCount - issuecountPerRuleAG; + compliancePercentage = Math + .floor(((assetCount - issuecountPerRuleAG) * HUNDRED) / assetCount); + if(assetCount==0){ + compliancePercentage = 100; + issuecountPerRuleAG = 0l; + passed = 0l; + contributionPercentage = 0.0; } openIssuesByRule.put(SEVERITY, ruleSevDetails.get(ruleId)); openIssuesByRule.put(NAME, ruleIdDetails.get(DISPLAY_NAME).toString()); openIssuesByRule.put(COMPLIANCE_PERCENT, compliancePercentage); - openIssuesByRule.put(LAST_SCAN, repository.getScanDate(ruleId, ruleIdwithsScanDateMap)); - openIssuesByRule.put(RULE_CATEGORY, ruleCatDetails.get(ruleId)); - openIssuesByRule.put(RESOURCE_TYPE, resourceType); - openIssuesByRule.put(RULEID, ruleId); - openIssuesByRule.put(ASSETS_SCANNED, assetCount); - openIssuesByRule.put(PASSED, assetCount - issuecountPerRuleAG); - openIssuesByRule.put(FAILED, issuecountPerRuleAG); - openIssuesByRule.put("contribution_percent", contributionPercentage); + String lastScanDate = repository.getScanDate(ruleId, ruleIdwithsScanDateMap); + if(lastScanDate!=null){ + openIssuesByRule.put(LAST_SCAN, lastScanDate); + }else{ + openIssuesByRule.put(LAST_SCAN, ""); + } + final String resourceTypeFinal = resourceType; + openIssuesByRule.put(RULE_CATEGORY, ruleCatDetails.get(ruleId)); + openIssuesByRule.put(RESOURCE_TYPE, resourceType); + openIssuesByRule.put(PROVIDER, dataSourceTargetType.stream() + .filter(datasourceObj -> datasourceObj.get(TYPE).equals(resourceTypeFinal)) + .findFirst().get().get(PROVIDER)); + openIssuesByRule.put(RULEID, ruleId); + openIssuesByRule.put(ASSETS_SCANNED, assetCount); + openIssuesByRule.put(PASSED, passed); + openIssuesByRule.put(FAILED, issuecountPerRuleAG); + openIssuesByRule.put("contribution_percent", contributionPercentage); + openIssuesByRule.put("autoFixEnabled", ruleAutoFixDetails.get(ruleId)); + if(exemptedAssetsCount.containsKey(ruleId)) { + openIssuesByRule.put("exempted", exemptedAssetsCount.get(ruleId)); + openIssuesByRule.put("isAssetsExempted", exemptedAssetsCount.get(ruleId).intValue()>0?true:false); + } else { + openIssuesByRule.put("exempted", 0); + openIssuesByRule.put("isAssetsExempted", false); + } + if (!Strings.isNullOrEmpty(searchText)) { - if (openIssuesByRule.containsValue(searchText)) { - openIssuesByRuleList.add(openIssuesByRule); - } + for (Map.Entry issueByRule : openIssuesByRule.entrySet()) { + if (null != issueByRule.getValue() && issueByRule.getValue().toString().toLowerCase() + .contains(searchText.toLowerCase())) { + openIssuesByRuleList.add(openIssuesByRule); + break; + } + + } } else { openIssuesByRuleList.add(openIssuesByRule); @@ -511,6 +597,7 @@ public ResponseWithOrder getRulecompliance(Request request) throws ServiceExcept response = new ResponseWithOrder(openIssuesByRuleListFinal, openIssuesByRuleListFinal.size()); } } catch (DataException e) { + logger.error("Error @ getRulecompliance while getting the data from ES", e); throw new ServiceException(e); } } @@ -536,67 +623,64 @@ public Map closeIssuesByRule(final RuleDetails ruleDetails) { } /* - * (non-Javadoc) - * - * @see com.tmobile.pacman.api.compliance.service.ComplianceService# - * getRuleDetailsbyApplication(java.lang.String, java.lang.String, - * java.lang.String) - */ - @SuppressWarnings("unchecked") - public List> getRuleDetailsbyApplication(String assetGroup, String ruleId, String searchText) - throws ServiceException { - Map assetcountbyAplications; - List> applicationList = new ArrayList<>(); - String targetType = null; - JsonArray buckets; - try { - buckets = repository.getRuleDetailsByApplicationFromES(assetGroup, ruleId, searchText); - } catch (DataException e) { - throw new ServiceException(e); - } - Gson googleJson = new Gson(); - List> issuesByApplcationList = googleJson.fromJson(buckets, ArrayList.class); - Map issuesByApplcationListMap = issuesByApplcationList.parallelStream().collect( - Collectors.toMap(issue -> issue.get(KEY).toString(), - issue -> (long) Double.parseDouble(issue.get(DOC_COUNT).toString()))); - targetType = getTargetTypeByRuleId(ruleId); - if (!Strings.isNullOrEmpty(targetType)) { - // Get AssetCount By application for Rule TargetType - - if (ruleId.equalsIgnoreCase(EC2_KERNEL_COMPLIANCE_RULE)) { - try { - assetcountbyAplications = repository.getPatchableAssetsByApplication(assetGroup, searchText, - targetType); - } catch (DataException e) { - throw new ServiceException(e); - } - } else if ((ruleId.equalsIgnoreCase(ONPREM_KERNEL_COMPLIANCE_RULE))) { - try { - assetcountbyAplications = repository.getPatchableAssetsByApplication(assetGroup, searchText, - ONPREMSERVER); - } catch (DataException e) { - throw new ServiceException(e); - } - } else if ((ruleId.equalsIgnoreCase(CLOUD_QUALYS_RULE) && qualysEnabled) || ruleId.equalsIgnoreCase(SSM_AGENT_RULE)) { - try{ - assetcountbyAplications = repository.getInstanceCountForQualysByAppsOrEnv(assetGroup, "policydetailsbyapplication","",""); - } catch (DataException e) { - throw new ServiceException(e); - } - } else { - assetcountbyAplications = repository.getAllApplicationsAssetCountForTargetType(assetGroup, targetType); - } - // Form Compliance Details by Application - formComplianceDetailsByApplication(applicationList, assetcountbyAplications, - issuesByApplcationListMap); - } else { - throw new ServiceException("No Target Type associated"); - } - return applicationList; - - } + * (non-Javadoc) + * + * @see com.tmobile.pacman.api.compliance.repository.ComplianceRepository# + * getRuleDetailsByApplicationFromES(java.lang.String, java.lang.String, + * java.lang.String) + */ + public JsonArray getRuleDetailsByApplicationFromES(String assetGroup, String ruleId, String searchText) + throws DataException { + String responseJson = null; + JsonParser jsonParser; + JsonObject resultJson; + StringBuilder requestBody = null; + StringBuilder urlToQueryBuffer = new StringBuilder(esUrl).append("/").append(assetGroup).append("/") + .append(SEARCH); + requestBody = new StringBuilder( + "{\"size\":0,\"query\":{\"bool\":{\"must\":[{\"term\":{\"type.keyword\":{\"value\":\"issue\"}}},{\"term\":{\"ruleId.keyword\":{\"value\":\"" + + ruleId + "\"}}},{\"term\":{\"issueStatus.keyword\":{\"value\":\"open\"}}}"); + if (!StringUtils.isEmpty(searchText)) { + requestBody.append(",{\"match_phrase_prefix\":{\"_all\":\"" + searchText + "\"}}"); + } + // additional filters for kernel compliance rule + if (EC2_KERNEL_COMPLIANCE_RULE.equalsIgnoreCase(ruleId)) { + requestBody.append( + ",{\"has_parent\":{\"parent_type\":\"ec2\",\"query\":{\"bool\":{\"must\":[{\"match\":{\"latest\":\"true\"}},{\"match\":{\"statename\":\"running\"}}],\"must_not\":[{\"match\":{\"platform\":\"windows\"}}]}}}}"); + } else if (VIRTUALMACHINE_KERNEL_COMPLIANCE_RULE.equalsIgnoreCase(ruleId)) { + requestBody.append( + ",{\"has_parent\":{\"parent_type\":\"virtualmachine\",\"query\":{\"bool\":{\"must\":[{\"match\":{\"latest\":\"true\"}},{\"match\":{\"status\":\"" + + RUNNING + "\"}}],\"must_not\":[{\"match\":{\"osType\":\"" + AZURE_WINDOWS + "\"}}]}}}}"); + } + requestBody.append("]"); + // additional filters for Tagging compliance rule + if (ruleId.contains(TAGGING_POLICY)) { + List tagsList = new ArrayList<>(Arrays.asList(mandatoryTags.split(","))); + if (!tagsList.isEmpty()) { + requestBody = requestBody.append(",\"should\":["); + for (String tag : tagsList) { + requestBody = requestBody.append("{\"match_phrase_prefix\":{\"missingTags\":\"" + tag + "\"}},"); + } + requestBody.setLength(requestBody.length() - 1); + requestBody.append("]"); + requestBody.append(",\"minimum_should_match\":1"); + } + } + requestBody + .append("}},\"aggs\":{\"NAME\":{\"terms\":{\"field\":\"tags.Application.keyword\",\"size\":1000}}}}"); + try { + responseJson = PacHttpUtils.doHttpPost(urlToQueryBuffer.toString(), requestBody.toString()); + } catch (Exception e) { + logger.error(ERROR_IN_US, e); + throw new DataException(e); + } + jsonParser = new JsonParser(); + resultJson = (JsonObject) jsonParser.parse(responseJson); + JsonObject aggsJson = (JsonObject) jsonParser.parse(resultJson.get(AGGREGATIONS).toString()); + return aggsJson.getAsJsonObject("NAME").getAsJsonArray(BUCKETS); + } - /* + /* * (non-Javadoc) * * @see com.tmobile.pacman.api.compliance.service.ComplianceService# @@ -607,14 +691,15 @@ public List> getRuleDetailsbyEnvironment(String assetGroup, String searchText) throws ServiceException { List> environmentList = new ArrayList<>(); String targetType = getTargetTypeByRuleId(ruleId); + JsonArray buckets; try { buckets = repository.getRuleDetailsByEnvironmentFromES(assetGroup, ruleId, application, searchText,targetType); } catch (DataException e) { + logger.error("Error @ getRuleDetailsbyEnvironment while getting the env by rule and application from ES", e); throw new ServiceException(e); } - Gson googleJson = new Gson(); List> issuesForApplcationByEnvList = googleJson.fromJson(buckets, ArrayList.class); Map issuesByApplcationListMap = issuesForApplcationByEnvList.parallelStream().collect( @@ -623,7 +708,7 @@ public List> getRuleDetailsbyEnvironment(String assetGroup, Map assetCountByEnv = repository.getTotalAssetCountByEnvironment(assetGroup, application, targetType); - formComplianceDetailsForApplicationByEnvironment(ruleId, assetCountByEnv, issuesByApplcationListMap,assetGroup,application,environmentList); + formComplianceDetailsForApplicationByEnvironment(ruleId, assetCountByEnv, issuesByApplcationListMap,assetGroup,application,environmentList,targetType,searchText); return environmentList; } @@ -1153,7 +1238,7 @@ public IssueExceptionResponse revokeMultipleIssueException(List issueIds } private List> formComplianceDetailsForApplicationByEnvironment(String ruleId, - Map assetCountbyEnvs, Map issuesForApplcationByEnvMap,String assetGroup,String application,List> environmentList) throws ServiceException { + Map assetCountbyEnvs, Map issuesForApplcationByEnvMap,String assetGroup,String application,List> environmentList,String targetType,String searchText) throws ServiceException { Map environment; Long assetCount; long issueCount = 0; @@ -1166,10 +1251,20 @@ private List> formComplianceDetailsForApplicationByEnvironme assetCount = assetCountByEnv.getValue(); envFromAsset = assetCountByEnv.getKey(); - if ((ruleId.equalsIgnoreCase(CLOUD_QUALYS_RULE) && qualysEnabled) || ruleId.equalsIgnoreCase(SSM_AGENT_RULE)) { + if ((ruleId.contains(CLOUD_QUALYS_POLICY) && qualysEnabled) || ruleId.equalsIgnoreCase(SSM_AGENT_RULE)) { try { - assetCount = repository.getInstanceCountForQualys(assetGroup, "policydetailsbyenvironment", application, envFromAsset); + assetCount = repository.getInstanceCountForQualys(assetGroup, "policydetailsbyenvironment", application, envFromAsset,targetType); }catch (DataException e) { + logger.error("Error @ formComplianceDetailsForApplicationByEnvironment while getting the asset count from the qualys or ssm from ES", e); + throw new ServiceException(e); + } + } + + if (ruleId.contains(CLOUD_KERNEL_COMPLIANCE_POLICY)) { + try { + assetCount = repository.getPatchabeAssetsCount(assetGroup,targetType, application, envFromAsset,searchText); + }catch (DataException e) { + logger.error("Error @ formComplianceDetailsForApplicationByEnvironment while getting the asset count from the cloud kernel rule from ES", e); throw new ServiceException(e); } } @@ -1197,4 +1292,64 @@ private List> formComplianceDetailsForApplicationByEnvironme } return environmentList; } + + @SuppressWarnings("unchecked") + public List> getRuleDetailsbyApplication(String assetGroup, String ruleId, String searchText) + throws ServiceException { + Map assetcountbyAplications; + List> applicationList = new ArrayList<>(); + String targetType = null; + JsonArray buckets; + try { + buckets = repository.getRuleDetailsByApplicationFromES(assetGroup, ruleId, searchText); + } catch (DataException e) { + logger.error("Error @ getRuleDetailsbyApplication while getting the application by rule from ES", e); + throw new ServiceException(e); + } + Gson googleJson = new Gson(); + List> issuesByApplcationList = googleJson.fromJson(buckets, ArrayList.class); + Map issuesByApplcationListMap = issuesByApplcationList.parallelStream().collect( + Collectors.toMap(issue -> issue.get(KEY).toString(), + issue -> (long) Double.parseDouble(issue.get(DOC_COUNT).toString()))); + targetType = getTargetTypeByRuleId(ruleId); + if (!Strings.isNullOrEmpty(targetType)) { + // Get AssetCount By application for Rule TargetType + + if (ruleId.contains(CLOUD_KERNEL_COMPLIANCE_POLICY)) { + try { + assetcountbyAplications = repository.getPatchableAssetsByApplication(assetGroup, searchText, + targetType); + } catch (DataException e) { + logger.error("Error @ getRuleDetailsbyApplication while getting the instance count for cloud kernel rule from ES", e); + throw new ServiceException(e); + } + } else if ((ruleId.equalsIgnoreCase(ONPREM_KERNEL_COMPLIANCE_RULE))) { + try { + assetcountbyAplications = repository.getPatchableAssetsByApplication(assetGroup, searchText, + ONPREMSERVER); + } catch (DataException e) { + logger.error("Error @ getRuleDetailsbyApplication while getting the instance count for onprem kernel rule from ES", e); + throw new ServiceException(e); + } + } else if ((ruleId.contains(CLOUD_QUALYS_POLICY) && qualysEnabled) || ruleId.equalsIgnoreCase(SSM_AGENT_RULE)) { + try{ + assetcountbyAplications = repository.getInstanceCountForQualysByAppsOrEnv(assetGroup, "policydetailsbyapplication","","",targetType); + } catch (DataException e) { + logger.error("Error @ getRuleDetailsbyApplication while getting the instance count for qualys from ES", e); + throw new ServiceException(e); + } + }else { + assetcountbyAplications = repository.getAllApplicationsAssetCountForTargetType(assetGroup, targetType); + } + // Form Compliance Details by Application + formComplianceDetailsByApplication(applicationList, assetcountbyAplications, + issuesByApplcationListMap); + } else { + throw new ServiceException("No Target Type associated"); + } + return applicationList; + + } + + } diff --git a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/IssueTrendServiceImpl.java b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/IssueTrendServiceImpl.java index 021a00359..2938161c7 100644 --- a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/IssueTrendServiceImpl.java +++ b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/IssueTrendServiceImpl.java @@ -350,7 +350,7 @@ public void useRealTimeDataForLatestDate( break; case "patching": - baseApiReturnMap = complianceService.getPatching(ag, null); + baseApiReturnMap = complianceService.getPatching(ag, null,null); compliantQuantity = baseApiReturnMap.get("patched_instances"); noncompliantQuantity = baseApiReturnMap .get("unpatched_instances"); diff --git a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/ProjectionServiceImpl.java b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/ProjectionServiceImpl.java index aacb86d4c..7a56b9c19 100644 --- a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/ProjectionServiceImpl.java +++ b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/ProjectionServiceImpl.java @@ -1,622 +1,629 @@ -/******************************************************************************* - * Copyright 2018 T Mobile, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy - * of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - ******************************************************************************/ -package com.tmobile.pacman.api.compliance.service; - -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.time.temporal.IsoFields; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -import com.tmobile.pacman.api.commons.Constants; -import com.tmobile.pacman.api.commons.exception.DataException; -import com.tmobile.pacman.api.commons.exception.ServiceException; -import com.tmobile.pacman.api.compliance.domain.PatchingProgressResponse; -import com.tmobile.pacman.api.compliance.domain.ProjectionRequest; -import com.tmobile.pacman.api.compliance.domain.ProjectionResponse; -import com.tmobile.pacman.api.compliance.repository.ComplianceRepository; -import com.tmobile.pacman.api.compliance.repository.PatchingRepository; -import com.tmobile.pacman.api.compliance.repository.ProjectionRepository; - -/** - * The Class ProjectionServiceImpl. - */ -@Service -public class ProjectionServiceImpl implements ProjectionService, Constants { - - /** The logger. */ - private final Logger logger = LoggerFactory.getLogger(getClass()); - - /** The repository. */ - @Autowired - private ProjectionRepository repository; - - /** The compliance service. */ - @Autowired - private ComplianceService complianceService; - - /** The compliance repository. */ - @Autowired - private ComplianceRepository complianceRepository; - - /** The patching repository. */ - @Autowired - private PatchingRepository patchingRepository; - - /** The projection assetgroups. */ - @Value("${projections.assetgroups}") - private String projectionAssetgroups; - - /** The projection target types. */ - @Value("${projections.targetTypes}") - private String projectionTargetTypes; - - /* (non-Javadoc) - * @see com.tmobile.pacman.api.compliance.service.ProjectionService#updateProjection(com.tmobile.pacman.api.compliance.domain.ProjectionRequest) - */ - @Override - public Boolean updateProjection(ProjectionRequest projectionRequest) - throws ServiceException { - try { - return repository.updateProjectionByTargetType(projectionRequest); - } catch (DataException e) { - logger.error(e.getMessage()); - throw new ServiceException(e); - } - } - - /* (non-Javadoc) - * @see com.tmobile.pacman.api.compliance.service.ProjectionService#getProjection(java.lang.String, int, int) - */ - @Override - public ProjectionResponse getProjection(String resourceType, int year, - int quarter) throws ServiceException { - Long totalAssets = 0l; - List> projectionList = new ArrayList<>(); - if (projectionTargetTypes.contains(resourceType)) { - try { - // get projection from Database - projectionList = repository.getProjectionDetailsFromDb( - resourceType, year, quarter); - // get total Inscope Assets from ES - totalAssets = repository - .getTotalAssetCountByTargetType(resourceType); - } catch (DataException e) { - throw new ServiceException(e); - } - } - - return new ProjectionResponse("", resourceType, totalAssets, year, - quarter, projectionList); - } - - /** - * Gets the total asse count by target type. - * - * @param targetType the target type - * @return the total asse count by target type - * @throws ServiceException the service exception - */ - public Long getTotalAsseCountByTargetType(String targetType) - throws ServiceException { - try { - return repository.getTotalAssetCountByTargetType(targetType); - } catch (DataException e) { - throw new ServiceException(e); - } - } - - /* (non-Javadoc) - * @see com.tmobile.pacman.api.compliance.service.ProjectionService#getPatchingAndProjectionByWeek(java.lang.String) - */ - @Override - public ProjectionResponse getPatchingAndProjectionByWeek(String assetGroup) - throws ServiceException { - if (projectionAssetgroups.contains(assetGroup)) { - List> patchingAndProjectionProgressList = new ArrayList<>(); - Long totalAssets = 0l; - LocalDate todayDate = LocalDate.now(); - DateTimeFormatter formatter = DateTimeFormatter - .ofPattern("yyyy-MM-dd"); - int quarter = todayDate.get(IsoFields.QUARTER_OF_YEAR); - int year = todayDate.getYear(); - int weekNumber = 0; - StringBuilder targetType = new StringBuilder(); - String targetTypes = complianceRepository.getTargetTypeForAG( - assetGroup, null); - List targetTypesList = new ArrayList<>( - Arrays.asList(targetTypes.split(","))); - Map> onpremProjectionByWeekMap = new HashMap<>(); - Map> ec2ProjectionByWeekMap = new HashMap<>(); - long totalPatchCount = 0; - long patchCount = 0; - long previoudWeekTotalPatchCount = 0; - long projectionCount; - long onpremProjectionCount; - long ec2ProjectionCount; - long totalProjectionCount = 0; - long previousWeekProjectionCount = 0; - Map patchingAndProjection = null; - // get data from repository - for (String resourceType : targetTypesList) { - try { - resourceType = resourceType.replaceAll("\'", ""); - - if (resourceType.equalsIgnoreCase(ONPREMSERVER)) { - if (targetType.length() > 0) { - targetType.append(",").append(resourceType); - } else { - targetType.append(resourceType); - } - Long onpremTotalAssets = 0l; - if(complianceService.getPatching( - assetGroup, resourceType).containsKey(TOTAL_INSTANCES)){ - onpremTotalAssets = complianceService.getPatching( - assetGroup, resourceType).get(TOTAL_INSTANCES); - } - - totalAssets += onpremTotalAssets; - List> onpremProjectionByWeekList = repository - .getProjectionDetailsFromDb(resourceType, year, - quarter); - if (!onpremProjectionByWeekList.isEmpty()) { - onpremProjectionByWeekMap = onpremProjectionByWeekList - .parallelStream() - .collect( - Collectors.toMap( - projection -> Integer - .parseInt(projection - .get("week") - .toString()), - projection -> projection)); - } - } else if (resourceType.equalsIgnoreCase(EC2)) { - if (targetType.length() > 0) { - targetType.append(",").append(resourceType); - } else { - targetType.append(resourceType); - } - Long ec2TotalAssets = 0l; - if(complianceService.getPatching( - assetGroup, resourceType).containsKey(TOTAL_INSTANCES)){ - ec2TotalAssets = complianceService.getPatching( - assetGroup, resourceType).get(TOTAL_INSTANCES); - } - - totalAssets += ec2TotalAssets; - - List> ec2ProjectionByWeekList = repository - .getProjectionDetailsFromDb(resourceType, year, - quarter); - if (!ec2ProjectionByWeekList.isEmpty()) { - ec2ProjectionByWeekMap = ec2ProjectionByWeekList - .parallelStream() - .collect( - Collectors.toMap( - projection -> Integer - .parseInt(projection - .get("week") - .toString()), - projection -> projection)); - } - } - } catch (DataException e) { - throw new ServiceException(e); - } - } - Map patchingSnapshot; - try { - patchingSnapshot = repository.getPatchingSnapshot(assetGroup); - } catch (DataException e) { - throw new ServiceException(e); - } - List lastDayOfEachWeek = repository - .getListOfLastWeekDateOfQuarter(); - for (LocalDate lastdayofWeek : lastDayOfEachWeek) { - onpremProjectionCount = 0; - ec2ProjectionCount = 0; - projectionCount = 0; - - patchingAndProjection = new HashMap<>(); - - weekNumber = repository.getWeekNoByDate(lastdayofWeek); - if (null != patchingSnapshot.get(weekNumber)) { - totalPatchCount = patchingSnapshot.get(weekNumber); - } - if (totalPatchCount > previoudWeekTotalPatchCount) { - patchCount = totalPatchCount - previoudWeekTotalPatchCount; - } else if (totalPatchCount == 0) { - patchCount = 0; - totalPatchCount = previoudWeekTotalPatchCount; - } else { - patchCount = totalPatchCount; - } - - previoudWeekTotalPatchCount = totalPatchCount; - if (null != onpremProjectionByWeekMap.get(weekNumber)) { - Map onpremProjectionDetails = onpremProjectionByWeekMap - .get(weekNumber); - if (!onpremProjectionDetails.isEmpty()) { - onpremProjectionCount = Long - .parseLong(onpremProjectionDetails.get( - "projection").toString()); - } - } - - if (null != ec2ProjectionByWeekMap.get(weekNumber)) { - Map ec2ProjectionDetails = ec2ProjectionByWeekMap - .get(weekNumber); - if (!ec2ProjectionDetails.isEmpty()) { - ec2ProjectionCount = Long - .parseLong(ec2ProjectionDetails.get( - "projection").toString()); - } - } - - if (onpremProjectionCount > 0 || ec2ProjectionCount > 0) { - projectionCount = onpremProjectionCount - + ec2ProjectionCount; - } - - totalProjectionCount = projectionCount - + previousWeekProjectionCount; - previousWeekProjectionCount += projectionCount; - patchingAndProjection.put("week", weekNumber); - patchingAndProjection.put("date", - lastdayofWeek.format(formatter)); - patchingAndProjection.put("patched", patchCount); - patchingAndProjection.put("projected", projectionCount); - patchingAndProjection.put("totalPatched", totalPatchCount); - patchingAndProjection.put("totalProjected", - totalProjectionCount); - patchingAndProjectionProgressList.add(patchingAndProjection); - - } - - return new ProjectionResponse(assetGroup, targetType.toString(), - totalAssets, year, quarter, - patchingAndProjectionProgressList); - } else { - throw new ServiceException( - NOT_ELIGIBLE_PROJECTIONS); - } - } - - /* (non-Javadoc) - * @see com.tmobile.pacman.api.compliance.service.ProjectionService#getPatchingProgressByDirector(java.lang.String) - */ - @Override - public PatchingProgressResponse getPatchingProgressByDirector( - String assetGroup) throws ServiceException { - if (projectionAssetgroups.contains(assetGroup)) { - LocalDate todayDate = LocalDate.now(); - int year = todayDate.getYear(); - int quarter = todayDate.get(IsoFields.QUARTER_OF_YEAR); - Long totalAssets = 0l; - StringBuilder targetType = new StringBuilder(); - String quarterScope = "q" + quarter + " scope"; - List> patchingProgressByDirectorList = new ArrayList<>(); - String targetTypes = complianceRepository.getTargetTypeForAG( - assetGroup, null); - List targetTypesList = new ArrayList<>( - Arrays.asList(targetTypes.split(","))); - Map> directorListMap = new ConcurrentHashMap<>(); - for (String resourceType : targetTypesList) { - try { - resourceType = resourceType.replaceAll("\'", ""); - if (resourceType.equalsIgnoreCase(ONPREMSERVER)) { - if (targetType.length() > 0) { - targetType.append(",").append(resourceType); - } else { - targetType.append(resourceType); - } - Map onpremAssetsByApplicationMap = repository - .getAssetDetailsByApplication(assetGroup, - resourceType); - Map onpremUnPatchedCountByApplicationMap = patchingRepository - .getNonCompliantNumberForAgAndResourceType( - assetGroup, resourceType); - - List> appsDetails = repository.getAppsDetails("OnPrem"); - - if (!appsDetails.isEmpty()) { - Long onpremTotalAssets = complianceService.getPatching( - assetGroup, resourceType).get( - TOTAL_INSTANCES); - totalAssets += onpremTotalAssets; - directorListMap = getDirectorsOrExecutorsPatchingProgress( - DIRECTOR, quarterScope, resourceType, - onpremAssetsByApplicationMap, appsDetails, - onpremUnPatchedCountByApplicationMap, - directorListMap, - patchingProgressByDirectorList); - } - } else if (resourceType.equalsIgnoreCase(EC2)) { - if (targetType.length() > 0) { - targetType.append(",").append(resourceType); - } else { - targetType.append(resourceType); - } - Map ec2AssetsByApplicationMap = repository - .getAssetDetailsByApplication(assetGroup, - resourceType); - Map ec2UnPatchedCountByApplicationMap = patchingRepository - .getNonCompliantNumberForAgAndResourceType( - assetGroup, resourceType); - - List> appsDetails = repository.getAppsDetails("Cloud"); - if (!appsDetails.isEmpty()) { - Long ec2TotalAssets = complianceService.getPatching( - assetGroup, resourceType).get( - TOTAL_INSTANCES); - totalAssets += ec2TotalAssets; - directorListMap = getDirectorsOrExecutorsPatchingProgress( - DIRECTOR, quarterScope, resourceType, - ec2AssetsByApplicationMap, appsDetails, - ec2UnPatchedCountByApplicationMap, - directorListMap, - patchingProgressByDirectorList); - } - } - } catch (DataException e) { - throw new ServiceException(e); - } - } - - for (Map.Entry> entry : directorListMap - .entrySet()) { - Map directorMap = entry.getValue(); - if (null != directorMap) { - patchingProgressByDirectorList.add(directorMap); - } - } - Comparator> comp = (m1, m2) -> Integer.compare( - new Integer(m2.get(quarterScope).toString()), new Integer( - m1.get(quarterScope).toString())); - Collections.sort(patchingProgressByDirectorList, comp); - - return new PatchingProgressResponse(assetGroup, - targetType.toString(), totalAssets, year, quarter, - patchingProgressByDirectorList); - } else { - throw new ServiceException( - NOT_ELIGIBLE_PROJECTIONS); - } - } - - /* (non-Javadoc) - * @see com.tmobile.pacman.api.compliance.service.ProjectionService#patchProgByExSponsor(java.lang.String) - */ - @Override - public PatchingProgressResponse patchProgByExSponsor(String assetGroup) - throws ServiceException { - if (projectionAssetgroups.contains(assetGroup)) { - LocalDate todayDate = LocalDate.now(); - int year = todayDate.getYear(); - int quarter = todayDate.get(IsoFields.QUARTER_OF_YEAR); - Long totalAssets = 0l; - StringBuilder targetType = new StringBuilder(); - String quarterScope = "q" + quarter + " scope"; - List> patchingProgressByExecutorsList = new ArrayList<>(); - String targetTypes = complianceRepository.getTargetTypeForAG( - assetGroup, null); - List targetTypesList = new ArrayList<>( - Arrays.asList(targetTypes.split(","))); - Map> executorsListMap = new ConcurrentHashMap<>(); - for (String resourceType : targetTypesList) { - - resourceType = resourceType.replaceAll("\'", ""); - if (resourceType.equalsIgnoreCase(ONPREMSERVER)) { - if (targetType.length() > 0) { - targetType.append(",").append(resourceType); - } else { - targetType.append(resourceType); - } - try { - Map onpremAssetsByApplicationMap = repository - .getAssetDetailsByApplication(assetGroup, - resourceType); - Map onpremUnPatchedCountByApplicationMap = patchingRepository - .getNonCompliantNumberForAgAndResourceType( - assetGroup, resourceType); - - List> appsDetails = repository.getAppsDetails("OnPrem"); - if (!appsDetails.isEmpty()) { - Long onpremTotalAssets = complianceService.getPatching( - assetGroup, resourceType).get(TOTAL_INSTANCES); - totalAssets += onpremTotalAssets; - executorsListMap = getDirectorsOrExecutorsPatchingProgress( - EXCUTIVE_SPONSOR, quarterScope, resourceType, - onpremAssetsByApplicationMap, appsDetails, - onpremUnPatchedCountByApplicationMap, - executorsListMap, - patchingProgressByExecutorsList); - } - } catch (DataException e) { - throw new ServiceException(e); - } - - } else if (resourceType.equalsIgnoreCase(EC2)) { - if (targetType.length() > 0) { - targetType.append(",").append(resourceType); - } else { - targetType.append(resourceType); - } - try { - Map ec2AssetsByApplicationMap = repository - .getAssetDetailsByApplication(assetGroup, - resourceType); - Map ec2UnPatchedCountByApplicationMap = patchingRepository - .getNonCompliantNumberForAgAndResourceType( - assetGroup, resourceType); - - List> appsDetails = repository.getAppsDetails("Cloud"); - if (!appsDetails.isEmpty()) { - Long ec2TotalAssets = complianceService.getPatching( - assetGroup, resourceType).get(TOTAL_INSTANCES); - totalAssets += ec2TotalAssets; - executorsListMap = getDirectorsOrExecutorsPatchingProgress( - EXCUTIVE_SPONSOR, quarterScope, resourceType, - ec2AssetsByApplicationMap, appsDetails, - ec2UnPatchedCountByApplicationMap, - executorsListMap, - patchingProgressByExecutorsList); - } - } catch (DataException e) { - throw new ServiceException(e); - } - - } - - } - - for (Map.Entry> entry : executorsListMap - .entrySet()) { - Map executorsMap = entry.getValue(); - if (null != executorsMap) { - patchingProgressByExecutorsList.add(executorsMap); - } - } - - Comparator> comp = (m1, m2) -> Integer.compare( - new Integer(m2.get(quarterScope).toString()), new Integer( - m1.get(quarterScope).toString())); - Collections.sort(patchingProgressByExecutorsList, comp); - return new PatchingProgressResponse(assetGroup, - targetType.toString(), totalAssets, year, quarter, - patchingProgressByExecutorsList); - } else { - throw new ServiceException( - NOT_ELIGIBLE_PROJECTIONS); - } - - } - - /** - * Gets the directors or executors patching progress. - * - * @param type the type - * @param quarterScope the quarter scope - * @param resourceType the resource type - * @param assetsByApplicationMap the assets by application map - * @param appsDetails the apps details - * @param unPatchedCountByApplicationMap the un patched count by application map - * @param directorOrExeceutorListMap the director or execeutor list map - * @param patchingProgressByDirectorList the patching progress by director list - * @return the directors or executors patching progress - */ - @SuppressWarnings("unused") - private Map> getDirectorsOrExecutorsPatchingProgress( - String type, String quarterScope, String resourceType, - Map assetsByApplicationMap, - List> appsDetails, - Map unPatchedCountByApplicationMap, - Map> directorOrExeceutorListMap, - List> patchingProgressByDirectorList) { - Map applicationByDirectorOrExecutor = appsDetails - .parallelStream() - .filter(apps -> apps.get(type) != null) - .collect( - Collectors.toMap(apps -> apps.get("appTag").toString(), - apps -> apps.get(type), - (oldValue, newValue) -> newValue)); - for (Entry assetDetails : assetsByApplicationMap - .entrySet()) { - - Map patchingProgressByDirectorOrExecutor = new HashMap<>(); - Long unPatched = 0l; - long assetCount = 0l; - long patched = 0l; - String name; - double patchPercentage = 0.0D; - if (assetDetails.getKey() != null - && !"".equals(assetDetails.getKey())) { - if (null != applicationByDirectorOrExecutor.get(assetDetails - .getKey()) - && !("".equals(applicationByDirectorOrExecutor - .get(assetDetails.getKey())))) { - name = applicationByDirectorOrExecutor.get( - assetDetails.getKey()).toString(); - } else { - name = "unknown"; - } - // assetCount - if (null != assetDetails.getValue()) { - assetCount = assetDetails.getValue(); - } - // unpatchedCount - if (null != unPatchedCountByApplicationMap.get(assetDetails - .getKey())) { - unPatched = unPatchedCountByApplicationMap.get(assetDetails - .getKey()); - } - if (!directorOrExeceutorListMap.isEmpty() - && null != directorOrExeceutorListMap.get(name)) { - Map exisitngPatProgByDir; - exisitngPatProgByDir = directorOrExeceutorListMap.get(name); - - assetCount += Long.parseLong(exisitngPatProgByDir.get( - quarterScope).toString()); - unPatched += Long.parseLong(exisitngPatProgByDir.get( - "unpatched").toString()); - } - if (unPatched > assetCount) { - unPatched = assetCount; - } - if (assetCount > 0 && assetCount >= unPatched) { - patched = assetCount - unPatched; - patchPercentage = (patched) * HUNDRED / (assetCount); - patchPercentage = Math.floor(patchPercentage); - } - if (DIRECTOR.equals(type)) { - patchingProgressByDirectorOrExecutor.put(DIRECTOR, name); - } else { - patchingProgressByDirectorOrExecutor.put(EXCUTIVE_SPONSOR, - name); - } - patchingProgressByDirectorOrExecutor.put(quarterScope, - assetCount); - patchingProgressByDirectorOrExecutor.put("patched", patched); - patchingProgressByDirectorOrExecutor.put("%patched", - patchPercentage); - patchingProgressByDirectorOrExecutor - .put("unpatched", unPatched); - if (null != patchingProgressByDirectorOrExecutor) { - directorOrExeceutorListMap.put(name, - patchingProgressByDirectorOrExecutor); - } - } - - } - return directorOrExeceutorListMap; - - } - -} +/******************************************************************************* + * Copyright 2018 T Mobile, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +package com.tmobile.pacman.api.compliance.service; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.temporal.IsoFields; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import com.tmobile.pacman.api.commons.Constants; +import com.tmobile.pacman.api.commons.exception.DataException; +import com.tmobile.pacman.api.commons.exception.ServiceException; +import com.tmobile.pacman.api.compliance.domain.PatchingProgressResponse; +import com.tmobile.pacman.api.compliance.domain.ProjectionRequest; +import com.tmobile.pacman.api.compliance.domain.ProjectionResponse; +import com.tmobile.pacman.api.compliance.repository.ComplianceRepository; +import com.tmobile.pacman.api.compliance.repository.PatchingRepository; +import com.tmobile.pacman.api.compliance.repository.ProjectionRepository; + +/** + * The Class ProjectionServiceImpl. + */ +@Service +public class ProjectionServiceImpl implements ProjectionService, Constants { + + /** The logger. */ + private final Logger logger = LoggerFactory.getLogger(getClass()); + + /** The repository. */ + @Autowired + private ProjectionRepository repository; + + /** The compliance service. */ + @Autowired + private ComplianceService complianceService; + + /** The compliance repository. */ + @Autowired + private ComplianceRepository complianceRepository; + + /** The patching repository. */ + @Autowired + private PatchingRepository patchingRepository; + + /** The projection assetgroups. */ + @Value("${projections.assetgroups}") + private String projectionAssetgroups; + + /** The projection target types. */ + @Value("${projections.targetTypes}") + private String projectionTargetTypes; + + /* (non-Javadoc) + * @see com.tmobile.pacman.api.compliance.service.ProjectionService#updateProjection(com.tmobile.pacman.api.compliance.domain.ProjectionRequest) + */ + @Override + public Boolean updateProjection(ProjectionRequest projectionRequest) + throws ServiceException { + try { + return repository.updateProjectionByTargetType(projectionRequest); + } catch (DataException e) { + logger.error("Error @ updateProjection", e); + throw new ServiceException(e); + } + } + + /* (non-Javadoc) + * @see com.tmobile.pacman.api.compliance.service.ProjectionService#getProjection(java.lang.String, int, int) + */ + @Override + public ProjectionResponse getProjection(String resourceType, int year, + int quarter) throws ServiceException { + Long totalAssets = 0l; + List> projectionList = new ArrayList<>(); + if (projectionTargetTypes.contains(resourceType)) { + try { + // get projection from Database + projectionList = repository.getProjectionDetailsFromDb( + resourceType, year, quarter); + // get total Inscope Assets from ES + totalAssets = repository + .getTotalAssetCountByTargetType(resourceType); + } catch (DataException e) { + logger.error("Error @ getProjection", e); + throw new ServiceException(e); + } + } + + return new ProjectionResponse("", resourceType, totalAssets, year, + quarter, projectionList); + } + + /** + * Gets the total asse count by target type. + * + * @param targetType the target type + * @return the total asse count by target type + * @throws ServiceException the service exception + */ + public Long getTotalAsseCountByTargetType(String targetType) + throws ServiceException { + try { + return repository.getTotalAssetCountByTargetType(targetType); + } catch (DataException e) { + logger.error("Error @ getTotalAsseCountByTargetType", e); + throw new ServiceException(e); + } + } + + /* (non-Javadoc) + * @see com.tmobile.pacman.api.compliance.service.ProjectionService#getPatchingAndProjectionByWeek(java.lang.String) + */ + @Override + public ProjectionResponse getPatchingAndProjectionByWeek(String assetGroup) + throws ServiceException { + if (projectionAssetgroups.contains(assetGroup)) { + List> patchingAndProjectionProgressList = new ArrayList<>(); + Long totalAssets = 0l; + LocalDate todayDate = LocalDate.now(); + DateTimeFormatter formatter = DateTimeFormatter + .ofPattern("yyyy-MM-dd"); + int quarter = todayDate.get(IsoFields.QUARTER_OF_YEAR); + int year = todayDate.getYear(); + int weekNumber = 0; + StringBuilder targetType = new StringBuilder(); + String targetTypes = complianceRepository.getTargetTypeForAG( + assetGroup, null); + List targetTypesList = new ArrayList<>( + Arrays.asList(targetTypes.split(","))); + Map> onpremProjectionByWeekMap = new HashMap<>(); + Map> ec2ProjectionByWeekMap = new HashMap<>(); + long totalPatchCount = 0; + long patchCount = 0; + long previoudWeekTotalPatchCount = 0; + long projectionCount; + long onpremProjectionCount; + long ec2ProjectionCount; + long totalProjectionCount = 0; + long previousWeekProjectionCount = 0; + Map patchingAndProjection = null; + // get data from repository + for (String resourceType : targetTypesList) { + try { + resourceType = resourceType.replaceAll("\'", ""); + + if (resourceType.equalsIgnoreCase(ONPREMSERVER)) { + if (targetType.length() > 0) { + targetType.append(",").append(resourceType); + } else { + targetType.append(resourceType); + } + Long onpremTotalAssets = 0l; + if(complianceService.getPatching( + assetGroup, resourceType, null).containsKey(TOTAL_INSTANCES)){ + onpremTotalAssets = complianceService.getPatching( + assetGroup, resourceType, null).get(TOTAL_INSTANCES); + } + + totalAssets += onpremTotalAssets; + List> onpremProjectionByWeekList = repository + .getProjectionDetailsFromDb(resourceType, year, + quarter); + if (!onpremProjectionByWeekList.isEmpty()) { + onpremProjectionByWeekMap = onpremProjectionByWeekList + .parallelStream() + .collect( + Collectors.toMap( + projection -> Integer + .parseInt(projection + .get("week") + .toString()), + projection -> projection)); + } + } else if (resourceType.equalsIgnoreCase(EC2)) { + if (targetType.length() > 0) { + targetType.append(",").append(resourceType); + } else { + targetType.append(resourceType); + } + Long ec2TotalAssets = 0l; + if(complianceService.getPatching( + assetGroup, resourceType, null).containsKey(TOTAL_INSTANCES)){ + ec2TotalAssets = complianceService.getPatching( + assetGroup, resourceType, null).get(TOTAL_INSTANCES); + } + + totalAssets += ec2TotalAssets; + + List> ec2ProjectionByWeekList = repository + .getProjectionDetailsFromDb(resourceType, year, + quarter); + if (!ec2ProjectionByWeekList.isEmpty()) { + ec2ProjectionByWeekMap = ec2ProjectionByWeekList + .parallelStream() + .collect( + Collectors.toMap( + projection -> Integer + .parseInt(projection + .get("week") + .toString()), + projection -> projection)); + } + } + } catch (DataException e) { + logger.error("Error @ getPatchingAndProjectionByWeek", e); + throw new ServiceException(e); + } + } + Map patchingSnapshot; + try { + patchingSnapshot = repository.getPatchingSnapshot(assetGroup); + } catch (DataException e) { + logger.error("Error @ getPatchingAndProjectionByWeek while getting the patching snapshot", e); + throw new ServiceException(e); + } + List lastDayOfEachWeek = repository + .getListOfLastWeekDateOfQuarter(); + for (LocalDate lastdayofWeek : lastDayOfEachWeek) { + onpremProjectionCount = 0; + ec2ProjectionCount = 0; + projectionCount = 0; + + patchingAndProjection = new HashMap<>(); + + weekNumber = repository.getWeekNoByDate(lastdayofWeek); + if (null != patchingSnapshot.get(weekNumber)) { + totalPatchCount = patchingSnapshot.get(weekNumber); + } + if (totalPatchCount > previoudWeekTotalPatchCount) { + patchCount = totalPatchCount - previoudWeekTotalPatchCount; + } else if (totalPatchCount == 0) { + patchCount = 0; + totalPatchCount = previoudWeekTotalPatchCount; + } else { + patchCount = totalPatchCount; + } + + previoudWeekTotalPatchCount = totalPatchCount; + if (null != onpremProjectionByWeekMap.get(weekNumber)) { + Map onpremProjectionDetails = onpremProjectionByWeekMap + .get(weekNumber); + if (!onpremProjectionDetails.isEmpty()) { + onpremProjectionCount = Long + .parseLong(onpremProjectionDetails.get( + "projection").toString()); + } + } + + if (null != ec2ProjectionByWeekMap.get(weekNumber)) { + Map ec2ProjectionDetails = ec2ProjectionByWeekMap + .get(weekNumber); + if (!ec2ProjectionDetails.isEmpty()) { + ec2ProjectionCount = Long + .parseLong(ec2ProjectionDetails.get( + "projection").toString()); + } + } + + if (onpremProjectionCount > 0 || ec2ProjectionCount > 0) { + projectionCount = onpremProjectionCount + + ec2ProjectionCount; + } + + totalProjectionCount = projectionCount + + previousWeekProjectionCount; + previousWeekProjectionCount += projectionCount; + patchingAndProjection.put("week", weekNumber); + patchingAndProjection.put("date", + lastdayofWeek.format(formatter)); + patchingAndProjection.put("patched", patchCount); + patchingAndProjection.put("projected", projectionCount); + patchingAndProjection.put("totalPatched", totalPatchCount); + patchingAndProjection.put("totalProjected", + totalProjectionCount); + patchingAndProjectionProgressList.add(patchingAndProjection); + + } + + return new ProjectionResponse(assetGroup, targetType.toString(), + totalAssets, year, quarter, + patchingAndProjectionProgressList); + } else { + throw new ServiceException( + NOT_ELIGIBLE_PROJECTIONS); + } + } + + /* (non-Javadoc) + * @see com.tmobile.pacman.api.compliance.service.ProjectionService#getPatchingProgressByDirector(java.lang.String) + */ + @Override + public PatchingProgressResponse getPatchingProgressByDirector( + String assetGroup) throws ServiceException { + if (projectionAssetgroups.contains(assetGroup)) { + LocalDate todayDate = LocalDate.now(); + int year = todayDate.getYear(); + int quarter = todayDate.get(IsoFields.QUARTER_OF_YEAR); + Long totalAssets = 0l; + StringBuilder targetType = new StringBuilder(); + String quarterScope = "q" + quarter + " scope"; + List> patchingProgressByDirectorList = new ArrayList<>(); + String targetTypes = complianceRepository.getTargetTypeForAG( + assetGroup, null); + List targetTypesList = new ArrayList<>( + Arrays.asList(targetTypes.split(","))); + Map> directorListMap = new ConcurrentHashMap<>(); + for (String resourceType : targetTypesList) { + try { + resourceType = resourceType.replaceAll("\'", ""); + if (resourceType.equalsIgnoreCase(ONPREMSERVER)) { + if (targetType.length() > 0) { + targetType.append(",").append(resourceType); + } else { + targetType.append(resourceType); + } + Map onpremAssetsByApplicationMap = repository + .getAssetDetailsByApplication(assetGroup, + resourceType); + Map onpremUnPatchedCountByApplicationMap = patchingRepository + .getNonCompliantNumberForAgAndResourceType( + assetGroup, resourceType); + + List> appsDetails = repository.getAppsDetails("OnPrem"); + + if (!appsDetails.isEmpty()) { + Long onpremTotalAssets = complianceService.getPatching( + assetGroup, resourceType, null).get( + TOTAL_INSTANCES); + totalAssets += onpremTotalAssets; + directorListMap = getDirectorsOrExecutorsPatchingProgress( + DIRECTOR, quarterScope, resourceType, + onpremAssetsByApplicationMap, appsDetails, + onpremUnPatchedCountByApplicationMap, + directorListMap, + patchingProgressByDirectorList); + } + } else if (resourceType.equalsIgnoreCase(EC2)) { + if (targetType.length() > 0) { + targetType.append(",").append(resourceType); + } else { + targetType.append(resourceType); + } + Map ec2AssetsByApplicationMap = repository + .getAssetDetailsByApplication(assetGroup, + resourceType); + Map ec2UnPatchedCountByApplicationMap = patchingRepository + .getNonCompliantNumberForAgAndResourceType( + assetGroup, resourceType); + + List> appsDetails = repository.getAppsDetails("Cloud"); + if (!appsDetails.isEmpty()) { + Long ec2TotalAssets = complianceService.getPatching( + assetGroup, resourceType, null).get( + TOTAL_INSTANCES); + totalAssets += ec2TotalAssets; + directorListMap = getDirectorsOrExecutorsPatchingProgress( + DIRECTOR, quarterScope, resourceType, + ec2AssetsByApplicationMap, appsDetails, + ec2UnPatchedCountByApplicationMap, + directorListMap, + patchingProgressByDirectorList); + } + } + } catch (DataException e) { + logger.error("Error @ getPatchingProgressByDirector", e); + throw new ServiceException(e); + } + } + + for (Map.Entry> entry : directorListMap + .entrySet()) { + Map directorMap = entry.getValue(); + if (null != directorMap) { + patchingProgressByDirectorList.add(directorMap); + } + } + Comparator> comp = (m1, m2) -> Integer.compare( + new Integer(m2.get(quarterScope).toString()), new Integer( + m1.get(quarterScope).toString())); + Collections.sort(patchingProgressByDirectorList, comp); + + return new PatchingProgressResponse(assetGroup, + targetType.toString(), totalAssets, year, quarter, + patchingProgressByDirectorList); + } else { + throw new ServiceException( + NOT_ELIGIBLE_PROJECTIONS); + } + } + + /* (non-Javadoc) + * @see com.tmobile.pacman.api.compliance.service.ProjectionService#patchProgByExSponsor(java.lang.String) + */ + @Override + public PatchingProgressResponse patchProgByExSponsor(String assetGroup) + throws ServiceException { + if (projectionAssetgroups.contains(assetGroup)) { + LocalDate todayDate = LocalDate.now(); + int year = todayDate.getYear(); + int quarter = todayDate.get(IsoFields.QUARTER_OF_YEAR); + Long totalAssets = 0l; + StringBuilder targetType = new StringBuilder(); + String quarterScope = "q" + quarter + " scope"; + List> patchingProgressByExecutorsList = new ArrayList<>(); + String targetTypes = complianceRepository.getTargetTypeForAG( + assetGroup, null); + List targetTypesList = new ArrayList<>( + Arrays.asList(targetTypes.split(","))); + Map> executorsListMap = new ConcurrentHashMap<>(); + for (String resourceType : targetTypesList) { + + resourceType = resourceType.replaceAll("\'", ""); + if (resourceType.equalsIgnoreCase(ONPREMSERVER)) { + if (targetType.length() > 0) { + targetType.append(",").append(resourceType); + } else { + targetType.append(resourceType); + } + try { + Map onpremAssetsByApplicationMap = repository + .getAssetDetailsByApplication(assetGroup, + resourceType); + Map onpremUnPatchedCountByApplicationMap = patchingRepository + .getNonCompliantNumberForAgAndResourceType( + assetGroup, resourceType); + + List> appsDetails = repository.getAppsDetails("OnPrem"); + if (!appsDetails.isEmpty()) { + Long onpremTotalAssets = complianceService.getPatching( + assetGroup, resourceType, null).get(TOTAL_INSTANCES); + totalAssets += onpremTotalAssets; + executorsListMap = getDirectorsOrExecutorsPatchingProgress( + EXCUTIVE_SPONSOR, quarterScope, resourceType, + onpremAssetsByApplicationMap, appsDetails, + onpremUnPatchedCountByApplicationMap, + executorsListMap, + patchingProgressByExecutorsList); + } + } catch (DataException e) { + logger.error("Error @ patchProgByExSponsor", e); + throw new ServiceException(e); + } + + } else if (resourceType.equalsIgnoreCase(EC2)) { + if (targetType.length() > 0) { + targetType.append(",").append(resourceType); + } else { + targetType.append(resourceType); + } + try { + Map ec2AssetsByApplicationMap = repository + .getAssetDetailsByApplication(assetGroup, + resourceType); + Map ec2UnPatchedCountByApplicationMap = patchingRepository + .getNonCompliantNumberForAgAndResourceType( + assetGroup, resourceType); + + List> appsDetails = repository.getAppsDetails("Cloud"); + if (!appsDetails.isEmpty()) { + Long ec2TotalAssets = complianceService.getPatching( + assetGroup, resourceType, null).get(TOTAL_INSTANCES); + totalAssets += ec2TotalAssets; + executorsListMap = getDirectorsOrExecutorsPatchingProgress( + EXCUTIVE_SPONSOR, quarterScope, resourceType, + ec2AssetsByApplicationMap, appsDetails, + ec2UnPatchedCountByApplicationMap, + executorsListMap, + patchingProgressByExecutorsList); + } + } catch (DataException e) { + logger.error("Error @ patchProgByExSponsor", e); + throw new ServiceException(e); + } + + } + + } + + for (Map.Entry> entry : executorsListMap + .entrySet()) { + Map executorsMap = entry.getValue(); + if (null != executorsMap) { + patchingProgressByExecutorsList.add(executorsMap); + } + } + + Comparator> comp = (m1, m2) -> Integer.compare( + new Integer(m2.get(quarterScope).toString()), new Integer( + m1.get(quarterScope).toString())); + Collections.sort(patchingProgressByExecutorsList, comp); + return new PatchingProgressResponse(assetGroup, + targetType.toString(), totalAssets, year, quarter, + patchingProgressByExecutorsList); + } else { + throw new ServiceException( + NOT_ELIGIBLE_PROJECTIONS); + } + + } + + /** + * Gets the directors or executors patching progress. + * + * @param type the type + * @param quarterScope the quarter scope + * @param resourceType the resource type + * @param assetsByApplicationMap the assets by application map + * @param appsDetails the apps details + * @param unPatchedCountByApplicationMap the un patched count by application map + * @param directorOrExeceutorListMap the director or execeutor list map + * @param patchingProgressByDirectorList the patching progress by director list + * @return the directors or executors patching progress + */ + @SuppressWarnings("unused") + private Map> getDirectorsOrExecutorsPatchingProgress( + String type, String quarterScope, String resourceType, + Map assetsByApplicationMap, + List> appsDetails, + Map unPatchedCountByApplicationMap, + Map> directorOrExeceutorListMap, + List> patchingProgressByDirectorList) { + Map applicationByDirectorOrExecutor = appsDetails + .parallelStream() + .filter(apps -> apps.get(type) != null) + .collect( + Collectors.toMap(apps -> apps.get("appTag").toString(), + apps -> apps.get(type), + (oldValue, newValue) -> newValue)); + for (Entry assetDetails : assetsByApplicationMap + .entrySet()) { + + Map patchingProgressByDirectorOrExecutor = new HashMap<>(); + Long unPatched = 0l; + long assetCount = 0l; + long patched = 0l; + String name; + double patchPercentage = 0.0D; + if (assetDetails.getKey() != null + && !"".equals(assetDetails.getKey())) { + if (null != applicationByDirectorOrExecutor.get(assetDetails + .getKey()) + && !("".equals(applicationByDirectorOrExecutor + .get(assetDetails.getKey())))) { + name = applicationByDirectorOrExecutor.get( + assetDetails.getKey()).toString(); + } else { + name = "unknown"; + } + // assetCount + if (null != assetDetails.getValue()) { + assetCount = assetDetails.getValue(); + } + // unpatchedCount + if (null != unPatchedCountByApplicationMap.get(assetDetails + .getKey())) { + unPatched = unPatchedCountByApplicationMap.get(assetDetails + .getKey()); + } + if (!directorOrExeceutorListMap.isEmpty() + && null != directorOrExeceutorListMap.get(name)) { + Map exisitngPatProgByDir; + exisitngPatProgByDir = directorOrExeceutorListMap.get(name); + + assetCount += Long.parseLong(exisitngPatProgByDir.get( + quarterScope).toString()); + unPatched += Long.parseLong(exisitngPatProgByDir.get( + "unpatched").toString()); + } + if (unPatched > assetCount) { + unPatched = assetCount; + } + if (assetCount > 0 && assetCount >= unPatched) { + patched = assetCount - unPatched; + patchPercentage = (patched) * HUNDRED / (assetCount); + patchPercentage = Math.floor(patchPercentage); + } + if (DIRECTOR.equals(type)) { + patchingProgressByDirectorOrExecutor.put(DIRECTOR, name); + } else { + patchingProgressByDirectorOrExecutor.put(EXCUTIVE_SPONSOR, + name); + } + patchingProgressByDirectorOrExecutor.put(quarterScope, + assetCount); + patchingProgressByDirectorOrExecutor.put("patched", patched); + patchingProgressByDirectorOrExecutor.put("%patched", + patchPercentage); + patchingProgressByDirectorOrExecutor + .put("unpatched", unPatched); + if (null != patchingProgressByDirectorOrExecutor) { + directorOrExeceutorListMap.put(name, + patchingProgressByDirectorOrExecutor); + } + } + + } + return directorOrExeceutorListMap; + + } + +} diff --git a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/TaggingServiceImpl.java b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/TaggingServiceImpl.java index 503ab1508..869852514 100644 --- a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/TaggingServiceImpl.java +++ b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/TaggingServiceImpl.java @@ -195,7 +195,7 @@ public List> getUntaggingByTargetTypes(UntaggedTargetTypeReq Map filterTags = request.getFilter(); List> unTagsList = new ArrayList<>(); - Map assetCountByTypes = complainceRepository.getTotalAssetCount(request.getAg(), null); + Map assetCountByTypes = complainceRepository.getTotalAssetCount(request.getAg(), null,null,null); Map untaggedCountMap = getUntaggedTargetTypeIssues(request, tagsList); // process records to format the response for (Map targetType : targetTypes) { diff --git a/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/controller/ComplianceControllerTest.java b/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/controller/ComplianceControllerTest.java index f5a0d0fc6..ceb172500 100644 --- a/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/controller/ComplianceControllerTest.java +++ b/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/controller/ComplianceControllerTest.java @@ -121,11 +121,11 @@ public void getCertificatesTest() throws Exception { @Test public void getPatchingTest() throws Exception { - when(complianceService.getPatching(anyString(),anyString())).thenReturn(CommonTestUtil.getMapLong()); + when(complianceService.getPatching(anyString(),anyString(),anyString())).thenReturn(CommonTestUtil.getMapLong()); assertThat(complianceController.getPatching("ag"), is(notNullValue())); assertThat(complianceController.getPatching(""), is(notNullValue())); - when(complianceService.getPatching(anyString(),anyString())).thenThrow(new ServiceException()); + when(complianceService.getPatching(anyString(),anyString(),anyString())).thenThrow(new ServiceException()); when(complianceService.formatException(anyObject())).thenReturn(ResponseUtils.buildFailureResponse(new ServiceException())); ResponseEntity responseObj = complianceController.getPatching("ag"); assertTrue(responseObj.getStatusCode() == HttpStatus.EXPECTATION_FAILED); diff --git a/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/service/ComplianceServiceImplTest.java b/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/service/ComplianceServiceImplTest.java index 54dc813a2..791403384 100644 --- a/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/service/ComplianceServiceImplTest.java +++ b/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/service/ComplianceServiceImplTest.java @@ -265,15 +265,15 @@ public void getPatchingTest() throws Exception { when( complianceRepository.getPatchabeAssetsCount(anyString(), - anyString())).thenReturn(1000l); + anyString(),anyString(),anyString(),anyString())).thenReturn(1000l); when( complianceRepository.getUnpatchedAssetsCount(anyString(), - anyString())).thenReturn(1000l); + anyString(),anyString())).thenReturn(1000l); - assertThat(complianceService.getPatching("test", ""), + assertThat(complianceService.getPatching("test", "",""), is(notNullValue())); - assertThat(complianceService.getPatching("test", "test"), + assertThat(complianceService.getPatching("test", "test",""), is(notNullValue())); } @@ -298,7 +298,7 @@ public void addIssueExceptionTest() throws Exception { public void getRulecomplianceTest() throws Exception { when(complianceRepository.getTargetTypeForAG(anyString(), anyString())) .thenReturn(CommonTestUtil.getTargetTypes()); - when(complianceRepository.getInstanceCountForQualys(anyString(),anyString(),anyString(),anyString())) + when(complianceRepository.getInstanceCountForQualys(anyString(),anyString(),anyString(),anyString(),anyString())) .thenReturn(5000l); when( @@ -308,14 +308,14 @@ public void getRulecomplianceTest() throws Exception { CommonTestUtil.getMapList()); when(complianceRepository.getRulesLastScanDate()).thenReturn( CommonTestUtil.getMapList()); - when(complianceRepository.getTotalAssetCount(anyString(), anyString())) + when(complianceRepository.getTotalAssetCount(anyString(), anyString(),anyString(),anyString())) .thenReturn(CommonTestUtil.getMapLong()); when(complianceRepository.getRuleIdDetails(anyString())).thenReturn( CommonTestUtil.getMapList()); when(complianceRepository.getRuleIDsForTargetType(anyString())) .thenReturn(CommonTestUtil.getMapList()); - when(complianceRepository.getTaggingByAG(anyString(),anyString())).thenReturn(CommonTestUtil. + when(complianceRepository.getTaggingByAG(anyString(),anyString(),anyString())).thenReturn(CommonTestUtil. getMapObject()); when( complianceRepository.getNonCompliancePolicyByEsWithAssetGroup( @@ -379,7 +379,7 @@ public void getRuleDetailsbyApplicationTest() throws Exception { when( complianceRepository.getInstanceCountForQualysByAppsOrEnv( - anyString(), anyString(),anyString(), anyString())).thenReturn( + anyString(), anyString(),anyString(), anyString(),anyString())).thenReturn( CommonTestUtil.getMapLong()); assertThat( @@ -422,7 +422,7 @@ public void getRuleDetailsbyEnvironmentTest() throws Exception { when( complianceRepository.getInstanceCountForQualys( - anyString(), anyString(),anyString(), anyString())).thenReturn(5000l); + anyString(), anyString(),anyString(), anyString(),anyString())).thenReturn(5000l); assertThat( complianceService.getRuleDetailsbyEnvironment( diff --git a/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/service/IssueTrendServiceImplTest.java b/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/service/IssueTrendServiceImplTest.java index e3a721a5c..0cdd83dbc 100644 --- a/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/service/IssueTrendServiceImplTest.java +++ b/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/service/IssueTrendServiceImplTest.java @@ -99,7 +99,7 @@ public void getTrendProgressTest() throws Exception { when(complianceService.getRulecompliance(anyObject())) .thenReturn(CommonTestUtil.getResponseWithOrder()); - when(complianceService.getPatching(anyString(),anyString())) + when(complianceService.getPatching(anyString(),anyString(),anyString())) .thenReturn(taggingInfoMap); when(complianceService.getOverallComplianceByDomain(anyString(),anyString())) diff --git a/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/service/ProjectionServiceImplTest.java b/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/service/ProjectionServiceImplTest.java index 27a4a51ac..c2ee7a0eb 100644 --- a/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/service/ProjectionServiceImplTest.java +++ b/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/service/ProjectionServiceImplTest.java @@ -127,7 +127,7 @@ public void getPatchingAndProjectionByWeekTest() throws Exception { when(complainceRepository.getTargetTypeForAG(anyString(), anyString())) .thenReturn(targetTypes); - when(complianceService.getPatching(anyString(),anyString())) + when(complianceService.getPatching(anyString(),anyString(),anyString())) .thenReturn(patchingMap); when(repository.getProjectionDetailsFromDb(anyString(), anyInt(),anyInt())) @@ -194,7 +194,7 @@ public void getPatchingProgressByDirectorTest() throws Exception { when(repository.getAppsDetails(anyString())) .thenReturn(maintargetTypesList); - when(complianceService.getPatching(anyString(),anyString())) + when(complianceService.getPatching(anyString(),anyString(),anyString())) .thenReturn(assetMap); assertThat(projectionServiceImpl.getPatchingProgressByDirector("onprem-vm"), @@ -237,7 +237,7 @@ public void patchProgByExSponsorTest() throws Exception { when(repository.getAppsDetails(anyString())) .thenReturn(maintargetTypesList); - when(complianceService.getPatching(anyString(),anyString())) + when(complianceService.getPatching(anyString(),anyString(),anyString())) .thenReturn(assetMap); assertThat(projectionServiceImpl.patchProgByExSponsor("onprem-vm"), diff --git a/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/service/TaggingServiceImplTest.java b/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/service/TaggingServiceImplTest.java index 82c5ec46a..b180486db 100644 --- a/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/service/TaggingServiceImplTest.java +++ b/api/pacman-api-compliance/src/test/java/com/tmobile/pacman/api/compliance/service/TaggingServiceImplTest.java @@ -171,7 +171,7 @@ public void getUntaggingByTargetTypesTest() throws Exception { .thenReturn(maintargetTypesList); ReflectionTestUtils.setField(taggingServiceImpl, "mandatoryTags", "Application,Environment"); - when(complainceRepository.getTotalAssetCount(anyString(), anyString())) + when(complainceRepository.getTotalAssetCount(anyString(), anyString(),anyString(),anyString())) .thenReturn(assetCountMap); when(repository.getUntaggedTargetTypeIssues(anyObject(), anyObject())) From fcbcebc6a1eb90fc53e6f590e7a100b0ee32a965 Mon Sep 17 00:00:00 2001 From: Kanchana Date: Thu, 31 Oct 2019 17:15:36 +0530 Subject: [PATCH 033/107] Added common constants --- .../tmobile/pacman/api/commons/Constants.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/Constants.java b/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/Constants.java index 43e9b51db..eb88c7088 100644 --- a/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/Constants.java +++ b/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/Constants.java @@ -297,5 +297,45 @@ public interface Constants { String VIRTUALMACHINE = "virtualmachine"; String VIRTUALMACHINE_QUALYS_RULEID = "PacMan_Ec2InstanceScannedByQualys_version-1_VmInstanceScannedByQualys_virtualmachine"; String ONPREM_QUALYS_RULEID = "PacMan_Onprem-asset-scanned-by-qualys-API_version-1_OnpremassetscannedbyqualysAPI_onpremserver"; + String CLOUD_KERNEL_COMPLIANCE_POLICY = "PacMan_cloud-kernel-compliance_version-1"; + String AZURE_WINDOWS = "Windows"; + String VIRTUALMACHINE_KERNEL_COMPLIANCE_RULE = "PacMan_cloud-kernel-compliance_version-1_Virtualmachine-Kernel-Compliance-Rule_virtualmachine"; + String CLOUD_TYPE_KEYWORD = "_cloudType.keyword"; + String AGGS_NAME_PROVIDERS = "providers"; + String APPLICATION_COUNT = "applicationCount"; + String APPLICATION_PROVIDERS = "applicationproviders"; + String CISSCOREINDEX = "cisscore"; + String GROUPNAME = "groupName"; + String S3_MACIE_ENABLED_POLICY="PacMan_S3MacieEnabledRule_version-1"; + String S3_MACIE_ALERTS_POLICY="PacMan_S3MacieAlertsRule_version-1"; + String ROOTCAUSE="rootCause"; + String STACKTRACE="stackTrace"; + String UP = "UP"; + String DOWN = "DOWN"; + String SAME = "SAME"; + String INTERVAL_NAME = "name"; + String INTERVAL_ID = "id"; + String INTERVAL_HR = "hr"; + String TIME_INTERVAL = "timeinterval"; + String ISSUE_STATUS_OPENCLOSED = "OPENCLOSED"; + String ISSUE_STATUS_OPEN = "OPEN"; + String ISSUE_STATUS_CLOSED = "CLOSED"; + String TAGGING ="tagging"; + String ENVIRONMENTS = "environments"; + String ENV_COUNT = "envCount"; + String PRODUCTION_ENV = "Prod"; + String STAGE_ENV = "Stage"; + String DEV_ENV = "Dev"; + String NPE_ENV = "NPE"; + String OTHER_ENV = "Others"; + String PROD_PATTERN = "^(((prod)(uction)?)|((prd).*))(:+(((prod)(uction)?)|((prd).*)))?"; + String STG_PATTERN = "(^(stag|stg).*)|(.*:+(stag|stg).*)"; + String DEV_PATTERN = "(^(dev|development).*)|(.*:+(dev).*)"; + String NPE_PATTERN = "(^(npe|non.?prod(uction)?))|(.*:+(npe).*)"; + String UNTAGGED_ENV = "Untagged"; + String RESOURCE_IDS = "resourceIds"; + String RULE_IDS = "ruleIds"; + String TOTAL_VIOLATIONS = "totalViolations"; + String CLOUD_QUALYS_POLICY="PacMan_Ec2InstanceScannedByQualys_version-1"; } From 8922890f974b85502f0b98142ffabbd2883bf077 Mon Sep 17 00:00:00 2001 From: johnrexj Date: Fri, 1 Nov 2019 08:58:49 +0530 Subject: [PATCH 034/107] Azure changes for commons,asset,statistics and admin service. --- .../repository/TargetTypesRepository.java | 3 + .../service/AssetGroupServiceImpl.java | 6 +- .../pacman/api/asset/AssetConstants.java | 2 + .../api/asset/controller/AssetController.java | 5 +- .../controller/AssetCountController.java | 19 +- .../asset/controller/AssetListController.java | 2 +- .../pacman/api/asset/controller/Util.java | 2 +- .../api/asset/repository/AssetRepository.java | 35 +- .../asset/repository/AssetRepositoryImpl.java | 1284 ++++++++++------- .../repository/SearchRepositoryImpl.java | 13 +- .../api/asset/service/AssetService.java | 18 +- .../api/asset/service/AssetServiceImpl.java | 297 +++- .../asset/controller/AssetControllerTest.java | 6 +- .../controller/AssetCountControllerTest.java | 10 +- .../controller/AssetListControllerTest.java | 2 +- .../pacman/api/asset/controller/UtilTest.java | 5 +- .../asset/repository/AssetRepositoryTest.java | 14 +- .../repository/SearchRepositoryTest.java | 4 +- .../api/asset/service/AssetServiceTest.java | 20 +- .../repository/StatisticsRepositoryImpl.java | 5 +- .../service/StatisticsServiceImpl.java | 10 +- .../tmobile/pacman/api/commons/Constants.java | 24 + .../commons/repo/ElasticSearchRepository.java | 106 +- .../pacman/api/commons/utils/CommonUtils.java | 29 + 24 files changed, 1305 insertions(+), 616 deletions(-) diff --git a/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/TargetTypesRepository.java b/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/TargetTypesRepository.java index 75f4d9cab..5d76d2163 100644 --- a/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/TargetTypesRepository.java +++ b/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/TargetTypesRepository.java @@ -64,5 +64,8 @@ public interface TargetTypesRepository extends JpaRepository getAllTargetTypes(); + + @Query("SELECT dataSourceName FROM TargetTypes WHERE targetName = (:targetType) ") + public String findDataSourceByTargetType(@Param("targetType") String targetType); } diff --git a/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/service/AssetGroupServiceImpl.java b/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/service/AssetGroupServiceImpl.java index deec808b7..572853e8a 100644 --- a/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/service/AssetGroupServiceImpl.java +++ b/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/service/AssetGroupServiceImpl.java @@ -353,8 +353,9 @@ private boolean deleteAssetGroupAlias(final AssetGroupDetails assetGroupDetails) if(!targetTypes.isEmpty()) { targetTypes.forEach(targetType -> { + String targetName = targetType.getTargetType().toLowerCase().trim().replaceAll(" ", "-"); Map addObj = Maps.newHashMap(); - addObj.put("index", assetGroupDetails.getDataSource().toLowerCase().trim().replaceAll(" ", "-")+"_"+targetType.getTargetType().toLowerCase().trim().replaceAll(" ", "-")); + addObj.put("index", targetTypesRepository.findDataSourceByTargetType(targetName).toLowerCase().trim().replaceAll(" ", "-")+"_"+targetName); addObj.put("alias", aliasName); Map add = Maps.newHashMap(); add.put("remove", addObj); @@ -384,7 +385,8 @@ private Map createAliasForAssetGroup(final CreateUpdateAssetGrou final String aliasName = assetGroupDetailsJson.getGroupName().toLowerCase().trim().replaceAll(" ", "-"); for (int targetIndex = 0; targetIndex < targetTypes.size(); targetIndex++) { Map addObj = Maps.newHashMap(); - addObj.put("index", assetGroupDetailsJson.getDataSourceName().toLowerCase().trim().replaceAll(" ", "-") + "_" + targetTypes.get(targetIndex).getTargetName().toLowerCase().trim().replaceAll(" ", "-")); + String targetType = targetTypes.get(targetIndex).getTargetName().toLowerCase().trim().replaceAll(" ", "-"); + addObj.put("index", targetTypesRepository.findDataSourceByTargetType(targetType).toLowerCase().trim().replaceAll(" ", "-") + "_" + targetType); addObj.put("alias", aliasName); List attributes = Lists.newArrayList(); if (!targetTypes.get(targetIndex).isIncludeAll()) { diff --git a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/AssetConstants.java b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/AssetConstants.java index 30cc6e0a2..3a7ccb01c 100644 --- a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/AssetConstants.java +++ b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/AssetConstants.java @@ -100,6 +100,8 @@ private AssetConstants() { public static final String FILTER_CATEGORY = "category"; public static final String FILTER_GENERAL = "general"; public static final String FILTER_RECOMMENDATION_ID = "recommendationId"; + public static final String ASSET_TYPE = "assettype"; + public static final String TOTAL_ASSETS = "totalassets"; } diff --git a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/controller/AssetController.java b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/controller/AssetController.java index 56cdcbf21..ef4a696eb 100644 --- a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/controller/AssetController.java +++ b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/controller/AssetController.java @@ -69,9 +69,10 @@ public class AssetController { */ @GetMapping(value = "/v1/list/targettype") public ResponseEntity getListOfTargetTypes(@RequestParam(name = "ag", required = true) String assetGroup, - @RequestParam(name = "domain", required = false) String domain) { + @RequestParam(name = "domain", required = false) String domain, + @RequestParam(name = "provider", required = false) String provider) { Map targetTypesResponse = new HashMap<>(); - List> targetTypes = assetService.getTargetTypesForAssetGroup(assetGroup, domain); + List> targetTypes = assetService.getTargetTypesForAssetGroup(assetGroup, domain, provider); if (targetTypes.isEmpty()) { return ResponseUtils.buildFailureResponse(new Exception( "No target types found for the asset group . Please check the asset group configuration")); diff --git a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/controller/AssetCountController.java b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/controller/AssetCountController.java index d02d28605..5882f76a8 100644 --- a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/controller/AssetCountController.java +++ b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/controller/AssetCountController.java @@ -18,7 +18,9 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; +import java.util.LongSummaryStatistics; import java.util.Map; +import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; @@ -30,6 +32,7 @@ import com.tmobile.pacman.api.asset.AssetConstants; import com.tmobile.pacman.api.asset.service.AssetService; +import com.tmobile.pacman.api.commons.Constants; import com.tmobile.pacman.api.commons.utils.ResponseUtils; /** @@ -58,20 +61,20 @@ public class AssetCountController { @GetMapping(value = "/v1/count") public ResponseEntity geAssetCount(@RequestParam(name = "ag", required = true) String assetGroup, @RequestParam(name = "type", required = false) String type, - @RequestParam(name = "domain", required = false) String domain) { + @RequestParam(name = "domain", required = false) String domain, + @RequestParam(name = "application", required = false) String application, + @RequestParam(name = "provider", required = false) String provider) { if (type == null) { type = "all"; } - List> countMap = assetService.getAssetCountByAssetGroup(assetGroup, type, domain); - + List> countMap = assetService.getAssetCountAndEnvDistributionByAssetGroup(assetGroup, type, domain, application, provider); + LongSummaryStatistics totalCount = countMap.stream().collect(Collectors.summarizingLong(map -> (Long) map.get(Constants.COUNT))); Map response = new HashMap<>(); response.put("ag", assetGroup); response.put(AssetConstants.ASSET_COUNT, countMap); - if (!countMap.isEmpty()) { - return ResponseUtils.buildSucessResponse(response); - } else { - return ResponseUtils.buildFailureResponse(new Exception("No data found")); - } + response.put(AssetConstants.ASSET_TYPE, totalCount.getCount()); + response.put(AssetConstants.TOTAL_ASSETS, totalCount.getSum()); + return ResponseUtils.buildSucessResponse(response); } /** diff --git a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/controller/AssetListController.java b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/controller/AssetListController.java index b6c1757ac..6c9ac1ffa 100644 --- a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/controller/AssetListController.java +++ b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/controller/AssetListController.java @@ -446,7 +446,7 @@ public ResponseEntity getEditableFieldsByTargetType( return ResponseUtils.buildFailureResponse(new Exception("Asset group/TargetType is Mandatory")); } boolean isTargetTypePresent = false; - for (Map targetType : assetService.getTargetTypesForAssetGroup(assetGroup, null)) { + for (Map targetType : assetService.getTargetTypesForAssetGroup(assetGroup, null, null)) { if (targetType.get("type").toString().equals(resourceType)) { isTargetTypePresent = true; break; diff --git a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/controller/Util.java b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/controller/Util.java index f02c1eacf..9154a7c6a 100644 --- a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/controller/Util.java +++ b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/controller/Util.java @@ -59,7 +59,7 @@ public void setassetService(AssetService assetService) { */ public static boolean isValidTargetType(String ag, String type) { try { - List> targetTypes = assetService.getTargetTypesForAssetGroup(ag, null); + List> targetTypes = assetService.getTargetTypesForAssetGroup(ag, null, null); return targetTypes.stream().filter(obj -> type.equals(obj.get("type"))).count() > 0 ? true : false; } catch (Exception e) { LOGGER.error("Error in isValidTargetType ",e); diff --git a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/repository/AssetRepository.java b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/repository/AssetRepository.java index 8a3228af2..ad7709b58 100644 --- a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/repository/AssetRepository.java +++ b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/repository/AssetRepository.java @@ -38,11 +38,11 @@ public interface AssetRepository { * * @param aseetGroupName name of the asset group * @param type target type - * @param domain the domain of asset group + * @param application the application of asset group * * @return list of type and its asset count. */ - public Map getAssetCountByAssetGroup(String aseetGroupName, String type); + public Map getAssetCountByAssetGroup(String aseetGroupName, String type, String application); /** * Fetches all the target types for the particular asset group. If asset @@ -53,7 +53,7 @@ public interface AssetRepository { * * @return list of target types. */ - public List> getTargetTypesByAssetGroup(String aseetGroupName, String domain); + public List> getTargetTypesByAssetGroup(String aseetGroupName, String domain, String provider); /** * Fetches all the applications for the particular asset group. @@ -93,7 +93,7 @@ public interface AssetRepository { * * @return list of target type details. */ - public List> getAllTargetTypes(); + public List> getAllTargetTypes(String datasource); /** * Fetches all the asset groups and its name, display name, description, @@ -515,6 +515,31 @@ public List> getAssetLists(String assetGroup, Map getApplicationAssetCountByAssetGroup(String assetGroupName, String domain) throws DataException; + public Map getApplicationAssetCountByAssetGroup(String assetGroupName, String domain, String provider) throws DataException; + + /** + * Fetches all the datasource and its targetName for the list of targetNames + * + * @return list of target type details.s + */ + public List> getDataSourceForTargetTypes(List targetTypes); + + public Map getApplicationAssetCountByAssetGroupWithProvider(String assetGroupName, String domain, + String provider) throws DataException; + + /** + * Fetches the total count of assets and distribution based on environment for the particular asset group. If no + * type is passed, all the assets of valid target type for the asset group + * is considered. + * + * @param aseetGroupName name of the asset group + * @param type target type + * @param domain the domain of asset group + * @param application the application of asset group + * + * @return list of type and its asset count. + */ + public Map getAssetCountAndEnvDistributionByAssetGroup(String aseetGroupName, String type, + String application); } diff --git a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/repository/AssetRepositoryImpl.java b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/repository/AssetRepositoryImpl.java index 7896889fd..4858933b9 100644 --- a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/repository/AssetRepositoryImpl.java +++ b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/repository/AssetRepositoryImpl.java @@ -27,10 +27,12 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; +import java.util.ListIterator; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.annotation.PostConstruct; @@ -55,6 +57,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; import com.google.common.collect.HashMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; @@ -106,6 +110,9 @@ public class AssetRepositoryImpl implements AssetRepository { @Value("${vulnerability.types}") private String vulnTypes; + @Value("${patching.types:ec2,virtualmachine,onpremserver}") + private String patchingTypes; + @Value("${features.vulnerability.enabled:false}") private boolean qualysEnabled; @@ -124,62 +131,67 @@ void init() { esUrl = PROTOCOL + "://" + esHost + ":" + esPort; heimdallEsesUrl = PROTOCOL + "://" + heimdallEsHost + ":" + heimdallEsPort; } + + private static final String SIZE = "size"; + private static final String AGGS = "aggs"; + private static final String QUERY = "query"; + private static final String ERROR_RETRIEVING_INVENTORY_FROM_ES = "error retrieving inventory from ES"; @Override - public Map getAssetCountByAssetGroup(String aseetGroupName, String type) { + public Map getAssetCountByAssetGroup(String aseetGroupName, String type, String application) { - Map filter = new HashMap<>(); - filter.put(Constants.LATEST, Constants.TRUE); - filter.put(AssetConstants.UNDERSCORE_ENTITY, Constants.TRUE); - HashMultimap shouldFilter = HashMultimap.create(); - if (Constants.EC2.equals(type) || AssetConstants.ALL.equals(type)) { - shouldFilter.put(Constants.STATE_NAME, Constants.RUNNING); - shouldFilter.put(Constants.STATE_NAME, AssetConstants.STOPPED); - shouldFilter.put(Constants.STATE_NAME, AssetConstants.STOPPING); - } - Map countMap = new HashMap<>(); - try { - if (AssetConstants.ALL.equals(type)) { - try { - countMap = esRepository.getTotalDistributionForIndexAndType(aseetGroupName, null, filter, null, - null, AssetConstants.UNDERSCORE_TYPE, Constants.THOUSAND, null); - } catch (Exception e) { - LOGGER.error("Exception in getAssetCountByAssetGroup :" , e); - } - if (!countMap.isEmpty()) { - countMap.put(Constants.EC2, esRepository.getTotalDocumentCountForIndexAndType(aseetGroupName, - Constants.EC2, filter, null, shouldFilter, null, null)); - } - } else { - long count = esRepository.getTotalDocumentCountForIndexAndType(aseetGroupName, type, filter, null, - shouldFilter, null, null); - countMap.put(type, count); - } - } catch (Exception e) { - LOGGER.error("Exception in getAssetCountByAssetGroup :" , e); - } + Map filter = new HashMap<>(); + filter.put(Constants.LATEST, Constants.TRUE); + filter.put(AssetConstants.UNDERSCORE_ENTITY, Constants.TRUE); + if (application != null) { + filter.put(Constants.TAGS_APPS, application); + } + + Map countMap = new HashMap<>(); + try { + if (AssetConstants.ALL.equals(type)) { + try { + countMap = esRepository.getTotalDistributionForIndexAndType(aseetGroupName, null, filter, null, + null, AssetConstants.UNDERSCORE_TYPE, Constants.THOUSAND, null); + } catch (Exception e) { + LOGGER.error("Exception in getAssetCountByAssetGroup :", e); + } + } else { + long count = esRepository.getTotalDocumentCountForIndexAndType(aseetGroupName, type, filter, null, + null, null, null); + countMap.put(type, count); + } + } catch (Exception e) { + LOGGER.error("Exception in getAssetCountByAssetGroup :", e); + } - return countMap; - } + return countMap; + } @Override - public List> getTargetTypesByAssetGroup(String aseetGroupName, String domain) { + public List> getTargetTypesByAssetGroup(String aseetGroupName, String domain, String provider) { - String query = "select distinct targetType as type ,c.category as category,c.domain as domain from cf_AssetGroupTargetDetails a , cf_AssetGroupDetails b ,cf_Target c where a.groupId = b.groupId and a.targetType = c.targetName and b.groupName ='" - + aseetGroupName.trim() + "'"; - if (!StringUtils.isEmpty(domain)) { - query = query + " and lower(c.domain) = '" + domain.toLowerCase().trim() + "'"; - } - return rdsRepository.getDataFromPacman(query); - } + String query = "select distinct targetType as type ,c.category as category,c.domain as domain, dataSourceName as " + Constants.PROVIDER + " from cf_AssetGroupTargetDetails a , cf_AssetGroupDetails b ,cf_Target c where a.groupId = b.groupId and a.targetType = c.targetName and b.groupName ='" + + aseetGroupName.trim() + "'"; + if (!StringUtils.isEmpty(domain)) { + query = query + " and lower(c.domain) = '" + domain.toLowerCase().trim() + "'"; + } + if (!StringUtils.isEmpty(provider)) { + query = query + " and lower(c.dataSourceName) = '" + provider.toLowerCase().trim() + "'"; + } + return rdsRepository.getDataFromPacman(query); + } - @Override - public List> getAllTargetTypes() { + @Override + public List> getAllTargetTypes(String datasource) { - String query = "select distinct targetName as type, category from cf_Target"; - return rdsRepository.getDataFromPacman(query); + String query = "select distinct targetName as type, category, dataSourceName as " + Constants.PROVIDER + " from cf_Target "; + if(datasource!=null) { + query = query + "where lower(dataSourceName) = '"+datasource.toLowerCase()+"'"; + } + return rdsRepository.getDataFromPacman(query); - } + } @Override public List getApplicationByAssetGroup(String aseetGroupName) throws DataException { @@ -199,26 +211,154 @@ public List getApplicationByAssetGroup(String aseetGroupName) throws Dat } @Override - public List getApplicationByAssetGroup(String aseetGroupName, String domain) throws DataException { + public List getApplicationByAssetGroup(String assetGroupName, String domain) throws DataException { + Map applicationMap = getApplicationAssetCountByAssetGroup(assetGroupName, domain, null); + return new ArrayList<>(applicationMap.keySet()); + } - List targetTypes = getTargetTypesByAssetGroup(aseetGroupName, domain).stream() - .map(obj -> obj.get(Constants.TYPE).toString()).collect(Collectors.toList()); - Map filter = new HashMap<>(); - filter.put(Constants.LATEST, Constants.TRUE); - filter.put(AssetConstants.UNDERSCORE_ENTITY, Constants.TRUE); - Map mustTermsFilter = new HashMap<>(); - mustTermsFilter.put(AssetConstants.UNDERSCORE_TYPE, targetTypes); - Map applicationMap = new HashMap<>() ; - - try { - applicationMap = esRepository.getTotalDistributionForIndexAndType(aseetGroupName, null, - filter, null, null, Constants.TAGS_APPS, Constants.THOUSAND, mustTermsFilter); - } catch (Exception e) { - LOGGER.error(AssetConstants.ERROR_GETAPPSBYAG, e); - throw new DataException(e); - } - return new ArrayList<>(applicationMap.keySet()); - } + @Override + public Map getApplicationAssetCountByAssetGroup(String assetGroupName, String domain, String provider) + throws DataException { + + Map applicationMap = new HashMap<>(); + try { + Map applicationAssetData = getApplicationAssetCountByAssetGroupWithProvider(assetGroupName, domain, provider); + applicationMap = (Map) applicationAssetData.get(Constants.APPLICATION_COUNT); + } catch (Exception e) { + LOGGER.error(AssetConstants.ERROR_GETAPPSBYAG, e); + throw new DataException(e); + } + + return applicationMap; + } + + @Override + public Map getApplicationAssetCountByAssetGroupWithProvider (String assetGroupName, String domain, String provider) + throws DataException { + + List targetTypes = getTargetTypesByAssetGroup(assetGroupName, domain, provider).stream() + .map(obj -> obj.get(Constants.TYPE).toString()).collect(Collectors.toList()); + Map filter = new HashMap<>(); + filter.put(Constants.LATEST, Constants.TRUE); + filter.put(AssetConstants.UNDERSCORE_ENTITY, Constants.TRUE); + Map mustTermsFilter = new HashMap<>(); + mustTermsFilter.put(AssetConstants.UNDERSCORE_TYPE, targetTypes); + Map applicationMap = new HashMap<>(); + + try { + applicationMap = getTotalDistributionForIndexAndTypeWithProviders(assetGroupName, null, filter, null, null, + Constants.TAGS_APPS, Constants.TEN_THOUSAND, mustTermsFilter); + } catch (Exception e) { + LOGGER.error(AssetConstants.ERROR_GETAPPSBYAG, e); + throw new DataException(e); + } + + return applicationMap; + } + + /** + * + * @param index + * @param type + * @param mustFilter + * @param mustNotFilter + * @param shouldFilter + * @param aggsFilter + * @param size + * @param mustTermsFilter + * @return + * @throws Exception + */ + private Map getTotalDistributionForIndexAndTypeWithProviders(String index, String type, + Map mustFilter, Map mustNotFilter, + HashMultimap shouldFilter, String aggsFilter, int size, Map mustTermsFilter) + throws Exception { + Map distributionDataList = new HashMap(); + Map distributionCount = new HashMap(); + Map>> distributionProviders = new HashMap>>(); + try { + Map nestedaggs = esRepository.buildAggs(Constants.CLOUD_TYPE_KEYWORD, size, Constants.AGGS_NAME_PROVIDERS, null); + Map response = getDistributionDataFromES (index, type, mustFilter, mustNotFilter, shouldFilter, aggsFilter, size, null, nestedaggs, mustTermsFilter); + Map aggregations = (Map) response.get(Constants.AGGREGATIONS); + Map name = (Map) aggregations.get(Constants.NAME); + List> buckets = (List>) name.get(Constants.BUCKETS); + + for (int i = 0; i < buckets.size(); i++) { + Map bucket = buckets.get(i); + distributionCount.put(bucket.get("key").toString(), ((Double) bucket.get("doc_count")).longValue()); + Map esProviders = (Map) bucket.get(Constants.AGGS_NAME_PROVIDERS); + List> providerbuckets = (List>) esProviders.get(Constants.BUCKETS); + List> providers = new ArrayList>(); + + for (Map esProvider : providerbuckets) { + Map provider = new HashMap(); + provider.put(Constants.PROVIDER, esProvider.get("key").toString()); + provider.put(Constants.TYPE_COUNT, ((Double) esProvider.get("doc_count")).longValue()); + providers.add(provider); + } + distributionProviders.put(bucket.get("key").toString(), providers); + } + + distributionDataList.put(Constants.APPLICATION_COUNT, distributionCount); + distributionDataList.put(Constants.APPLICATION_PROVIDERS, distributionProviders); + + } catch (Exception e) { + LOGGER.error(ERROR_RETRIEVING_INVENTORY_FROM_ES, e); + throw e; + } + return distributionDataList; + } + + /** + * Function for getting the distribution data from ES + * @param index + * @param type + * @param mustFilter + * @param mustNotFilter + * @param shouldFilter + * @param aggsFilter + * @param size + * @param mustTermsFilter + * @return + * @throws Exception + */ + private Map getDistributionDataFromES(String index, String type, Map mustFilter, + Map mustNotFilter, HashMultimap shouldFilter, String aggsFilter, int size, + String aggsName, Map nestedaggs, Map mustTermsFilter) throws Exception { + + String urlToQuery = esRepository.buildAggsURL(esUrl, index, type); + Map requestBody = new HashMap(); + Map matchFilters = Maps.newHashMap(); + Map distributionData = new HashMap(); + if (mustFilter == null) { + matchFilters.put("match_all", new HashMap()); + } else { + matchFilters.putAll(mustFilter); + } + if (null != mustFilter) { + requestBody.put(QUERY, esRepository.buildQuery(matchFilters, mustNotFilter, shouldFilter, null, mustTermsFilter,null)); + requestBody.put(AGGS, esRepository.buildAggs(aggsFilter, size, aggsName, nestedaggs)); + + if (!Strings.isNullOrEmpty(aggsFilter)) { + requestBody.put(SIZE, "0"); + } + + } else { + requestBody.put(QUERY, matchFilters); + } + String responseDetails = null; + Gson gson = new GsonBuilder().create(); + + try { + String requestJson = gson.toJson(requestBody, Object.class); + responseDetails = PacHttpUtils.doHttpPost(urlToQuery, requestJson); + distributionData = (Map) gson.fromJson(responseDetails, Map.class); + } catch (Exception e) { + LOGGER.error(ERROR_RETRIEVING_INVENTORY_FROM_ES, e); + throw e; + } + return distributionData; + } @Override public List getEnvironmentsByAssetGroup(String assetGroup, String application, String domain) { @@ -231,7 +371,7 @@ public List getEnvironmentsByAssetGroup(String assetGroup, String applic } Map mustTermsFilter; if (!StringUtils.isEmpty(domain)) { - List targetTypes = getTargetTypesByAssetGroup(assetGroup, domain).stream() + List targetTypes = getTargetTypesByAssetGroup(assetGroup, domain, null).stream() .map(obj -> obj.get(Constants.TYPE).toString()).collect(Collectors.toList()); mustTermsFilter = new HashMap<>(); mustTermsFilter.put(AssetConstants.UNDERSCORE_TYPE, targetTypes); @@ -472,94 +612,146 @@ public List> getAssetCountByEnvironment(String assetGroup, S return appList; } - @Override - public List> saveAndAppendAssetGroup(String userId, String assetGroup) throws DataException { - - String lowerCaseUserId = userId.toLowerCase(); - String recentView = null; - boolean isDuplicate = false; - boolean isGreaterThanTen = false; - boolean isValidAssetGroup = false; - String assetGroupUpdateWithListQuery = null; - List recentViewList = new ArrayList<>(); - List> assets = getAllAssetGroups(); - List> recentlyViewed = new ArrayList<>(); - Map recentViewMap = new HashMap<>(); - for (Map ag : assets) { - if (assetGroup.equals(ag.get("name"))) { - isValidAssetGroup = true; - String userCountQuery = "SELECT COUNT(userId) FROM pac_v2_userpreferences WHERE userId=\"" - + lowerCaseUserId + "\""; - String recentlyViewedAgQuery = "SELECT recentlyViewedAG FROM pac_v2_userpreferences WHERE userId=\"" - + lowerCaseUserId + "\""; - String assetGroupUpdateQuery = "UPDATE pac_v2_userpreferences SET recentlyViewedAG='" + assetGroup - + "' WHERE userId='" + lowerCaseUserId + "'"; - String assetGroupUpdateAndAppendQuery = "UPDATE pac_v2_userpreferences SET recentlyViewedAG = concat(recentlyViewedAG,'" - + "," + assetGroup + "') WHERE userId='" + lowerCaseUserId + "'"; - String assetGroupInsertQuery = "INSERT INTO pac_v2_userpreferences (userId, recentlyViewedAG) VALUES (?, ?)"; - int userCount = rdsRepository.count(userCountQuery); - List> recentlyViewedAgMap = rdsRepository.getDataFromPacman(recentlyViewedAgQuery); - for (Map recentlyViewedAg : recentlyViewedAgMap) { - if (recentlyViewedAg.get(AssetConstants.RECENTLY_VIEWED_AG) != null) { - recentView = recentlyViewedAg.get(AssetConstants.RECENTLY_VIEWED_AG).toString(); - recentViewList = new CopyOnWriteArrayList(Arrays.asList(recentView.split(","))); - } - } + @SuppressWarnings({ "unchecked", "rawtypes" }) + @Override + public List> saveAndAppendAssetGroup(String userId, String assetGroup) throws DataException { + String lowerCaseUserId = userId.toLowerCase(); + String recentView = null; + boolean isDuplicate = false; + boolean isGreaterThanTen = false; + boolean isValidAssetGroup = false; + String assetGroupUpdateWithListQuery = null; + List recentViewList = new ArrayList<>(); + List> assets = getAllAssetGroups(); + List> recentlyViewed = new ArrayList<>(); + Map recentViewMap = new HashMap<>(); + for (Map ag : assets) { + if (assetGroup.equals(ag.get("name"))) { + isValidAssetGroup = true; + String userCountQuery = "SELECT COUNT(userId) FROM pac_v2_userpreferences WHERE userId=\"" + + lowerCaseUserId + "\""; + String recentlyViewedAgQuery = "SELECT recentlyViewedAG FROM pac_v2_userpreferences WHERE userId=\"" + + lowerCaseUserId + "\""; + String assetGroupUpdateQuery = "UPDATE pac_v2_userpreferences SET recentlyViewedAG='" + assetGroup + + "' WHERE userId='" + lowerCaseUserId + "'"; + String assetGroupUpdateAndAppendQuery = "UPDATE pac_v2_userpreferences SET recentlyViewedAG = concat(recentlyViewedAG,'" + + "," + assetGroup + "') WHERE userId='" + lowerCaseUserId + "'"; + String assetGroupInsertQuery = "INSERT INTO pac_v2_userpreferences (userId, recentlyViewedAG) VALUES (?, ?)"; + int userCount = rdsRepository.count(userCountQuery); + List> recentlyViewedAgMap = rdsRepository.getDataFromPacman(recentlyViewedAgQuery); + for (Map recentlyViewedAg : recentlyViewedAgMap) { + if (recentlyViewedAg.get(AssetConstants.RECENTLY_VIEWED_AG) != null) { + recentView = recentlyViewedAg.get(AssetConstants.RECENTLY_VIEWED_AG).toString(); + recentViewList = new CopyOnWriteArrayList(Arrays.asList(recentView.split(","))); + } + } + + if (userCount > 0) { + if (!StringUtils.isEmpty(recentView)) { + if (recentViewList.size() <= AssetConstants.NINE) { + if (recentViewList.contains(assetGroup)) { + recentViewList.remove(assetGroup); + isDuplicate = true; + } + } else { + if (recentViewList.contains(assetGroup)) { + recentViewList.remove(assetGroup); + isDuplicate = true; + } else { + recentViewList.remove(0); + isGreaterThanTen = true; + } + } + if (isDuplicate || isGreaterThanTen) { + recentViewList.add(assetGroup); + String assetGroups = String.join(",", recentViewList); + assetGroupUpdateWithListQuery = "UPDATE pac_v2_userpreferences SET recentlyViewedAG='" + + assetGroups + "' WHERE userId='" + lowerCaseUserId + "'"; + rdsRepository.update(assetGroupUpdateWithListQuery); + recentViewMap.put(AssetConstants.RECENTLY_VIEWED_AG, + buildRecentViewDetails(recentViewList)); + recentlyViewed.add(recentViewMap); + return recentlyViewed; + + } else { + rdsRepository.update(assetGroupUpdateAndAppendQuery); + recentViewList.add(assetGroup); + recentViewMap.put(AssetConstants.RECENTLY_VIEWED_AG, + buildRecentViewDetails(recentViewList)); + recentlyViewed.add(recentViewMap); + return recentlyViewed; + } + } else { + rdsRepository.update(assetGroupUpdateQuery); + recentViewList.add(assetGroup); + recentViewMap.put(AssetConstants.RECENTLY_VIEWED_AG, buildRecentViewDetails(recentViewList)); + recentlyViewed.add(recentViewMap); + return recentlyViewed; + } + } else { + rdsRepository.update(assetGroupInsertQuery, lowerCaseUserId, assetGroup); + recentViewList.add(assetGroup); + recentViewMap.put(AssetConstants.RECENTLY_VIEWED_AG, buildRecentViewDetails(recentViewList)); + recentlyViewed.add(recentViewMap); + return recentlyViewed; + } + } + } + if (!isValidAssetGroup) { + throw new DataException("Not A Valid Asset Group"); + } + return recentlyViewed; + } - if (userCount > 0) { - if (!StringUtils.isEmpty(recentView)) { - if (recentViewList.size() <= AssetConstants.NINE) { - if (recentViewList.contains(assetGroup)) { - recentViewList.remove(assetGroup); - isDuplicate = true; - } - } else { - if (recentViewList.contains(assetGroup)) { - recentViewList.remove(assetGroup); - isDuplicate = true; - } else { - recentViewList.remove(0); - isGreaterThanTen = true; - } - } - if (isDuplicate || isGreaterThanTen) { - recentViewList.add(assetGroup); - String assetGroups = String.join(",", recentViewList); - assetGroupUpdateWithListQuery = "UPDATE pac_v2_userpreferences SET recentlyViewedAG='" - + assetGroups + "' WHERE userId='" + lowerCaseUserId + "'"; - rdsRepository.update(assetGroupUpdateWithListQuery); - recentViewMap.put(AssetConstants.RECENTLY_VIEWED_AG, recentViewList); - recentlyViewed.add(recentViewMap); - return recentlyViewed; + private List> buildRecentViewDetails(List recentViewList) { + List> recentlyViewedAgMapList = Lists.newArrayList(); + if (recentViewList.size() > 0) { + ListIterator iterator = recentViewList.listIterator(recentViewList.size()); + while (iterator.hasPrevious()) { + String ag = iterator.previous(); + String query = "SELECT displayName FROM cf_AssetGroupDetails WHERE groupName = '" + ag + "'"; + String displayName = rdsRepository.queryForString(query); + if (displayName != null) { + Map details = Maps.newHashMap(); + details.put("ag", ag); + details.put("displayName", displayName); + details.put(Constants.PROVIDERS, providersDetailForAssetGroup(ag)); + recentlyViewedAgMapList.add(details); + } + } + } + return recentlyViewedAgMapList; + } - } else { - rdsRepository.update(assetGroupUpdateAndAppendQuery); - recentViewList.add(assetGroup); - recentViewMap.put(AssetConstants.RECENTLY_VIEWED_AG, recentViewList); - recentlyViewed.add(recentViewMap); - return recentlyViewed; - } - } else { - rdsRepository.update(assetGroupUpdateQuery); - recentViewList.add(assetGroup); - recentViewMap.put(AssetConstants.RECENTLY_VIEWED_AG, recentViewList); - recentlyViewed.add(recentViewMap); - return recentlyViewed; - } - } else { - rdsRepository.update(assetGroupInsertQuery, lowerCaseUserId, assetGroup); - recentViewList.add(assetGroup); - recentViewMap.put(AssetConstants.RECENTLY_VIEWED_AG, recentViewList); - recentlyViewed.add(recentViewMap); - return recentlyViewed; - } - } - } - if (!isValidAssetGroup) { - throw new DataException("Not A Valid Asset Group"); - } - return recentlyViewed; - } + /** + * To get the provider details for an asset group + * @param assetGroup + * @return + */ + private List> providersDetailForAssetGroup ( String assetGroup) { + + Map countMap = getAssetCountByAssetGroup(assetGroup, "all", null); + List> targetTypes = getTargetTypesByAssetGroup(assetGroup, "Infra & Platforms", null); + List validTypes = targetTypes.stream().map(obj -> obj.get(Constants.TYPE).toString()) + .collect(Collectors.toList()); + List countTypes = new ArrayList<>(countMap.keySet()); + for (String _type : countTypes) { + if (!validTypes.contains(_type)) { + countMap.remove(_type); + } + } + List> datasourceForAssettypes = getDataSourceForTargetTypes(validTypes); + Map providerMap = datasourceForAssettypes.stream().filter(typeInfo-> countTypes.contains(typeInfo.get(Constants.TYPE))).collect(Collectors.groupingBy(typeInfo->typeInfo.get(Constants.PROVIDER).toString(),Collectors.counting())); + + List> providersDetails = new ArrayList>(); + providerMap.forEach((k,v)-> { + Map newProvider = new HashMap(); + newProvider.put(Constants.PROVIDER,k); + newProvider.put(Constants.TYPE_COUNT, v); + providersDetails.add(newProvider); + }); + return providersDetails; + } @SuppressWarnings("rawtypes") @Override @@ -588,7 +780,7 @@ public List> getListAssets(String assetGroup, Map validTypes = getTargetTypesByAssetGroup(assetGroup, domain).stream() + List validTypes = getTargetTypesByAssetGroup(assetGroup, domain, null).stream() .map(obj -> obj.get(Constants.TYPE).toString()).collect(Collectors.toList()); if (validTypes.size() > 1) { try { @@ -655,7 +847,7 @@ public long getAssetCount(String assetGroup, Map filter, String Map mustTermFilter = null; if (StringUtils.isEmpty(targetType)) { mustTermFilter = new HashMap<>(); - List validTypes = getTargetTypesByAssetGroup(assetGroup, domain).stream() + List validTypes = getTargetTypesByAssetGroup(assetGroup, domain, null).stream() .map(obj -> obj.get(Constants.TYPE).toString()).collect(Collectors.toList()); mustTermFilter.put(AssetConstants.UNDERSCORE_ENTITY_TYPE_KEYWORD, validTypes); } else { @@ -854,116 +1046,77 @@ public List> getEc2ResourceBlockDevicesDetail(String resourc } @Override - public List> getListAssetsPatchable(String assetGroup, Map filter) { - - LOGGER.info("Inside getListAssetsPatchable"); - List> assetList = new ArrayList<>(); - List> assetDetails = new ArrayList<>(); - List sourceFields = null; - String resourceType = null; - if (filter.containsKey(AssetConstants.FILTER_RES_TYPE)) { - resourceType = filter.get(AssetConstants.FILTER_RES_TYPE); - if (Constants.ONPREMSERVER.equals(resourceType)) { - sourceFields = getDisplayFieldsForTargetType(resourceType); - assetDetails.addAll(getListAssetsPathachableForOnPrem(assetGroup, filter, sourceFields)); - } else if (Constants.EC2.equals(resourceType)) { - assetDetails.addAll(getListAssetsPathachableForEC2(assetGroup, filter, sourceFields)); - sourceFields = getDisplayFieldsForTargetType(resourceType); - } - } else { - boolean agContainsOnPrem = false; - boolean agContainsEc2 = false; - - for (Map targettype : getTargetTypesByAssetGroup(assetGroup, null)) { - if (StringUtils.isNotBlank(targettype.get(Constants.TYPE).toString())) { - if (Constants.ONPREMSERVER.equals(targettype.get(Constants.TYPE).toString())) { - agContainsOnPrem = true; - } - if (Constants.EC2.equals(targettype.get(Constants.TYPE).toString())) { - agContainsEc2 = true; - } - } - } - if (agContainsEc2 && agContainsOnPrem) { - // source common fields for onprem & cloud - sourceFields = getDisplayFieldsForTargetType("all_patchable"); - assetDetails.addAll(getListAssetsPathachableForEC2(assetGroup, filter, sourceFields)); - assetDetails.addAll(getListAssetsPathachableForOnPrem(assetGroup, filter, sourceFields)); - } else if (agContainsEc2) { - sourceFields = getDisplayFieldsForTargetType(Constants.EC2); - assetDetails.addAll(getListAssetsPathachableForEC2(assetGroup, filter, sourceFields)); - } else if (agContainsOnPrem) { - sourceFields = getDisplayFieldsForTargetType(Constants.ONPREMSERVER); - assetDetails.addAll(getListAssetsPathachableForOnPrem(assetGroup, filter, sourceFields)); - } - } - - try { - List executiveapps = new ArrayList<>(); - if (filter.containsKey(AssetConstants.FILTER_EXEC_SPONSOR)) { - executiveapps = fetchExecDirectorApps(filter.get(AssetConstants.FILTER_EXEC_SPONSOR), - "executiveSponsor"); - } - if (filter.containsKey(AssetConstants.FILTER_DIRECTOR)) { - executiveapps = fetchExecDirectorApps(filter.get(AssetConstants.FILTER_DIRECTOR), "director"); - } - for (Map assetDetail : assetDetails) { - Map asset = new LinkedHashMap<>(); - if (CollectionUtils.isEmpty(sourceFields)) { - asset.put(Constants.RESOURCE_DISPLAY_ID, assetDetail.get(Constants.RESOURCEID)); - asset.put(Constants.ACCOUNT_DISPALY_NAME, assetDetail.get(Constants.ACCOUNT_NAME)); - asset.put(Constants.REGION_DISPALY_NAME, assetDetail.get(Constants.REGION)); - if (!assetDetail.containsKey(Constants.TAGS_APPLICATION)) { - asset.put(Constants.APPLICATION, ""); - } else { - asset.put(Constants.APPLICATION, assetDetail.get(Constants.TAGS_APPLICATION)); - } - if (!assetDetail.containsKey(Constants.TAGS_ENVIRONMENT)) { - asset.put(Constants.ENVIRONMENT, ""); - } else { - asset.put(Constants.ENVIRONMENT, assetDetail.get(Constants.TAGS_ENVIRONMENT)); - } - asset.put(Constants.ACCOUNT_DISPLAYI_D, assetDetail.get(Constants.ACCOUNT_ID)); + public List> getListAssetsPatchable(String assetGroup, Map filter) { + + LOGGER.info("Inside getListAssetsPatchable"); + List> assetList = new ArrayList<>(); + List> assetDetails = new ArrayList<>(); + List sourceFields = null; + String resourceType = null; + if (filter.containsKey(AssetConstants.FILTER_RES_TYPE)) { + resourceType = filter.get(AssetConstants.FILTER_RES_TYPE); + if (Constants.ONPREMSERVER.equals(resourceType)) { + sourceFields = getDisplayFieldsForTargetType(resourceType); + assetDetails.addAll(getListAssetsPathachableForOnPrem(assetGroup, filter, sourceFields)); + } else if (Constants.EC2.equals(resourceType) || Constants.VIRTUALMACHINE.equals(resourceType)) { + assetDetails.addAll(getListAssetsPathachableForEC2(assetGroup, filter, sourceFields)); + + } + } else { + + List validPatchingTypes= Arrays.asList(patchingTypes.split(",")); + List validTargetTypes = new ArrayList<>(getAssetCountByAssetGroup(assetGroup,AssetConstants.ALL,filter.get(AssetConstants.FILTER_APPLICATION)).keySet()); + List agPatchingTypes =validTargetTypes.stream().filter(validPatchingTypes::contains).collect(Collectors.toList()); + + if (agPatchingTypes.size()>1) { + // source common fields for onprem & cloud + sourceFields = getDisplayFieldsForTargetType("all_patchable"); + } + for(String type: agPatchingTypes) { + if(type.equals(Constants.ONPREMSERVER)) { + if(agPatchingTypes.size()==1) { + sourceFields = getDisplayFieldsForTargetType(Constants.ONPREMSERVER); + } + assetDetails.addAll(getListAssetsPathachableForOnPrem(assetGroup, filter, sourceFields)); + }else { + filter.put(AssetConstants.FILTER_RES_TYPE,type); + assetDetails.addAll(getListAssetsPathachableForEC2(assetGroup, filter, sourceFields)); + } + } + } - } else { - // Loop through the displayble fields if the value is null - // pull null value - for (String field : sourceFields) { - if (!assetDetail.containsKey(field)) { - asset.put(field, ""); - } else { - asset.put(field, assetDetail.get(field)); - } - } - } - // common fileds for filters - if (filter.containsKey(AssetConstants.FILTER_PATCHED)) { - if (AssetConstants.FALSE.equals(filter.get(AssetConstants.FILTER_PATCHED))) { - asset.put(AssetConstants.UNDERSCORE_ENTITY_TYPE, assetDetail.get(Constants.TARGET_TYPE)); - } else { - asset.put(AssetConstants.UNDERSCORE_ENTITY_TYPE, - assetDetail.get(AssetConstants.UNDERSCORE_ENTITY_TYPE)); - } - } else { - asset.put(AssetConstants.UNDERSCORE_ENTITY_TYPE, - assetDetail.get(AssetConstants.UNDERSCORE_ENTITY_TYPE)); - } - if (filter.containsKey(AssetConstants.FILTER_EXEC_SPONSOR) - || filter.containsKey(AssetConstants.FILTER_DIRECTOR)) { - if (executiveapps.contains(assetDetail.get(Constants.TAGS_APPLICATION))) { - assetList.add(asset); - } - } else { - assetList.add(asset); - } - } - } catch (Exception e) { - LOGGER.error("Error in getListAssetsPatchable", e); - } + try { + final List executiveapps; + if (filter.containsKey(AssetConstants.FILTER_EXEC_SPONSOR)) { + executiveapps = fetchExecDirectorApps(filter.get(AssetConstants.FILTER_EXEC_SPONSOR), + "executiveSponsor"); + }else if (filter.containsKey(AssetConstants.FILTER_DIRECTOR)) { + executiveapps = fetchExecDirectorApps(filter.get(AssetConstants.FILTER_DIRECTOR), "director"); + }else { + executiveapps = new ArrayList<>(); + } + List fieldsToBeSkipped = Arrays.asList(Constants.DOCID, + AssetConstants.UNDERSCORE_ENTITY, Constants._ID, AssetConstants.UNDERSCORE_LOADDATE, + Constants.ES_DOC_PARENT_KEY, Constants.ES_DOC_ROUTING_KEY, AssetConstants.CREATE_TIME, + AssetConstants.FIRST_DISCOVEREDON, AssetConstants.DISCOVERY_DATE, Constants.LATEST, + AssetConstants.CREATION_DATE); + assetList.addAll(formGetListResponse(sourceFields, assetDetails, fieldsToBeSkipped)); + if (filter.containsKey(AssetConstants.FILTER_EXEC_SPONSOR) + || filter.containsKey(AssetConstants.FILTER_DIRECTOR)) { + + return assetList.parallelStream().filter(asset-> executiveapps.contains(asset.get(Constants.TAGS_APPLICATION))).collect(Collectors.toList()); + + }else { + return assetList; + } + + } catch (Exception e) { + LOGGER.error("Error in getListAssetsPatchable", e); + } - LOGGER.info("Exiting getListAssetsPatchable"); - return assetList; - } + LOGGER.info("Exiting getListAssetsPatchable"); + return assetList; + } @Override public List> getListAssetsTaggable(String assetGroup, Map filter) { @@ -1160,104 +1313,91 @@ public List> getListAssetsTaggable(String assetGroup, Map> getListAssetsVulnerable(String assetGroup, Map filter) { - - LOGGER.info("Inside getListAssetsVulnerable"); - List> assetList = new ArrayList<>(); - List> assetDetails = new ArrayList<>(); - - boolean agContainsOnPrem = false; - boolean agContainsEc2 = false; + public List> getListAssetsVulnerable(String assetGroup, Map filter) { - List validTargetTypes = getTargetTypesByAssetGroup(assetGroup, null).stream() - .map(obj -> obj.get(Constants.TYPE).toString()).collect(Collectors.toList()); - String[] vulnTypesArray = vulnTypes.split(","); - List vulnTargetTypes = new ArrayList<>(); + LOGGER.info("Inside getListAssetsVulnerable"); + List> assetList = new ArrayList<>(); + List> assetDetails = new ArrayList<>(); - for (String vulnType : vulnTypesArray) { - if (validTargetTypes.contains(vulnType.trim())) { - vulnTargetTypes.add(vulnType); - } - } + List validTargetTypes = new ArrayList<>(getAssetCountByAssetGroup(assetGroup,AssetConstants.ALL,filter.get(AssetConstants.FILTER_APPLICATION)).keySet()); + String[] vulnTypesArray = vulnTypes.split(","); + List vulnTargetTypes = new ArrayList<>(); - if (validTargetTypes.contains(Constants.ONPREMSERVER)) { - agContainsOnPrem = true; - } - if (validTargetTypes.contains(Constants.EC2)) { - agContainsEc2 = true; - } - List fieldNames = null; - try { - if (agContainsEc2 && agContainsOnPrem) { - fieldNames = getDisplayFieldsForTargetType("all_vulnerable"); - } else if (agContainsEc2) { - fieldNames = getDisplayFieldsForTargetType(Constants.EC2); - } else if (agContainsOnPrem) { - fieldNames = getDisplayFieldsForTargetType(Constants.ONPREMSERVER); - } - } catch (Exception e) { - LOGGER.error(AssetConstants.ERROR_FETCHING_FIELDNAMES , e); - } - if (!vulnTargetTypes.isEmpty()) { - for (String parentType : vulnTargetTypes) { - StringBuilder urlToQueryBuffer = new StringBuilder(esUrl).append("/").append(assetGroup); - urlToQueryBuffer.append("/").append(parentType); - urlToQueryBuffer.append("/").append(Constants.SEARCH).append("?scroll=") - .append(Constants.ES_PAGE_SCROLL_TTL); - - String urlToQuery = urlToQueryBuffer.toString(); - String urlToScroll = new StringBuilder(esUrl).append("/").append(Constants.SEARCH).append("/scroll") - .toString(); - - StringBuilder requestBody = new StringBuilder( - "{\"size\":10000,\"query\":{\"bool\":{\"must\":[{\"has_child\":{\"type\":\"vulninfo\",\"query\":{\"bool\":{\"must\":[{\"match\":{\"latest\":true}},{\"term\":{\"qid\":"); - requestBody.append(filter.get("qid")); - requestBody.append("}}]}}}},{\"term\":{\"latest\":true}}"); - if (filter.containsKey(AssetConstants.FILTER_APPLICATION)) { - requestBody.append(",{\"term\":{\"tags.Application.keyword\":\""); - requestBody.append(filter.get(AssetConstants.FILTER_APPLICATION)); - requestBody.append("\"}}"); - } - if (filter.containsKey(AssetConstants.FILTER_ENVIRONMENT)) { - requestBody.append(",{\"term\":{\"tags.Environment.keyword\":\""); - requestBody.append(filter.get(AssetConstants.FILTER_ENVIRONMENT)); - requestBody.append("\"}}"); - } - if (filter.containsKey(AssetConstants.FILTER_RES_TYPE)) { - requestBody.append(",{\"term\":{\"_entitytype.keyword\":\""); - requestBody.append(filter.get(AssetConstants.FILTER_RES_TYPE)); - requestBody.append("\"}}"); - } - requestBody.append("]}}}"); - Long totalDocs = getTotalDocCount(assetGroup, parentType, "{" + requestBody.toString().substring(14)); - String request = requestBody.toString(); - String scrollId = null; - if(totalDocs>0){ - for (int index = 0; index <= (totalDocs / Constants.ES_PAGE_SIZE); index++) { - String responseDetails = null; - try { - if (!Strings.isNullOrEmpty(scrollId)) { - request = esRepository.buildScrollRequest(scrollId, Constants.ES_PAGE_SCROLL_TTL); - urlToQuery = urlToScroll; - } - responseDetails = PacHttpUtils.doHttpPost(urlToQuery, request); - scrollId = esRepository.processResponseAndSendTheScrollBack(responseDetails, assetDetails); - } catch (Exception e) { - LOGGER.error("Error in getListAssetsVulnerable", e); - } - } - } - } - - List fieldsToBeSkipped = Arrays.asList(Constants.RESOURCEID, Constants.DOCID, - AssetConstants.UNDERSCORE_ENTITY, Constants._ID, AssetConstants.UNDERSCORE_LOADDATE, Constants.ES_DOC_PARENT_KEY, - Constants.ES_DOC_ROUTING_KEY, AssetConstants.CREATE_TIME, AssetConstants.FIRST_DISCOVEREDON, AssetConstants.DISCOVERY_DATE, - Constants.LATEST, AssetConstants.CREATION_DATE); - LOGGER.info("Exiting getListAssetsVulnerable"); - assetList.addAll(formGetListResponse(fieldNames, assetDetails, fieldsToBeSkipped)); - } - return assetList; - } + for (String vulnType : vulnTypesArray) { + if (validTargetTypes.contains(vulnType.trim())) { + vulnTargetTypes.add(vulnType); + } + } + + List fieldNames = null; + try { + if (vulnTargetTypes.size()>1) { + fieldNames = getDisplayFieldsForTargetType("all_vulnerable"); + } + } catch (Exception e) { + LOGGER.error(AssetConstants.ERROR_FETCHING_FIELDNAMES, e); + } + if (!vulnTargetTypes.isEmpty()) { + for (String parentType : vulnTargetTypes) { + StringBuilder urlToQueryBuffer = new StringBuilder(esUrl).append("/").append(assetGroup); + urlToQueryBuffer.append("/").append(parentType); + urlToQueryBuffer.append("/").append(Constants.SEARCH).append("?scroll=") + .append(Constants.ES_PAGE_SCROLL_TTL); + + String urlToQuery = urlToQueryBuffer.toString(); + String urlToScroll = new StringBuilder(esUrl).append("/").append(Constants.SEARCH).append("/scroll") + .toString(); + + StringBuilder requestBody = new StringBuilder( + "{\"size\":10000,\"query\":{\"bool\":{\"must\":[{\"has_child\":{\"type\":\"vulninfo\",\"query\":{\"bool\":{\"must\":[{\"match\":{\"latest\":true}},{\"term\":{\"qid\":"); + requestBody.append(filter.get("qid")); + requestBody.append("}}]}}}},{\"term\":{\"latest\":true}}"); + if (filter.containsKey(AssetConstants.FILTER_APPLICATION)) { + requestBody.append(",{\"term\":{\"tags.Application.keyword\":\""); + requestBody.append(filter.get(AssetConstants.FILTER_APPLICATION)); + requestBody.append("\"}}"); + } + if (filter.containsKey(AssetConstants.FILTER_ENVIRONMENT)) { + requestBody.append(",{\"term\":{\"tags.Environment.keyword\":\""); + requestBody.append(filter.get(AssetConstants.FILTER_ENVIRONMENT)); + requestBody.append("\"}}"); + } + if (filter.containsKey(AssetConstants.FILTER_RES_TYPE)) { + requestBody.append(",{\"term\":{\"_entitytype.keyword\":\""); + requestBody.append(filter.get(AssetConstants.FILTER_RES_TYPE)); + requestBody.append("\"}}"); + } + requestBody.append("]}}}"); + Long totalDocs = getTotalDocCount(assetGroup, parentType, "{" + requestBody.toString().substring(14)); + String request = requestBody.toString(); + String scrollId = null; + if (totalDocs > 0) { + for (int index = 0; index <= (totalDocs / Constants.ES_PAGE_SIZE); index++) { + String responseDetails = null; + try { + if (!Strings.isNullOrEmpty(scrollId)) { + request = esRepository.buildScrollRequest(scrollId, Constants.ES_PAGE_SCROLL_TTL); + urlToQuery = urlToScroll; + } + responseDetails = PacHttpUtils.doHttpPost(urlToQuery, request); + scrollId = esRepository.processResponseAndSendTheScrollBack(responseDetails, assetDetails); + } catch (Exception e) { + LOGGER.error("Error in getListAssetsVulnerable", e); + } + } + } + } + + List fieldsToBeSkipped = Arrays.asList(Constants.RESOURCEID, Constants.DOCID, + AssetConstants.UNDERSCORE_ENTITY, Constants._ID, AssetConstants.UNDERSCORE_LOADDATE, + Constants.ES_DOC_PARENT_KEY, Constants.ES_DOC_ROUTING_KEY, AssetConstants.CREATE_TIME, + AssetConstants.FIRST_DISCOVEREDON, AssetConstants.DISCOVERY_DATE, Constants.LATEST, + AssetConstants.CREATION_DATE); + LOGGER.info("Exiting getListAssetsVulnerable"); + assetList.addAll(formGetListResponse(fieldNames, assetDetails, fieldsToBeSkipped)); + } + return assetList; + } @Override public List> getListAssetsScanned(String assetGroup, Map filter) { @@ -1604,7 +1744,7 @@ public List> getOpenPortDetailsByInstanceId(String instanceI private String getTargetTypeByRuleId(String assetGroup, String ruleId) { LOGGER.info("Getting Target type for Rule id : " + ruleId); - List targetTypes = getTargetTypesByAssetGroup(assetGroup, null).stream() + List targetTypes = getTargetTypesByAssetGroup(assetGroup, null, null).stream() .map(obj -> obj.get(Constants.TYPE).toString()).collect(Collectors.toList()); String ttypesTemp; String ttypes = null; @@ -1626,43 +1766,156 @@ private String getTargetTypeByRuleId(String assetGroup, String ruleId) { } @Override - public Map getResourceCreateInfo(String resourceId) throws DataException { + public Map getResourceCreateInfo(String resourceId) throws DataException { + Pattern VALID_EMAIL_ADDRESS_REGEX = Pattern.compile("^[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,6}$", + Pattern.CASE_INSENSITIVE); - String url = heimdallEsesUrl + "/pacman-resource-claim/_search"; - String request = "{\"query\": {\"match\": {\"resourceid.keyword\": \"" + resourceId + "\"}}}"; - String responseDetails; - try { - responseDetails = PacHttpUtils.doHttpPost(url, request); - } catch (Exception e) { - LOGGER.error("Exception in getResourceCreateInfo ",e); - throw new DataException(e); - } + String url = heimdallEsesUrl + "/pacman-resource-claim/_search"; + String request = "{\"query\": {\"match\": {\"resourceid.keyword\": \"" + resourceId + "\"}}}"; + String responseDetails; + try { + responseDetails = PacHttpUtils.doHttpPost(url, request); + } catch (Exception e) { + LOGGER.error("Exception in getResourceCreateInfo ", e); + throw new DataException(e); + } - JsonObject responseDetailsjson = new JsonParser().parse(responseDetails).getAsJsonObject(); - JsonArray hits = responseDetailsjson.get("hits").getAsJsonObject().get("hits").getAsJsonArray(); + JsonObject responseDetailsjson = new JsonParser().parse(responseDetails).getAsJsonObject(); + JsonArray hits = responseDetailsjson.get("hits").getAsJsonObject().get("hits").getAsJsonArray(); + Map map = new HashMap<>(); + + if (hits.size() > 0) { + JsonObject createInfoObj = hits.get(0).getAsJsonObject().get(AssetConstants.UNDERSCORE_SOURCE) + .getAsJsonObject(); // Exp + Gson gson = new Gson(); + map = (Map) gson.fromJson(createInfoObj, map.getClass()); + + // User better key names for createdBy and creationDate + Object obj = map.remove("user"); + map.put("createdBy", obj); + + obj = map.remove("time"); + map.put(AssetConstants.CREATION_DATE, obj); + + convertNullToBlankStr(map); + + if (map.get("createdBy").toString().indexOf("/") != -1) { + String userIdStr = map.get("createdBy").toString() + .substring(map.get("createdBy").toString().indexOf("/") + 1); + try { + Map slashUserNameMustFilter = new HashMap<>(); + slashUserNameMustFilter.put("_resourceid.keyword", userIdStr); + List> adUserReturn = esRepository.getDataFromES("aws_aduser", null, + slashUserNameMustFilter, null, null, Arrays.asList("mail"), null); + map.put("email", getValueFromList(adUserReturn, "mail")); + } catch (Exception e) { + e.printStackTrace(); + } + } + convertNullToBlankStr(map); + + String userName = retrieveOu(map); + String appId = ""; + if (userName != null && userName.length() >= 4 && userName.substring(0, 4).endsWith("_")) { + appId = userName.substring(0, 3); + } else if (userName != null) { + try { + Map userNameMustFilter = new HashMap<>(); + userNameMustFilter.put("_resourceid.keyword", userName); + List> adUserReturn = esRepository.getDataFromES("aws_aduser", null, + userNameMustFilter, null, null, Arrays.asList("mail"), null); + if (!adUserReturn.isEmpty()) { + map.put("email", getValueFromList(adUserReturn, "mail")); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + Map plOuMustFilter = new HashMap<>(); + plOuMustFilter.put(CommonUtils.convertAttributetoKeyword("appID"), appId); + try { + List> plOuEmail = esRepository.getDataFromES("aws_apps", null, plOuMustFilter, null, + null, Arrays.asList("projectLead"), null); + if (!plOuEmail.isEmpty()) { + map.put("projectLead", getValueFromList(plOuEmail, "projectLead")); + } + } catch (Exception e) { + e.printStackTrace(); + } + + convertNullToBlankStr(map); - if (hits.size() > 0) { - JsonObject createInfoObj = hits.get(0).getAsJsonObject().get(AssetConstants.UNDERSCORE_SOURCE) - .getAsJsonObject(); // Exp - Gson gson = new Gson(); - Map map = new HashMap<>(); - map = (Map) gson.fromJson(createInfoObj, map.getClass()); + } - // User better key names for createdBy and creationDate - Object obj = map.remove("user"); - map.put("createdBy", obj); + try { + Map plResMustFilter = new HashMap<>(); + plResMustFilter.put(CommonUtils.convertAttributetoKeyword("_resourceid"), resourceId); + List> tagsInput = esRepository.getDataFromES(Constants.MASTER_ALIAS, null, plResMustFilter, null, null, + Arrays.asList("tags.Owner", "tags.Application"), null); + + map.put("ownerEmail", getValueFromList(tagsInput, "tags.Owner")); + + Map appTagMustFilter = new HashMap<>(); + appTagMustFilter.put(CommonUtils.convertAttributetoKeyword("appTag"), + getValueFromList(tagsInput, "tags.Application")); + List> plFromAppTag = esRepository.getDataFromES("aws_apps", null, appTagMustFilter, + null, null, Arrays.asList("projectLead"), null); + if (!plFromAppTag.isEmpty() && (map.get("projectLead") == null || map.get("projectLead").equals(""))) { + map.put("projectLead", getValueFromList(plFromAppTag, "projectLead")); + } + } catch (Exception e) { + e.printStackTrace(); + } + convertNullToBlankStr(map); - obj = map.remove("time"); - map.put(AssetConstants.CREATION_DATE, obj); + return map; + } + + private void convertNullToBlankStr(Map map) { + if (map.get("email") == null || "null".equals(map.get("email").toString())) { + map.put("email", ""); + } + if (map.get("projectLead") == null || "null".equals(map.get("projectLead").toString())) { + map.put("projectLead", ""); + } + if (map.get(AssetConstants.CREATION_DATE) == null + || "null".equals(map.get(AssetConstants.CREATION_DATE).toString())) { + map.put(AssetConstants.CREATION_DATE, ""); + } + if (map.get("createdBy") == null || "null".equals(map.get("createdBy").toString())) { + map.put("createdBy", ""); + } + } - // The string 'null' could come in email - if ("null".equals(map.get("email").toString())) { - map.put("email", ""); - } - return map; - } - return null; - } + private String retrieveOu(Map map) { + map = (Map) ((Map) map.get("detail")).get("userIdentity"); + + String userName = null; + if ("Root".equalsIgnoreCase(map.get("type").toString()) + || "IAMUser".equalsIgnoreCase(map.get("type").toString())) { + userName = map.get("userName").toString(); + } else if ("AssumedRole".equalsIgnoreCase(map.get("type").toString())) { + Map sessionContext = (Map) (map.get("sessionContext")); + Map sessionIssuer = (Map) sessionContext.get("sessionIssuer"); + userName = sessionIssuer.get("userName").toString(); + } + return userName; + } + + private Object getValueFromList(List> input, String str) { + Iterator> inputListIterator = input.iterator(); + + while (inputListIterator.hasNext()) { + + Map inputMap = inputListIterator.next(); + if (inputMap.get(str) != null) { + return inputMap.get(str).toString(); + } + + } + return ""; + } @Override public Map getNotificationSummary(String instanceId) throws DataException { @@ -2034,108 +2287,124 @@ public List> getAssetLists(String assetGroup, Map> getListAssetsPathachableForEC2(String assetGroup, Map filter, - List source) { - - List> assetDetails = new ArrayList<>(); - - Map mustFilter = new HashMap<>(); - Map mustFilterAsset = new HashMap<>(); - Map mustNotFilterAsset = new HashMap<>(); - - List> unpatchedInstances; - List> totalInstances; - - String targetAssetGroup; - mustFilter.put(CommonUtils.convertAttributetoKeyword(Constants.TYPE), Constants.ISSUE); - mustFilter.put(CommonUtils.convertAttributetoKeyword(Constants.ISSUE_STATUS), Constants.OPEN); - - Map parentBool = new HashMap<>(); - List> mustList = new ArrayList<>(); - Map matchMap = new HashMap<>(); - Map match = new HashMap<>(); + List source) { - mustFilter.put(CommonUtils.convertAttributetoKeyword(Constants.RULEID), Constants.EC2_KERNEL_COMPLIANCE_RULE); + String resourceType = filter.get(AssetConstants.FILTER_RES_TYPE); + + List> assetDetails = new ArrayList<>(); - // Changes to include only latest resources + Map mustFilter = new HashMap<>(); + Map mustFilterAsset = new HashMap<>(); + Map mustNotFilterAsset = new HashMap<>(); - match.put(Constants.LATEST, Constants.TRUE); + List> unpatchedInstances; + List> totalInstances; - matchMap.put(Constants.MATCH, match); - mustList.add(matchMap); - - match = new HashMap<>(); - match.put(Constants.STATE_NAME, Constants.RUNNING); - matchMap = new HashMap<>(); - matchMap.put(Constants.MATCH, match); - mustList.add(matchMap); + String targetAssetGroup; + mustFilter.put(CommonUtils.convertAttributetoKeyword(Constants.TYPE), Constants.ISSUE); + mustFilter.put(CommonUtils.convertAttributetoKeyword(Constants.ISSUE_STATUS), Constants.OPEN); - parentBool.put("must", mustList); + Map parentBool = new HashMap<>(); + List> mustList = new ArrayList<>(); + Map matchMap = new HashMap<>(); + Map match = new HashMap<>(); - match = new HashMap<>(); - match.put("platform", "windows"); - matchMap = new HashMap<>(); - matchMap.put(Constants.MATCH, match); + mustFilter.put(CommonUtils.convertAttributetoKeyword(Constants.POLICYID), Constants.CLOUD_KERNEL_COMPLIANCE_POLICY); - parentBool.put("must_not", matchMap); + // Changes to include only latest resources - Map queryMap = new HashMap<>(); - queryMap.put("bool", parentBool); + match.put(Constants.LATEST, Constants.TRUE); - Map parentEntryMap = new LinkedHashMap<>(); - parentEntryMap.put(Constants.TYPE, Constants.EC2); - parentEntryMap.put(AssetConstants.QUERY, queryMap); - mustFilter.put("has_parent", parentEntryMap); + matchMap.put(Constants.MATCH, match); + mustList.add(matchMap); - filter.entrySet() - .stream() - .forEach( - entry -> { - if (!(entry.getKey().equals(AssetConstants.FILTER_PATCHED) - || entry.getKey().equals(AssetConstants.FILTER_RES_TYPE) - || entry.getKey().equals(AssetConstants.FILTER_EXEC_SPONSOR) || entry - .getKey().equals(AssetConstants.FILTER_DIRECTOR))) { - if (entry.getKey().equals(AssetConstants.FILTER_APPLICATION)) { - mustFilter.put(Constants.TAGS_APPS, entry.getValue()); - mustFilterAsset.put(Constants.TAGS_APPS, entry.getValue()); - } - if (entry.getKey().equals(AssetConstants.FILTER_ENVIRONMENT)) { - mustFilter.put(Constants.TAGS_ENV, entry.getValue()); - mustFilterAsset.put(Constants.TAGS_ENV, entry.getValue()); - } - } - }); + match = new HashMap<>(); + if(Constants.EC2.equals(resourceType)) { + match.put(Constants.STATE_NAME, Constants.RUNNING); + } + if(Constants.VIRTUALMACHINE.equals(resourceType)) { + match.put(Constants.STATUS, Constants.RUNNING); + } + + matchMap = new HashMap<>(); + matchMap.put(Constants.MATCH, match); + mustList.add(matchMap); - mustFilterAsset.put(Constants.LATEST, true); - mustFilterAsset.put(CommonUtils.convertAttributetoKeyword(Constants.STATE_NAME), Constants.RUNNING); - mustNotFilterAsset.put(CommonUtils.convertAttributetoKeyword(Constants.PLATFORM), Constants.WINDOWS); - targetAssetGroup = assetGroup + "/" + Constants.EC2; + parentBool.put("must", mustList); - try { - if (filter.containsKey(AssetConstants.FILTER_PATCHED)) { + match = new HashMap<>(); + if(Constants.EC2.equals(resourceType)) { + match.put("platform", Constants.WINDOWS); + } + if(Constants.VIRTUALMACHINE.equals(resourceType)) { + match.put("osType",Constants.AZURE_WINDOWS); + } + matchMap = new HashMap<>(); + matchMap.put(Constants.MATCH, match); + + parentBool.put("must_not", matchMap); + + Map queryMap = new HashMap<>(); + queryMap.put("bool", parentBool); + + Map parentEntryMap = new LinkedHashMap<>(); + parentEntryMap.put(Constants.TYPE, resourceType); + parentEntryMap.put(AssetConstants.QUERY, queryMap); + mustFilter.put("has_parent", parentEntryMap); + + filter.entrySet().stream().forEach(entry -> { + if (!(entry.getKey().equals(AssetConstants.FILTER_PATCHED) + || entry.getKey().equals(AssetConstants.FILTER_RES_TYPE) + || entry.getKey().equals(AssetConstants.FILTER_EXEC_SPONSOR) + || entry.getKey().equals(AssetConstants.FILTER_DIRECTOR))) { + if (entry.getKey().equals(AssetConstants.FILTER_APPLICATION)) { + mustFilter.put(Constants.TAGS_APPS, entry.getValue()); + mustFilterAsset.put(Constants.TAGS_APPS, entry.getValue()); + } + if (entry.getKey().equals(AssetConstants.FILTER_ENVIRONMENT)) { + mustFilter.put(Constants.TAGS_ENV, entry.getValue()); + mustFilterAsset.put(Constants.TAGS_ENV, entry.getValue()); + } + } + }); + + mustFilterAsset.put(Constants.LATEST, true); + if(Constants.EC2.equals(resourceType)) { + mustFilterAsset.put(CommonUtils.convertAttributetoKeyword(Constants.STATE_NAME), Constants.RUNNING); + mustNotFilterAsset.put(CommonUtils.convertAttributetoKeyword(Constants.PLATFORM), Constants.WINDOWS); + } + if(Constants.VIRTUALMACHINE.equals(resourceType)) { + mustFilterAsset.put(CommonUtils.convertAttributetoKeyword(Constants.STATUS), Constants.RUNNING); + mustNotFilterAsset.put(CommonUtils.convertAttributetoKeyword("osType"), Constants.AZURE_WINDOWS); + } + targetAssetGroup = assetGroup + "/" + resourceType; - unpatchedInstances = esRepository.getDataFromES(assetGroup, null, mustFilter, null, null, null, null); - List unPatchedResourceIds = unpatchedInstances.parallelStream() - .map(obj -> obj.get(Constants.RESOURCEID).toString()).collect(Collectors.toList()); - totalInstances = esRepository.getDataFromES(targetAssetGroup, null, mustFilterAsset, - mustNotFilterAsset, null, null, null); - if (filter.get(AssetConstants.FILTER_PATCHED).equals(AssetConstants.FALSE)) { - assetDetails = totalInstances.parallelStream() - .filter(asset -> unPatchedResourceIds.contains(asset.get(Constants.RESOURCEID))) - .collect(Collectors.toList()); - } else if (filter.get(AssetConstants.FILTER_PATCHED).equals(Constants.TRUE)) { - assetDetails = totalInstances.parallelStream() - .filter(asset -> !unPatchedResourceIds.contains(asset.get(Constants.RESOURCEID))) - .collect(Collectors.toList()); - } - } else { - assetDetails = esRepository.getDataFromES(targetAssetGroup, null, mustFilterAsset, mustNotFilterAsset, - null, source, null); - } - } catch (Exception e) { - LOGGER.error("Error in getListAssetsPatchable", e); - } - return assetDetails; - } + try { + if (filter.containsKey(AssetConstants.FILTER_PATCHED)) { + + unpatchedInstances = esRepository.getDataFromES(assetGroup, null, mustFilter, null, null, null, null); + List unPatchedResourceIds = unpatchedInstances.parallelStream() + .map(obj -> obj.get(Constants.RESOURCEID).toString()).collect(Collectors.toList()); + totalInstances = esRepository.getDataFromES(targetAssetGroup, null, mustFilterAsset, mustNotFilterAsset, + null, null, null); + if (filter.get(AssetConstants.FILTER_PATCHED).equals(AssetConstants.FALSE)) { + assetDetails = totalInstances.parallelStream() + .filter(asset -> unPatchedResourceIds.contains(asset.get(Constants.RESOURCEID))) + .collect(Collectors.toList()); + } else if (filter.get(AssetConstants.FILTER_PATCHED).equals(Constants.TRUE)) { + assetDetails = totalInstances.parallelStream() + .filter(asset -> !unPatchedResourceIds.contains(asset.get(Constants.RESOURCEID))) + .collect(Collectors.toList()); + } + } else { + assetDetails = esRepository.getDataFromES(targetAssetGroup, null, mustFilterAsset, mustNotFilterAsset, + null, source, null); + } + } catch (Exception e) { + LOGGER.error("Error in getListAssetsPatchable", e); + } + return assetDetails; + } private List> getListAssetsPathachableForOnPrem(String assetGroup, Map filter, List source) { @@ -2360,27 +2629,60 @@ private List> formGetListResponse(List fieldNames, L } @Override - public Map getApplicationAssetCountByAssetGroup(String assetGroupName, String domain) - throws DataException { + public List> getDataSourceForTargetTypes(List targetTypes) { + String targetTypeQuery = targetTypes.stream().map(targettype -> "\"" + targettype.trim() + "\"") + .collect(Collectors.joining(",")); + String query = "SELECT dataSourceName as " + Constants.PROVIDER + ", targetName as " + Constants.TYPE + + " FROM cf_Target"; + if (!CollectionUtils.isEmpty(targetTypes)) { + query += " WHERE targetName IN (" + targetTypeQuery + ")"; + } + + return rdsRepository.getDataFromPacman(query); + } + + @Override + public Map getAssetCountAndEnvDistributionByAssetGroup(String aseetGroupName, String type, String application) { - List targetTypes = getTargetTypesByAssetGroup(assetGroupName, domain).stream() - .map(obj -> obj.get(Constants.TYPE).toString()).collect(Collectors.toList()); Map filter = new HashMap<>(); filter.put(Constants.LATEST, Constants.TRUE); filter.put(AssetConstants.UNDERSCORE_ENTITY, Constants.TRUE); - Map mustTermsFilter = new HashMap<>(); - mustTermsFilter.put(AssetConstants.UNDERSCORE_TYPE, targetTypes); - Map applicationMap = new HashMap<>(); - + if (application != null) { + filter.put(Constants.TAGS_APPS, application); + } + + Map countMap = new HashMap<>(); try { - applicationMap = esRepository.getTotalDistributionForIndexAndType(assetGroupName, null, filter, null, null, - Constants.TAGS_APPS, Constants.TEN_THOUSAND, mustTermsFilter); + if (AssetConstants.ALL.equals(type)) { + try { + Map nestedaggs = esRepository.buildAggs(Constants.TAGS_ENV, Constants.THOUSAND, Constants.ENVIRONMENTS, null); + + countMap = esRepository.getEnvAndTotalDistributionForIndexAndType(aseetGroupName, null, filter, null, + null, AssetConstants.UNDERSCORE_TYPE, nestedaggs, Constants.THOUSAND, null); + } catch (Exception e) { + LOGGER.error("Exception in getAssetCountByAssetGroup :", e); + } + } + else { + long count = esRepository.getTotalDocumentCountForIndexAndType(aseetGroupName, type, filter, null, null, + null, null); + Map envMap = esRepository.getTotalDistributionForIndexAndType(aseetGroupName, type, filter, null, null, + Constants.TAGS_ENV, Constants.THOUSAND, null); + + Map countDetails = new HashMap<>(); + countDetails.put(type, count); + Map envDetails = new HashMap<>(); + envDetails.put(type, envMap); + countMap.put(Constants.ASSET_COUNT, countDetails); + countMap.put(Constants.ENV_COUNT, envDetails); + + } + } catch (Exception e) { - LOGGER.error(AssetConstants.ERROR_GETAPPSBYAG, e); - throw new DataException(e); + LOGGER.error("Exception in getAssetCountByAssetGroup :", e); } - return applicationMap; + return countMap; } } diff --git a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/repository/SearchRepositoryImpl.java b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/repository/SearchRepositoryImpl.java index 33a7feb8b..4d8094e6d 100644 --- a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/repository/SearchRepositoryImpl.java +++ b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/repository/SearchRepositoryImpl.java @@ -74,6 +74,8 @@ public class SearchRepositoryImpl implements SearchRepository { private int esPort; @Value("${vulnerability.types}") private String configuredVulnTargetTypes; + @Value("${datasource.types:aws,azure}") + private String dataSourceTypes; @Autowired ElasticSearchRepository esRepository; @@ -347,7 +349,7 @@ public List> fetchTargetTypes(String ag, String searchText, } private List getTypesForDomain(String ag, String domain) { - List> domainData = assetService.getTargetTypesForAssetGroup(ag, domain); + List> domainData = assetService.getTargetTypesForAssetGroup(ag, domain, null); List typesForDomain = new ArrayList<>(); domainData.forEach(domainMap -> { domainMap.forEach((key, value) -> { @@ -465,13 +467,18 @@ private List> getDistributionFromAggResult(String responseJs JsonArray types = resultJson.get("aggregations").getAsJsonObject().get(aggName).getAsJsonObject().get("buckets") .getAsJsonArray(); List> bucketList = new ArrayList<>(); + String dsArray[] = dataSourceTypes.split(","); for (JsonElement type : types) { JsonObject typeObj = type.getAsJsonObject(); String fieldName = typeObj.get("key").getAsString(); // To handle vulnerabilities type - if (fieldName.startsWith("aws_")) { - fieldName = fieldName.substring(4); + + for(String ds : dsArray) { + if (fieldName.startsWith(ds+"_")) { + fieldName = fieldName.substring(ds.length()+1); + break; + } } long count = typeObj.get("doc_count").getAsLong(); diff --git a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/service/AssetService.java b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/service/AssetService.java index aada77a97..1531fae86 100644 --- a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/service/AssetService.java +++ b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/service/AssetService.java @@ -40,7 +40,8 @@ public interface AssetService { * * @return list of type and its asset count. */ - public List> getAssetCountByAssetGroup(String aseetGroupName, String type, String domain); + public List> getAssetCountByAssetGroup(String assetGroup, String type, String domain, + String application, String provider); /** * Fetches all the target types for the particular asset group. If asset @@ -51,7 +52,7 @@ public interface AssetService { * * @return list of target types. */ - public List> getTargetTypesForAssetGroup(String aseetGroupName, String domain); + public List> getTargetTypesForAssetGroup(String aseetGroupName, String domain, String provider); /** * Fetches all the applications for the particular asset group. @@ -485,5 +486,18 @@ public List> getAssetLists(String assetGroup, Map> getDataTypeInfoByTargetType(String resourceId) throws ServiceException; + /** + * Fetches the total count of assets for the particular asset group and distribution of assets based on environment. If no + * type is passed, all the assets of valid target type for the asset group + * is considered., + * + * @param aseetGroupName name of the asset group + * @param type target type + * @param domain the domain of asset group + * + * @return list of type, asset count and env distribution. + */ + public List> getAssetCountAndEnvDistributionByAssetGroup(String assetGroup, String type, String domain, + String application, String provider); } diff --git a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/service/AssetServiceImpl.java b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/service/AssetServiceImpl.java index 7986ba53c..7cd479e2b 100644 --- a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/service/AssetServiceImpl.java +++ b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/service/AssetServiceImpl.java @@ -20,6 +20,7 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.Iterator; @@ -27,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Set; import java.util.StringTokenizer; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -40,6 +42,7 @@ import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import com.google.common.collect.Lists; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; @@ -54,6 +57,7 @@ import com.tmobile.pacman.api.commons.exception.DataException; import com.tmobile.pacman.api.commons.exception.NoDataFoundException; import com.tmobile.pacman.api.commons.exception.ServiceException; +import com.tmobile.pacman.api.commons.utils.CommonUtils; import com.tmobile.pacman.api.commons.utils.PacHttpUtils; /** @@ -80,55 +84,67 @@ public class AssetServiceImpl implements AssetService { String svcCorpPassword; @Override - public List> getAssetCountByAssetGroup(String assetGroup, String type, String domain) { - // TODO : Need to see if its better to get the count based on target - // types in domain. Currently we are fetching everything and filtering - // the unwanted one. - LOGGER.debug("Fetch counts from elastic search"); - - // ES query may possibly return other types as well. - Map countMap = repository.getAssetCountByAssetGroup(assetGroup, type); - - if (AssetConstants.ALL.equals(type)) { - LOGGER.debug("Remove the entries which are not valid types"); - List> targetTypes = getTargetTypesForAssetGroup(assetGroup, domain); - List validTypes = targetTypes.stream().map(obj -> obj.get(Constants.TYPE).toString()) - .collect(Collectors.toList()); - List countTypes = new ArrayList<>(countMap.keySet()); - for (String _type : validTypes) { - if (!countMap.containsKey(_type)) { - countMap.put(_type, 0L); - } - } - - for (String _type : countTypes) { - if (!validTypes.contains(_type)) { - countMap.remove(_type); - } - } - } - - LOGGER.debug("Creating response objects "); - List> countList = new ArrayList<>(); - countMap.entrySet().stream().forEach(entry -> { - Map typeMap = new HashMap<>(); - typeMap.put(Constants.TYPE, entry.getKey()); - typeMap.put(Constants.COUNT, entry.getValue()); - countList.add(typeMap); - }); - - return countList; - } + public List> getAssetCountByAssetGroup(String assetGroup, String type, String domain, + String application, String provider) { + + LOGGER.debug("Fetch counts from elastic search"); + + // ES query may possibly return other types as well. + Map countMap = repository.getAssetCountByAssetGroup(assetGroup, type, application); + List validTypes = Lists.newArrayList(); + if (AssetConstants.ALL.equals(type)) { + LOGGER.debug("Remove the entries which are not valid types"); + List> targetTypes = getTargetTypesForAssetGroup(assetGroup, domain, provider); + validTypes = targetTypes.stream().map(obj -> obj.get(Constants.TYPE).toString()) + .collect(Collectors.toList()); + List countTypes = new ArrayList<>(countMap.keySet()); + for (String _type : validTypes) { + if (!countMap.containsKey(_type)) { + countMap.put(_type, 0L); + } + } + for (String _type : countTypes) { + if (!validTypes.contains(_type)) { + countMap.remove(_type); + } + } + }else { + validTypes.add(type); + } + + List> datasourceForAssettypes = repository.getDataSourceForTargetTypes(validTypes); + + LOGGER.debug("Creating response objects "); + List> countList = new ArrayList<>(); + countMap.entrySet().stream().forEach(entry -> { + if (!Integer.valueOf(entry.getValue().toString()).equals(0)) { + Map typeMap = new HashMap<>(); + + String providerInfo = datasourceForAssettypes.stream() + .filter(data -> data.get(Constants.TYPE).equals(entry.getKey())).findFirst().get() + .get(Constants.PROVIDER).toString(); + + typeMap.put(Constants.TYPE, entry.getKey()); + typeMap.put(Constants.COUNT, entry.getValue()); + typeMap.put(Constants.PROVIDER, providerInfo); + countList.add(typeMap); + } + }); + + return countList; + } @Override - @Cacheable(cacheNames = "assets", unless = "#result == null") - public List> getTargetTypesForAssetGroup(String assetGroup, String domain) { - if (Constants.AWS.equals(assetGroup)) { - return repository.getAllTargetTypes(); - } else { - return repository.getTargetTypesByAssetGroup(assetGroup, domain); - } - } + @Cacheable(cacheNames = "assets", unless = "#result == null") + public List> getTargetTypesForAssetGroup(String assetGroup, String domain, String provider) { + if (Constants.AWS.equals(assetGroup) || Constants.AZURE.equals(assetGroup) ) { + return repository.getAllTargetTypes(assetGroup); + } else if (Constants.MASTER_ALIAS.equals(assetGroup) || Constants.ROOT_ALIAS.equals(assetGroup)) { + return repository.getAllTargetTypes(null); + }else { + return repository.getTargetTypesByAssetGroup(assetGroup, domain, provider); + } + } @Override public List> getApplicationsByAssetGroup(String assetGroup, String domain) throws DataException { @@ -182,23 +198,53 @@ public List> getAllAssetGroups() { } @Override - public Map getAssetGroupInfo(String assetGroup) { - Map assetGroupInfoMap = repository.getAssetGroupInfo(assetGroup); - if (!assetGroupInfoMap.isEmpty()) { - List applications = new ArrayList<>(); - try { - applications = repository.getApplicationByAssetGroup(assetGroup, null); - } catch (Exception e) { - LOGGER.error("Error in getAssetGroupInfo " , e); - } - assetGroupInfoMap.put("appcount", applications.size()); - List> countMap = getAssetCountByAssetGroup(assetGroup, AssetConstants.ALL, null); - assetGroupInfoMap.put("assetcount", - countMap.stream().mapToLong(obj -> Long.valueOf(obj.get(Constants.COUNT).toString())).sum()); - assetGroupInfoMap.put("domains", getDomains(assetGroup)); - } - return assetGroupInfoMap; - } + public Map getAssetGroupInfo(String assetGroup) { + Map assetGroupInfoMap = repository.getAssetGroupInfo(assetGroup); + if (!assetGroupInfoMap.isEmpty()) { + List applications = new ArrayList<>(); + try { + applications = repository.getApplicationByAssetGroup(assetGroup, null); + } catch (Exception e) { + LOGGER.error("Error in getAssetGroupInfo ", e); + } + assetGroupInfoMap.put("appcount", applications.size()); + List> countMap = getAssetCountByAssetGroup(assetGroup, AssetConstants.ALL, null, null, null); + assetGroupInfoMap.put("assetcount", + countMap.stream().mapToLong(obj -> Long.valueOf(obj.get(Constants.COUNT).toString())).sum()); + assetGroupInfoMap.put("domains", getDomains(assetGroup)); + assetGroupInfoMap.put(Constants.PROVIDERS, getProviderWithTypeCount(assetGroup,countMap)); + } + return assetGroupInfoMap; + } + + /** + * Function for getting the provider details along with the target type count + * + * @param countMap + * @return + */ + private List> getProviderWithTypeCount (String assetGroup,List> countMap) { + List> providersData = new ArrayList<>(); + + Map providerMap = countMap.stream().collect(Collectors.groupingBy(countObj-> countObj.get(Constants.PROVIDER).toString(), Collectors.counting())); + + if(providerMap.isEmpty()) { + List> targetTypes = repository.getTargetTypesByAssetGroup(assetGroup, "Infra & Platforms", null); + List validTypes = targetTypes.stream().map(obj -> obj.get(Constants.TYPE).toString()) + .collect(Collectors.toList()); + List> datasourceForAssettypes = repository.getDataSourceForTargetTypes(validTypes); + Set mappedProviders = datasourceForAssettypes.stream().map(obj->obj.get(Constants.PROVIDER).toString()).collect(Collectors.toSet()); + mappedProviders.forEach(provider->providerMap.put(provider,0L)); + } + + providerMap.forEach((k,v)-> { + Map newProvider = new HashMap(); + newProvider.put(Constants.PROVIDER,k); + newProvider.put(Constants.TYPE_COUNT, v); + providersData.add(newProvider); + }); + return providersData; + } @Override public List> getAssetCountByApplication(String assetGroup, String type) throws DataException { @@ -949,4 +995,127 @@ public List> getDataTypeInfoByTargetType(String resourceType } return dataTypeList; } + + @Override + public List> getAssetCountAndEnvDistributionByAssetGroup(String assetGroup, String type, + String domain, String application, String provider) { + + LOGGER.debug("Fetch counts from elastic search"); + + // ES query may possibly return other types as well. + Map distribution = repository.getAssetCountAndEnvDistributionByAssetGroup(assetGroup, type, application); + + Map countMap = (Map) distribution.get(Constants.ASSET_COUNT); + Map envMap = (Map) distribution.get(Constants.ENV_COUNT); + + List validTypes = Lists.newArrayList(); + if (AssetConstants.ALL.equals(type)) { + LOGGER.debug("Remove the entries which are not valid types"); + List> targetTypes = getTargetTypesForAssetGroup(assetGroup, domain, provider); + validTypes = targetTypes.stream().map(obj -> obj.get(Constants.TYPE).toString()) + .collect(Collectors.toList()); + List countTypes = new ArrayList<>(countMap.keySet()); + for (String _type : validTypes) { + if (!countMap.containsKey(_type)) { + countMap.put(_type, 0L); + } + } + for (String _type : countTypes) { + if (!validTypes.contains(_type)) { + countMap.remove(_type); + } + } + }else { + validTypes.add(type); + } + + List> datasourceForAssettypes = repository.getDataSourceForTargetTypes(validTypes); + + LOGGER.debug("Creating response objects "); + List> countList = new ArrayList<>(); + countMap.entrySet().stream().forEach(entry -> { + if (!Integer.valueOf(entry.getValue().toString()).equals(0)) { + Map typeMap = new HashMap<>(); + + String providerInfo = datasourceForAssettypes.stream() + .filter(data -> data.get(Constants.TYPE).equals(entry.getKey())).findFirst().get() + .get(Constants.PROVIDER).toString(); + + Long totalCount = entry.getValue(); + + typeMap.put(Constants.TYPE, entry.getKey()); + typeMap.put(Constants.COUNT, totalCount); + typeMap.put(Constants.PROVIDER, providerInfo); + + List> envDistribution = calculateEnvironmentDistribution((Map) envMap.get(entry.getKey()), totalCount); + + typeMap.put(Constants.ENVIRONMENTS, envDistribution); + + countList.add(typeMap); + } + }); + + return countList; + } + + /* + * categorise the environment tags to different env like dev, stg, prod and calculate the percentage for each env + * + * assets for which the tag is not present will be categoried under Nil category + * + * asset types for which tag is not applicable will return empty list + * + */ + private List> calculateEnvironmentDistribution(Map envDetails, Long totalCount){ + List> envDistribution = new ArrayList<>(); + + if (!envDetails.isEmpty()) { + //categorise env based on env tag + Map envCategories = new HashMap<>(); + envDetails.entrySet().stream().forEach(environment -> { + String env = CommonUtils.getEnvironmentForTag(environment.getKey()); + Long count = environment.getValue(); + if (envCategories.containsKey(env)) { + count = count + envCategories.get(env); + } + envCategories.put(env, count); + }); + //calculate % for each env + envCategories.entrySet().stream().forEach(environment -> { + Map map = new HashMap<>(); + map.put(Constants.ENV, environment.getKey()); + String percentage = String.format("%2.1f%%", ((float) environment.getValue() / totalCount * 100)); + map.put(Constants.PERCENTAGE, percentage); + envDistribution.add(map); + }); + + //get untagged asset count + Long bucketTotal = envDetails.entrySet().stream() + .collect(Collectors.summarizingLong(map -> (Long) map.getValue())).getSum(); + + if ((totalCount - bucketTotal) > 0) { + Map map = new HashMap<>(); + map.put(Constants.ENV, Constants.UNTAGGED_ENV); + String percentage = String.format("%2.1f%%", ((float) (totalCount - bucketTotal) / totalCount * 100)); + map.put(Constants.PERCENTAGE, percentage); + envDistribution.add(map); + } + + Map envOrder = getEnvDistributionOrder(); + + envDistribution.sort(Comparator.comparing((Map env) -> envOrder.get(env.get(Constants.ENV)))); + } + return envDistribution; + } + + private Map getEnvDistributionOrder() { + Map envOrder = new HashMap<>(); + envOrder.put(Constants.PRODUCTION_ENV, 1); + envOrder.put(Constants.STAGE_ENV, 2); + envOrder.put(Constants.DEV_ENV, 3); + envOrder.put(Constants.NPE_ENV, 4); + envOrder.put(Constants.OTHER_ENV, 5); + envOrder.put(Constants.UNTAGGED_ENV, 6); + return envOrder; + } } diff --git a/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/controller/AssetControllerTest.java b/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/controller/AssetControllerTest.java index ce1394bdd..2f38d8fb6 100644 --- a/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/controller/AssetControllerTest.java +++ b/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/controller/AssetControllerTest.java @@ -54,10 +54,10 @@ public class AssetControllerTest { public void testgetListOfTargetTypes() throws Exception { List> tTypeList = new ArrayList<>(); - when(service.getTargetTypesForAssetGroup(anyObject(), anyObject())).thenReturn(tTypeList); + when(service.getTargetTypesForAssetGroup(anyObject(), anyObject(), anyObject() )).thenReturn(tTypeList); ReflectionTestUtils.setField(controller, "assetService", service); - ResponseEntity responseObj0 = controller.getListOfTargetTypes("ag", "domain"); + ResponseEntity responseObj0 = controller.getListOfTargetTypes("ag", "domain", "provider"); assertTrue(responseObj0.getStatusCode() == HttpStatus.EXPECTATION_FAILED); Map tTypeMap = new HashMap<>(); @@ -66,7 +66,7 @@ public void testgetListOfTargetTypes() throws Exception { tTypeMap.put("domain", "Infra & Platforms"); tTypeList.add(tTypeMap); - ResponseEntity responseObj = controller.getListOfTargetTypes("ag", "domain"); + ResponseEntity responseObj = controller.getListOfTargetTypes("ag", "domain", "provider"); assertTrue(responseObj.getStatusCode() == HttpStatus.OK); assertTrue(((Map) responseObj.getBody()).get("data") != null); } diff --git a/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/controller/AssetCountControllerTest.java b/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/controller/AssetCountControllerTest.java index b8d12f28b..697fb7bc4 100644 --- a/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/controller/AssetCountControllerTest.java +++ b/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/controller/AssetCountControllerTest.java @@ -53,10 +53,10 @@ public class AssetCountControllerTest { public void testgeAssetCount() throws Exception { List> tTypeList = new ArrayList<>(); - when(service.getAssetCountByAssetGroup(anyObject(), anyObject(), anyObject())).thenReturn(tTypeList); + when(service.getAssetCountByAssetGroup(anyObject(), anyObject(), anyObject(), anyObject(), anyObject())).thenReturn(tTypeList); ReflectionTestUtils.setField(controller, "assetService", service); - ResponseEntity responseObj3 = controller.geAssetCount("ag", "type", "domain"); + ResponseEntity responseObj3 = controller.geAssetCount("ag", "type", "domain", null, null); assertTrue(responseObj3.getStatusCode() == HttpStatus.EXPECTATION_FAILED); @@ -65,15 +65,15 @@ public void testgeAssetCount() throws Exception { tTypeMap.put("type", "ec2"); tTypeList.add(tTypeMap); - when(service.getAssetCountByAssetGroup(anyObject(), anyObject(), anyObject())).thenReturn(tTypeList); + when(service.getAssetCountByAssetGroup(anyObject(), anyObject(), anyObject(), anyObject(), anyObject())).thenReturn(tTypeList); ReflectionTestUtils.setField(controller, "assetService", service); - ResponseEntity responseObj = controller.geAssetCount("ag", "type", "domain"); + ResponseEntity responseObj = controller.geAssetCount("ag", "type", "domain", null, null); assertTrue(responseObj.getStatusCode() == HttpStatus.OK); assertTrue(((Map) responseObj.getBody()).get("data") != null); - ResponseEntity responseObj2 = controller.geAssetCount("ag", null, "domain"); + ResponseEntity responseObj2 = controller.geAssetCount("ag", null, "domain", null, null); assertTrue(responseObj2.getStatusCode() == HttpStatus.OK); } diff --git a/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/controller/AssetListControllerTest.java b/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/controller/AssetListControllerTest.java index f7dd5ffe7..93155f76f 100644 --- a/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/controller/AssetListControllerTest.java +++ b/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/controller/AssetListControllerTest.java @@ -298,7 +298,7 @@ public void testgetEditableFieldsByTargetType() throws Exception{ aMap.put("type", "ec2"); aList.add(aMap); - when(service.getTargetTypesForAssetGroup(anyString(),anyString())).thenReturn(aList); + when(service.getTargetTypesForAssetGroup(anyString(),anyString(),anyString())).thenReturn(aList); ReflectionTestUtils.setField(controller, "assetService", service); ResponseEntity responseObj2 = controller.getEditableFieldsByTargetType("ag","ec2"); assertTrue(responseObj2.getStatusCode()==HttpStatus.OK); diff --git a/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/controller/UtilTest.java b/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/controller/UtilTest.java index 65f6c49e8..a0884bf79 100644 --- a/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/controller/UtilTest.java +++ b/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/controller/UtilTest.java @@ -17,6 +17,7 @@ import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.anyObject; +import static org.mockito.Matchers.anyString; import static org.powermock.api.mockito.PowerMockito.doThrow; import static org.powermock.api.mockito.PowerMockito.when; @@ -52,14 +53,14 @@ public void testisValidTargetType() throws Exception { tTypeMap.put("domain", "Infra & Platforms"); tTypeList.add(tTypeMap); - when(service.getTargetTypesForAssetGroup(anyObject(), anyObject())).thenReturn(tTypeList); + when(service.getTargetTypesForAssetGroup(anyObject(), anyObject(), anyString())).thenReturn(tTypeList); ReflectionTestUtils.setField(Util.class, "assetService", service); boolean valid = Util.isValidTargetType("aws-all", "ec2"); assertTrue(valid); - doThrow(new NullPointerException()).when(service).getTargetTypesForAssetGroup(anyObject(), anyObject()); + doThrow(new NullPointerException()).when(service).getTargetTypesForAssetGroup(anyObject(), anyObject(), anyString()); valid = Util.isValidTargetType("aws-all", "ec2"); assertTrue(!valid); diff --git a/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/repository/AssetRepositoryTest.java b/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/repository/AssetRepositoryTest.java index 12898a77c..88e0e46ff 100644 --- a/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/repository/AssetRepositoryTest.java +++ b/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/repository/AssetRepositoryTest.java @@ -84,7 +84,7 @@ public void testGetAssetCountByAssetGroupForTypeAll() throws Exception { ReflectionTestUtils.setField(repository, "esRepository", elasticSearchRepository); Map countMap = new HashMap<>(); - countMap = repository.getAssetCountByAssetGroup("aws-all", "all"); + countMap = repository.getAssetCountByAssetGroup("aws-all", "all", null); assertTrue(countMap.size() > 2); } @@ -96,7 +96,7 @@ public void testGetAssetCountByAssetGroupForTypeOtherThanAll() throws Exception ReflectionTestUtils.setField(repository, "esRepository", elasticSearchRepository); Map countMap = new HashMap<>(); - countMap = repository.getAssetCountByAssetGroup("aws-all", "s3"); + countMap = repository.getAssetCountByAssetGroup("aws-all", "s3", null); assertEquals(1, countMap.size()); } @@ -107,7 +107,7 @@ public void testGetAssetCountByAssetGroupForInvalidAG() throws Exception { ReflectionTestUtils.setField(repository, "esRepository", elasticSearchRepository); Map countMap = new HashMap<>(); - countMap = repository.getAssetCountByAssetGroup("invalid-ag", "s3"); + countMap = repository.getAssetCountByAssetGroup("invalid-ag", "s3", null); assertEquals(1, countMap.size()); assertEquals(0, countMap.get("s3").longValue()); } @@ -119,7 +119,7 @@ public void testGetAssetCountByAssetGroupForInvalidType() throws Exception { ReflectionTestUtils.setField(repository, "esRepository", elasticSearchRepository); Map countMap = new HashMap<>(); - countMap = repository.getAssetCountByAssetGroup("aws-all", "invalid-type"); + countMap = repository.getAssetCountByAssetGroup("aws-all", "invalid-type", null); assertEquals("0", countMap.get("invalid-type").toString()); } @@ -131,7 +131,7 @@ public void testGetTargetTypesByAssetGroupForInvalidDomain() { when(pacmanRdsRepository.getDataFromPacman(anyString())).thenReturn(tTypeList); ReflectionTestUtils.setField(repository, "rdsRepository", pacmanRdsRepository); - List> targetTypesList = repository.getTargetTypesByAssetGroup("aws-all", "invalid-domain"); + List> targetTypesList = repository.getTargetTypesByAssetGroup("aws-all", "invalid-domain", null); assertEquals(0, targetTypesList.size()); @@ -154,7 +154,7 @@ public void testGetTargetTypesByAssetGroupForInfraDomain() { ReflectionTestUtils.setField(repository, "rdsRepository", pacmanRdsRepository); List> targetTypesList = repository.getTargetTypesByAssetGroup("aws-all", - "Infra & Platforms"); + "Infra & Platforms", null); assertTrue(targetTypesList.size() > 1); } @@ -174,7 +174,7 @@ public void testGetAllTargetTypes() { when(pacmanRdsRepository.getDataFromPacman(anyString())).thenReturn(tTypeList); ReflectionTestUtils.setField(repository, "rdsRepository", pacmanRdsRepository); - List> allTypes = repository.getAllTargetTypes(); + List> allTypes = repository.getAllTargetTypes("*"); assertTrue(allTypes.size() > 1); } diff --git a/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/repository/SearchRepositoryTest.java b/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/repository/SearchRepositoryTest.java index 34ac29322..97dcd5295 100644 --- a/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/repository/SearchRepositoryTest.java +++ b/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/repository/SearchRepositoryTest.java @@ -120,7 +120,7 @@ public void testgetAssetCountByAssetGroup() throws Exception { when(pacmanRdsRepository.getDataFromPacman(anyString())).thenReturn(returnFieldsData); ReflectionTestUtils.setField(repository, "rdsRepository", pacmanRdsRepository); - when(assetService.getTargetTypesForAssetGroup(anyString(), anyString())).thenReturn(tTypeList); + when(assetService.getTargetTypesForAssetGroup(anyString(), anyString(), anyString())).thenReturn(tTypeList); ReflectionTestUtils.setField(repository, "assetService", assetService); when(elasticSearchRepository.buildQuery(anyObject(), anyObject(), anyObject(), anyObject(), anyObject(), anyObject())).thenReturn(queryMap); @@ -197,7 +197,7 @@ public void testfetchTargetTypes() throws Exception { Matchers.
anyVararg())).thenReturn(response); ReflectionTestUtils.setField(repository, "restClient", restClient); - when(assetService.getTargetTypesForAssetGroup(anyString(), anyString())).thenReturn(tTypeList); + when(assetService.getTargetTypesForAssetGroup(anyString(), anyString(), anyString())).thenReturn(tTypeList); ReflectionTestUtils.setField(repository, "assetService", assetService); List> x = repository.fetchTargetTypes("aws-all", "pacman", "Assets", "", true); diff --git a/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/service/AssetServiceTest.java b/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/service/AssetServiceTest.java index 0dc85a98a..562597653 100644 --- a/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/service/AssetServiceTest.java +++ b/api/pacman-api-asset/src/test/java/com/tmobile/pacman/api/asset/service/AssetServiceTest.java @@ -41,6 +41,7 @@ import com.tmobile.pacman.api.asset.domain.ResponseWithFieldsByTargetType; import com.tmobile.pacman.api.asset.repository.AssetRepository; import com.tmobile.pacman.api.asset.repository.PacmanRedshiftRepository; +import com.tmobile.pacman.api.commons.Constants; import com.tmobile.pacman.api.commons.repo.ElasticSearchRepository; import com.tmobile.pacman.api.commons.repo.PacmanRdsRepository; import com.tmobile.pacman.api.commons.utils.PacHttpUtils; @@ -90,13 +91,24 @@ public void testgetAssetCountByAssetGroup() throws Exception { mockMap.put("s3", (long) 655); mockMap.put("stack", (long) 655); - when(assetRepository.getAllTargetTypes()).thenReturn(tTypeList); - when(assetRepository.getTargetTypesByAssetGroup(anyObject(), anyObject())).thenReturn(tTypeList); - when(assetRepository.getAssetCountByAssetGroup(anyObject(), anyObject())).thenReturn(mockMap); + List> typeDataSource = new ArrayList<>(); + Map dataSource = new HashMap<>(); + dataSource.put(Constants.TYPE, "ec2"); + dataSource.put(Constants.PROVIDER, "aws"); + typeDataSource.add(dataSource); + dataSource = new HashMap<>(); + dataSource.put(Constants.TYPE, "s3"); + dataSource.put(Constants.PROVIDER, "aws"); + typeDataSource.add(dataSource); + + when(assetRepository.getAllTargetTypes(anyString())).thenReturn(tTypeList); + when(assetRepository.getTargetTypesByAssetGroup(anyObject(), anyObject(), anyObject())).thenReturn(tTypeList); + when(assetRepository.getAssetCountByAssetGroup(anyObject(), anyObject(), anyObject())).thenReturn(mockMap); + when(assetRepository.getDataSourceForTargetTypes(anyObject())).thenReturn(typeDataSource); ReflectionTestUtils.setField(service, "repository", assetRepository); List> listOfCountMaps = new ArrayList<>(); - listOfCountMaps = service.getAssetCountByAssetGroup("testAg", "all", "testDomain"); + listOfCountMaps = service.getAssetCountByAssetGroup("aws-all", "all", "Infra & Platforms", null, null); assertTrue(listOfCountMaps.size() == 2); } diff --git a/api/pacman-api-statistics/src/main/java/com/tmobile/pacman/api/statistics/repository/StatisticsRepositoryImpl.java b/api/pacman-api-statistics/src/main/java/com/tmobile/pacman/api/statistics/repository/StatisticsRepositoryImpl.java index b3b1da9fb..d42d2951f 100644 --- a/api/pacman-api-statistics/src/main/java/com/tmobile/pacman/api/statistics/repository/StatisticsRepositoryImpl.java +++ b/api/pacman-api-statistics/src/main/java/com/tmobile/pacman/api/statistics/repository/StatisticsRepositoryImpl.java @@ -78,9 +78,6 @@ public class StatisticsRepositoryImpl implements StatisticsRepository, Constants /** The Constant LOGGER. */ private static final Logger LOGGER = LoggerFactory.getLogger(StatisticsRepositoryImpl.class); - /** The Constant AG_NAME_AWS. */ - private static final String AG_NAME_AWS = "aws"; - /** The Constant PROTOCOL. */ private static final String PROTOCOL = "http"; @@ -201,7 +198,7 @@ public String getNumberOfPoliciesEvaluated() throws DataException { public JsonArray getTotalViolations() throws DataException { try { JsonParser parser = new JsonParser(); - StringBuilder urlToQueryBuffer = new StringBuilder(esUrl).append("/").append(AG_NAME_AWS).append("/") + StringBuilder urlToQueryBuffer = new StringBuilder(esUrl).append("/").append(MASTER_ALIAS).append("/") .append(SEARCH); StringBuilder requestBody = new StringBuilder( "{\"query\":{\"bool\":{\"must\":[{\"term\":{\"issueStatus.keyword\":{\"value\":\"open\"}}},{\"term\":{\"type.keyword\":{\"value\":\"issue\"}}}]}},\"aggs\":{\"severity\":{\"terms\":{\"field\":\"severity.keyword\",\"size\":10000}}}}"); diff --git a/api/pacman-api-statistics/src/main/java/com/tmobile/pacman/api/statistics/service/StatisticsServiceImpl.java b/api/pacman-api-statistics/src/main/java/com/tmobile/pacman/api/statistics/service/StatisticsServiceImpl.java index bd4f2fbb9..9c0fbe104 100644 --- a/api/pacman-api-statistics/src/main/java/com/tmobile/pacman/api/statistics/service/StatisticsServiceImpl.java +++ b/api/pacman-api-statistics/src/main/java/com/tmobile/pacman/api/statistics/service/StatisticsServiceImpl.java @@ -72,9 +72,6 @@ public class StatisticsServiceImpl implements StatisticsService, Constants { @Autowired(required=false) private HeimdallElasticSearchRepository heimdallElasticSearchRepository; - /** The Constant AG_AWS_ALL. */ - private static final String AWS = "aws"; - /** The Constant LOGGER. */ private static final Logger LOGGER = LoggerFactory.getLogger(StatisticsServiceImpl.class); @@ -200,7 +197,7 @@ public List> getStats() throws Exception { Long totalAssets = getTotalAssetCount(); Long eventsProcessed = getTotalEventProcessed(); Map violationsMap = getIssueDistribution(); - String targettypes = repository.getTargetTypeForAG(AWS, null); + String targettypes = repository.getTargetTypeForAG(MASTER_ALIAS, null); ExecutorService executor = Executors.newCachedThreadPool(); executor.execute(() -> { numberOfPoliciesEnforced = getNumberOfPoliciesEnforced(targettypes); @@ -267,8 +264,7 @@ private Map getIssueDistribution() { JsonParser parser = new JsonParser(); try { LOGGER.info("before the client call {}",complianceClient.toString()); - LOGGER.info("before the client call "+complianceClient.toString()); - String distributionStr = complianceClient.getDistributionAsJson(AWS, null); + String distributionStr = complianceClient.getDistributionAsJson(MASTER_ALIAS, null); LOGGER.info("after the client call {}",complianceClient.toString()); if (!Strings.isNullOrEmpty(distributionStr)) { JsonObject responseDetailsjson = parser.parse(distributionStr).getAsJsonObject(); @@ -355,7 +351,7 @@ private Long getTotalAssetCount() { JsonParser parser = new JsonParser(); try{ LOGGER.debug("before the client call",assetClient.toString()); - Map assetCounts = assetClient.getTypeCounts(AWS, null, null); + Map assetCounts = assetClient.getTypeCounts(MASTER_ALIAS, null, null); LOGGER.debug("after the client call",assetClient.toString()); // Get Total Asset Count assetCounts.entrySet().stream().forEach(entry->{ diff --git a/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/Constants.java b/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/Constants.java index 43e9b51db..c9be6f349 100644 --- a/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/Constants.java +++ b/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/Constants.java @@ -297,5 +297,29 @@ public interface Constants { String VIRTUALMACHINE = "virtualmachine"; String VIRTUALMACHINE_QUALYS_RULEID = "PacMan_Ec2InstanceScannedByQualys_version-1_VmInstanceScannedByQualys_virtualmachine"; String ONPREM_QUALYS_RULEID = "PacMan_Onprem-asset-scanned-by-qualys-API_version-1_OnpremassetscannedbyqualysAPI_onpremserver"; + String AZURE = "azure"; + String MASTER_ALIAS = "ds-all"; + String ROOT_ALIAS = "*"; + String CLOUD_TYPE_KEYWORD = "_cloudType.keyword"; + String AGGS_NAME_PROVIDERS = "providers"; + String APPLICATION_COUNT = "applicationCount"; + String APPLICATION_PROVIDERS = "applicationproviders"; + String TYPE_COUNT = "typecount"; + String PROVIDERS = "providers"; + String ENVIRONMENTS = "environments"; + String ENV_COUNT = "envCount"; + String PRODUCTION_ENV = "Prod"; + String STAGE_ENV = "Stage"; + String DEV_ENV = "Dev"; + String NPE_ENV = "NPE"; + String OTHER_ENV = "Others"; + String UNTAGGED_ENV = "Untagged"; + String PERCENTAGE="percentage"; + String PROD_PATTERN = "^(((prod)(uction)?)|((prd).*))(:+(((prod)(uction)?)|((prd).*)))?"; + String STG_PATTERN = "(^(stag|stg).*)|(.*:+(stag|stg).*)"; + String DEV_PATTERN = "(^(dev|development).*)|(.*:+(dev).*)"; + String NPE_PATTERN = "(^(npe|non.?prod(uction)?))|(.*:+(npe).*)"; + String AZURE_WINDOWS = "Windows"; + String CLOUD_KERNEL_COMPLIANCE_POLICY = "PacMan_cloud-kernel-compliance_version-1"; } diff --git a/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/repo/ElasticSearchRepository.java b/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/repo/ElasticSearchRepository.java index 223d3526e..867594184 100644 --- a/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/repo/ElasticSearchRepository.java +++ b/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/repo/ElasticSearchRepository.java @@ -347,6 +347,32 @@ private Map buildAggs(String distributionName, int size) { } return name; } + + /** + * + * @param distributionName + * @param size + * @param aggsName + * @param nestedAggs + * @return + */ + public Map buildAggs(String distributionName, int size, String aggsName, Map nestedAggs) { + Map name = new HashMap(); + if (!Strings.isNullOrEmpty(distributionName)) { + Map terms = new HashMap(); + Map termDetails = new HashMap(); + termDetails.put("field", distributionName); + if (size > 0) { + termDetails.put(SIZE, size); + } + terms.put(TERMS, termDetails); + if (nestedAggs != null && !nestedAggs.isEmpty()) { + terms.put(AGGS, nestedAggs); + } + name.put(( Strings.isNullOrEmpty(aggsName) ? "name" : aggsName ), terms); + } + return name; + } /** * @@ -684,13 +710,13 @@ public Map getTotalDistributionForIndexAndType(String index, Strin } /** - * + * * @param url * @param index * @param type * @return */ - private String buildAggsURL(String url, String index, String type) { + public String buildAggsURL(String url, String index, String type) { StringBuilder urlToQuery = new StringBuilder(url).append(FORWARD_SLASH).append(index); if (!Strings.isNullOrEmpty(type)) { @@ -1633,4 +1659,80 @@ public String buildESURL(String url, String index, String type, int size, int fr return urlToQuery.toString(); } + /** + * + * @param index + * @param type + * @param mustFilter + * @param mustNotFilter + * @param shouldFilter + * @param aggsFilter + * @param size + * @param mustTermsFilter + * @return + * @throws Exception + */ + public Map getEnvAndTotalDistributionForIndexAndType(String index, String type, + Map mustFilter, Map mustNotFilter, + HashMultimap shouldFilter, String aggsFilter, Map nestedaggs, int size, Map mustTermsFilter) + throws Exception { + + String urlToQuery = buildAggsURL(esUrl, index, type); + Map requestBody = new HashMap(); + Map matchFilters = Maps.newHashMap(); + Map distribution = new HashMap<>(); + Map countMap = new HashMap<>(); + Map envMap = new HashMap<>(); + + if (mustFilter == null) { + matchFilters.put("match_all", new HashMap()); + } else { + matchFilters.putAll(mustFilter); + } + if (null != mustFilter) { + requestBody.put(QUERY, buildQuery(matchFilters, mustNotFilter, shouldFilter, null, mustTermsFilter,null)); + requestBody.put(AGGS, buildAggs(aggsFilter, size, null, nestedaggs)); + + if (!Strings.isNullOrEmpty(aggsFilter)) { + requestBody.put(SIZE, "0"); + } + + } else { + requestBody.put(QUERY, matchFilters); + } + String responseDetails = null; + Gson gson = new GsonBuilder().create(); + + try { + String requestJson = gson.toJson(requestBody, Object.class); + responseDetails = PacHttpUtils.doHttpPost(urlToQuery, requestJson); + Map response = (Map) gson.fromJson(responseDetails, Map.class); + Map aggregations = (Map) response.get(AGGREGATIONS); + Map name = (Map) aggregations.get(NAME); + List> buckets = (List>) name.get(BUCKETS); + + for (int i = 0; i < buckets.size(); i++) { + Map bucket = buckets.get(i); + countMap.put(bucket.get("key").toString(), ((Double) bucket.get("doc_count")).longValue()); + + Map enviroments = (Map) bucket.get(ENVIRONMENTS); + List> envBuckets = (List>) enviroments.get(BUCKETS); + + Map environments = new HashMap<>(); + for(int j=0; j< envBuckets.size(); j++) { + Map env = envBuckets.get(j); + environments.put(env.get("key").toString(), ((Double) env.get("doc_count")).longValue()); + } + envMap.put(bucket.get("key").toString(), environments); + } + distribution.put(Constants.ASSET_COUNT, countMap); + distribution.put(Constants.ENV_COUNT, envMap); + + } catch (Exception e) { + LOGGER.error(ERROR_RETRIEVING_INVENTORY_FROM_ES, e); + throw e; + } + return distribution; + } + } diff --git a/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/utils/CommonUtils.java b/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/utils/CommonUtils.java index 1e9518063..296fdc2f1 100644 --- a/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/utils/CommonUtils.java +++ b/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/utils/CommonUtils.java @@ -42,6 +42,7 @@ import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; +import com.tmobile.pacman.api.commons.Constants; public class CommonUtils { @@ -370,4 +371,32 @@ public static String capitailizeWord(String mainStr) { } return s.toString().trim(); } + + /** + * returns environment from the env tag based on the regex match - + * prod/stg/dev/npe/others + * + * prod - either starts with production, prod or prd or if it + * is after ":" + * + * stg - starts with stg or stag or after ":" + * + * dev - starts with dev or development or after ":" + * + * npe - starts with npe or non production or after ":" + */ + public static String getEnvironmentForTag(String key) { + + if (key.toLowerCase().matches(Constants.PROD_PATTERN)) { + return Constants.PRODUCTION_ENV; + } else if (key.toLowerCase().matches(Constants.STG_PATTERN)) { + return Constants.STAGE_ENV; + } else if (key.toLowerCase().matches(Constants.DEV_PATTERN)) { + return Constants.DEV_ENV; + } else if (key.toLowerCase().matches(Constants.NPE_PATTERN)) { + return Constants.NPE_ENV; + } else { + return Constants.OTHER_ENV; + } + } } From 89dece9d76ab8da3ed0f2df6703ceffd27e0e8e9 Mon Sep 17 00:00:00 2001 From: Kanchana Date: Mon, 4 Nov 2019 17:56:16 +0530 Subject: [PATCH 035/107] security center rule --- .../AzureSecurityCenterRule.java | 89 +++++++++++++++++++ .../cloud/awsrules/utils/PacmanUtils.java | 48 ++++++++++ .../cloud/constants/PacmanRuleConstants.java | 4 + 3 files changed, 141 insertions(+) create mode 100644 jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/awsrules/securitycenter/AzureSecurityCenterRule.java diff --git a/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/awsrules/securitycenter/AzureSecurityCenterRule.java b/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/awsrules/securitycenter/AzureSecurityCenterRule.java new file mode 100644 index 000000000..a95cca290 --- /dev/null +++ b/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/awsrules/securitycenter/AzureSecurityCenterRule.java @@ -0,0 +1,89 @@ +package com.tmobile.cloud.awsrules.securitycenter; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; + +import com.amazonaws.util.StringUtils; +import com.tmobile.cloud.awsrules.utils.PacmanUtils; +import com.tmobile.cloud.constants.PacmanRuleConstants; +import com.tmobile.pacman.commons.PacmanSdkConstants; +import com.tmobile.pacman.commons.exception.InvalidInputException; +import com.tmobile.pacman.commons.exception.RuleExecutionFailedExeption; +import com.tmobile.pacman.commons.rule.Annotation; +import com.tmobile.pacman.commons.rule.BaseRule; +import com.tmobile.pacman.commons.rule.PacmanRule; +import com.tmobile.pacman.commons.rule.RuleResult; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@PacmanRule(key = "check-for-azure-security-rule", desc = "checks virtualmachines for network access control", severity = PacmanSdkConstants.SEV_HIGH, category = "networking") +public class AzureSecurityCenterRule extends BaseRule { + + private static final Logger logger = LoggerFactory.getLogger(AzureSecurityCenterRule.class); + + @Override + public RuleResult execute(Map ruleParam, Map resourceAttributes) { + + logger.debug("========AzureSecurityCenterRule started========="); + String entityId = ruleParam.get(PacmanSdkConstants.RESOURCE_ID); + String severity = ruleParam.get(PacmanRuleConstants.SEVERITY); + String category = ruleParam.get(PacmanRuleConstants.CATEGORY); + String targetType = ruleParam.get(PacmanRuleConstants.TARGET_TYPE); // need to specify the index type + String policyName = ruleParam.get("policyName").replaceAll("@", " "); + + MDC.put("executionId", ruleParam.get("executionId")); // this is the logback Mapped Diagnostic Contex + MDC.put("ruleId", ruleParam.get(PacmanSdkConstants.RULE_ID)); // this is the logback Mapped Diagnostic Contex + + if (!PacmanUtils.doesAllHaveValue(severity, category, targetType)) { + logger.info(PacmanRuleConstants.MISSING_CONFIGURATION); + throw new InvalidInputException(PacmanRuleConstants.MISSING_CONFIGURATION); + } + + String esUrl = null; + String url = CommonUtils.getEnvVariableValue(PacmanSdkConstants.ES_URI_ENV_VAR_NAME); + if (!StringUtils.isNullOrEmpty(url)) { + esUrl = url + "/azure_securitycenter/securitycenter/_search"; + } + + if (entityId != null && !entityId.isEmpty()) { + Map securityCenterData = new HashMap<>(); + try { + Map mustFilter = new HashMap<>(); + mustFilter.put(PacmanUtils.convertAttributetoKeyword(PacmanRuleConstants.POLICYNAME), policyName); + mustFilter.put(PacmanUtils.convertAttributetoKeyword(PacmanRuleConstants.AZURERESOURCEID), entityId.toLowerCase()); + mustFilter.put(PacmanRuleConstants.LATEST, true); + securityCenterData = PacmanUtils.checkResourceIdBypolicyName(esUrl, mustFilter); + } catch (Exception e) { + logger.error("unable to determine", e); + throw new RuleExecutionFailedExeption("unable to determine" + e); + } + if (!securityCenterData.isEmpty()) { + List> issueList = new ArrayList<>(); + LinkedHashMap issue = new LinkedHashMap<>(); + Annotation annotation = null; + annotation = Annotation.buildAnnotation(ruleParam, Annotation.Type.ISSUE); + annotation.put(PacmanSdkConstants.DESCRIPTION, policyName); + annotation.put(PacmanRuleConstants.SEVERITY, severity); + annotation.put(PacmanRuleConstants.CATEGORY, category); + issue.put(PacmanRuleConstants.VIOLATION_REASON, policyName+" Found!"); + issueList.add(issue); + annotation.put(PacmanRuleConstants.ISSUE_DETAILS, issueList.toString()); + logger.debug("========AzureSecurityCenterRule ended with annotation {} : =========", annotation); + return new RuleResult(PacmanSdkConstants.STATUS_FAILURE, PacmanRuleConstants.FAILURE_MESSAGE, annotation); + } + } + logger.debug("========AzureSecurityCenterRule Completed=========="); + return new RuleResult(PacmanSdkConstants.STATUS_SUCCESS, PacmanRuleConstants.SUCCESS_MESSAGE); + } + @Override + public String getHelpText() { + return "This rule checks the security center rules"; + } + +} diff --git a/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/awsrules/utils/PacmanUtils.java b/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/awsrules/utils/PacmanUtils.java index 247043160..73bf004b2 100644 --- a/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/awsrules/utils/PacmanUtils.java +++ b/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/awsrules/utils/PacmanUtils.java @@ -2908,5 +2908,53 @@ public static boolean checkInstanceIdForPortRuleInES(String instanceId, String e } return false; } + + /** + * Check Azure Security center rules. + * + * @param esUrl + * the es url + * @param mustfilter + * the must filter map + * @throws Exception + * the exception + */ + public static Map checkResourceIdBypolicyName(String esUrl, Map mustFilter) + throws Exception { + JsonParser jsonParser = new JsonParser(); + Map mustNotFilter = new HashMap<>(); + HashMultimap shouldFilter = HashMultimap.create(); + Map mustTermsFilter = new HashMap<>(); + Map secMap = new HashMap<>(); + + JsonObject resultJson = RulesElasticSearchRepositoryUtil.getQueryDetailsFromES(esUrl, mustFilter, mustNotFilter, + shouldFilter, null, 0, mustTermsFilter, null, null); + if (resultJson != null && resultJson.has(PacmanRuleConstants.HITS)) { + String hitsJsonString = resultJson.get(PacmanRuleConstants.HITS).toString(); + JsonObject hitsJson = (JsonObject) jsonParser.parse(hitsJsonString); + JsonArray jsonArray = hitsJson.getAsJsonObject().get(PacmanRuleConstants.HITS).getAsJsonArray(); + if (jsonArray.size() > 0) { + for (int i = 0; i < jsonArray.size(); i++) { + JsonObject firstObject = (JsonObject) jsonArray.get(i); + JsonObject sourceJson = (JsonObject) firstObject.get(PacmanRuleConstants.SOURCE); + if (null != sourceJson) { + JsonObject recomendationJson = (JsonObject) sourceJson.get(PacmanRuleConstants.RECOMMENDATION); + if ((null != recomendationJson.get(PacmanRuleConstants.RESOURCEID)) + && (!recomendationJson.get(PacmanRuleConstants.RESOURCEID).isJsonNull())) { + secMap.put(PacmanRuleConstants.RESOURCEID, + recomendationJson.get(PacmanRuleConstants.RESOURCEID).getAsString()); + if (null != recomendationJson.get(PacmanRuleConstants.DETAILS)) { + JsonObject detailJson = (JsonObject) sourceJson.get(PacmanRuleConstants.RECOMMENDATION); + secMap.put(PacmanRuleConstants.DETAILS, detailJson.get(PacmanRuleConstants.DETAILS)); + } + } + + } + + } + } + } + return secMap; + } } diff --git a/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/constants/PacmanRuleConstants.java b/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/constants/PacmanRuleConstants.java index 61af8b2ea..68a02ce34 100644 --- a/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/constants/PacmanRuleConstants.java +++ b/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/constants/PacmanRuleConstants.java @@ -355,4 +355,8 @@ private PacmanRuleConstants() { public static final String STATUS_EXEMPTED = "exempted"; public static final String ES_RESOURCE_WITH_VULN_INFO_SEVERITY_URL = "esResourceWithVulnInfoForSeverityUrl"; public static final int FIRST_DISCOVERED_DATE_FORMAT_LENGTH = 10; + public static final String POLICYNAME= "recommendation.policyName"; + public static final String AZURERESOURCEID= "recommendation._resourceIdLower"; + public static final String RECOMMENDATION = "recommendation"; + public static final String DETAILS = "details"; } From 14cb9dc34f382f819d5640be26d65201e6cd366f Mon Sep 17 00:00:00 2001 From: Kanchana Date: Mon, 4 Nov 2019 20:50:03 +0530 Subject: [PATCH 036/107] azure tagging rules & security center rules --- .../files/rule_engine_cloudwatch_rules.json | 462 ++++++++++++++++++ installer/resources/pacbot_app/files/DB.sql | 31 ++ 2 files changed, 493 insertions(+) diff --git a/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json b/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json index 2366ad25f..5b0ae9c89 100644 --- a/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json +++ b/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json @@ -2528,6 +2528,468 @@ "modifiedDate": "2019-10-25", "severity": "high", "category": "tagging" + }, + { + "ruleId": "PacMan_TaggingRule_version-1_BlobcontainerTaggingRule_blobcontainer", + "ruleUUID": "azure_blobcontainer_should_be_tagged_with_mandatory_tags", + "policyId": "PacMan_TaggingRule_version-1", + "ruleName": "BlobcontainerserverTaggingRule", + "targetType": "blobcontainer", + "assetGroup": "azure", + "alexaKeyword": "BlobcontainerserverTaggingRule", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-missing-mandatory-tags\",\"key\":\"ruleKey\"},{\"encrypt\":false,\"value\":\",\",\"key\":\"splitterChar\"},{\"encrypt\":false,\"value\":\"Application,Environment,Stack,Role\",\"key\":\"mandatoryTags\"},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"tagging\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_TaggingRule_version-1_BlobcontainerTaggingRule_blobcontainer\",\"autofix\":false,\"alexaKeyword\":\"BlobcontainerTaggingRule\",\"ruleRestUrl\":\"\",\"targetType\":\"blobcontainer\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_TaggingRule_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_blobcontainer_should_be_tagged_with_mandatory_tags\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_blobcontainer_should_be_tagged_with_mandatory_tags", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Blobcontainer should be tagged with mandatory tags", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "tagging" + }, + { + "ruleId": "PacMan_TaggingRule_version-1_SecuritycenterTaggingRule_securitycenter", + "ruleUUID": "azure_securitycenter_should_be_tagged_with_mandatory_tags", + "policyId": "PacMan_TaggingRule_version-1", + "ruleName": "SecuritycenterTaggingRule", + "targetType": "securitycenter", + "assetGroup": "azure", + "alexaKeyword": "SecuritycenterTaggingRule", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-missing-mandatory-tags\",\"key\":\"ruleKey\"},{\"encrypt\":false,\"value\":\",\",\"key\":\"splitterChar\"},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"Application,Environment,Stack,Role\",\"key\":\"mandatoryTags\"},{\"encrypt\":false,\"value\":\"tagging\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_TaggingRule_version-1_SecuritycenterTaggingRule_securitycenter\",\"autofix\":false,\"assetGroup\":\"azure\",\"alexaKeyword\":\"SecuritycenterTaggingRule\",\"ruleRestUrl\":\"\",\"targetType\":\"securitycenter\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_TaggingRule_version-1\",\"ruleUUID\":\"azure_securitycenter_should_be_tagged_with_mandatory_tags\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_securitycenter_should_be_tagged_with_mandatory_tags", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Securitycenter should be tagged with mandatory tags", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "tagging" + }, + { + "ruleId": "PacMan_TaggingRule_version-1_ResourcegroupTaggingRule_resourcegroup", + "ruleUUID": "azure_resourcegroup_should_be_tagged_with_mandatory_tags", + "policyId": "PacMan_TaggingRule_version-1", + "ruleName": "ResourcegroupTaggingRule", + "targetType": "resourcegroup", + "assetGroup": "azure", + "alexaKeyword": "ResourcegroupTaggingRule", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-missing-mandatory-tags\",\"key\":\"ruleKey\"},{\"encrypt\":false,\"value\":\",\",\"key\":\"splitterChar\"},{\"encrypt\":false,\"value\":\"Application,Environment,Stack,Role\",\"key\":\"mandatoryTags\"},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"tagging\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_TaggingRule_version-1_ResourcegroupTaggingRule_resourcegroup\",\"autofix\":false,\"alexaKeyword\":\"ResourcegroupTaggingRule\",\"ruleRestUrl\":\"\",\"targetType\":\"resourcegroup\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_TaggingRule_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_resourcegroup_should_be_tagged_with_mandatory_tags\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_resourcegroup_should_be_tagged_with_mandatory_tags", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Resourcegroup should be tagged with mandatory tags", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "tagging" + }, + { + "ruleId": "PacMan_TaggingRule_version-1_DiskTaggingRule_disk", + "ruleUUID": "azure_disk_should_be_tagged_with_mandatory_tags", + "policyId": "PacMan_TaggingRule_version-1", + "ruleName": "DiskTaggingRule", + "targetType": "disk", + "assetGroup": "azure", + "alexaKeyword": "DiskTaggingRule", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-missing-mandatory-tags\",\"key\":\"ruleKey\"},{\"encrypt\":false,\"value\":\",\",\"key\":\"splitterChar\"},{\"encrypt\":false,\"value\":\"Application,Environment,Stack,Role\",\"key\":\"mandatoryTags\"},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"tagging\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_TaggingRule_version-1_DiskTaggingRule_disk\",\"autofix\":false,\"alexaKeyword\":\"DiskTaggingRule\",\"ruleRestUrl\":\"\",\"targetType\":\"disk\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_TaggingRule_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_disk_should_be_tagged_with_mandatory_tags\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_disk_should_be_tagged_with_mandatory_tags", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Disk should be tagged with mandatory tags", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "tagging" + }, + { + "ruleId": "PacMan_TaggingRule_version-1_StorageaccountTaggingRule_storageaccount", + "ruleUUID": "azure_storageaccount_should_be_tagged_with_mandatory_tags", + "policyId": "PacMan_TaggingRule_version-1", + "ruleName": "StorageaccountTaggingRule", + "targetType": "storageaccount", + "assetGroup": "azure", + "alexaKeyword": "StorageaccountTaggingRule", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-missing-mandatory-tags\",\"key\":\"ruleKey\"},{\"encrypt\":false,\"value\":\",\",\"key\":\"splitterChar\"},{\"encrypt\":false,\"value\":\"Application,Environment,Stack,Role\",\"key\":\"mandatoryTags\"},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"tagging\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_TaggingRule_version-1_StorageaccountTaggingRule_storageaccount\",\"autofix\":false,\"alexaKeyword\":\"StorageaccountTaggingRule\",\"ruleRestUrl\":\"\",\"targetType\":\"storageaccount\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_TaggingRule_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_storageaccount_should_be_tagged_with_mandatory_tags\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_storageaccount_should_be_tagged_with_mandatory_tags", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Storageaccount should be tagged with mandatory tags", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "tagging" + }, + { + "ruleId": "PacMan_TaggingRule_version-1_NetworkinterfaceTaggingRule_networkinterface", + "ruleUUID": "azure_networkinterface_should_be_tagged_with_mandatory_tags", + "policyId": "PacMan_TaggingRule_version-1", + "ruleName": "NetworkinterfaceTaggingRule", + "targetType": "networkinterface", + "assetGroup": "azure", + "alexaKeyword": "NetworkinterfaceTaggingRule", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-missing-mandatory-tags\",\"key\":\"ruleKey\"},{\"encrypt\":false,\"value\":\",\",\"key\":\"splitterChar\"},{\"encrypt\":false,\"value\":\"Application,Environment,Stack,Role\",\"key\":\"mandatoryTags\"},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"tagging\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_TaggingRule_version-1_NetworkinterfaceTaggingRule_networkinterface\",\"autofix\":false,\"alexaKeyword\":\"NetworkinterfaceTaggingRule\",\"ruleRestUrl\":\"\",\"targetType\":\"networkinterface\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_TaggingRule_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_networkinterface_should_be_tagged_with_mandatory_tags\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_networkinterface_should_be_tagged_with_mandatory_tags", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Networkinterface should be tagged with mandatory tags", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "tagging" + }, + { + "ruleId": "PacMan_TaggingRule_version-1_NsgTaggingRule_nsg", + "ruleUUID": "azure_nsg_should_be_tagged_with_mandatory_tags", + "policyId": "PacMan_TaggingRule_version-1", + "ruleName": "NsgTaggingRule", + "targetType": "nsg", + "assetGroup": "azure", + "alexaKeyword": "NsgTaggingRule", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-missing-mandatory-tags\",\"key\":\"ruleKey\"},{\"encrypt\":false,\"value\":\",\",\"key\":\"splitterChar\"},{\"encrypt\":false,\"value\":\"Application,Environment,Stack,Role\",\"key\":\"mandatoryTags\"},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"tagging\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"pacbotdev@T-Mobile.com\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_TaggingRule_version-1_NsgTaggingRule_nsg\",\"autofix\":false,\"alexaKeyword\":\"NsgTaggingRule\",\"ruleRestUrl\":\"\",\"targetType\":\"nsg\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_TaggingRule_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_nsg_should_be_tagged_with_mandatory_tags\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_nsg_should_be_tagged_with_mandatory_tags", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Nsg should be tagged with mandatory tags", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "tagging" + }, + { + "ruleId": "PacMan_TaggingRule_version-1_VnetTaggingRule_vnet", + "ruleUUID": "azure_vnet_should_be_tagged_with_mandatory_tags", + "policyId": "PacMan_TaggingRule_version-1", + "ruleName": "VnetTaggingRule", + "targetType": "vnet", + "assetGroup": "azure", + "alexaKeyword": "VnetTaggingRule", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-missing-mandatory-tags\",\"key\":\"ruleKey\"},{\"encrypt\":false,\"value\":\",\",\"key\":\"splitterChar\"},{\"encrypt\":false,\"value\":\"Application,Environment,Stack,Role\",\"key\":\"mandatoryTags\"},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"tagging\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_TaggingRule_version-1_VnetTaggingRule_vnet\",\"autofix\":false,\"alexaKeyword\":\"VnetTaggingRule\",\"ruleRestUrl\":\"\",\"targetType\":\"vnet\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_TaggingRule_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_vnet_should_be_tagged_with_mandatory_tags\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_vnet_should_be_tagged_with_mandatory_tags", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Vnet should be tagged with mandatory tags", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "tagging" + }, + { + "ruleId": "PacMan_TaggingRule_version-1_DatabricksTaggingRule_databricks", + "ruleUUID": "azure_databricks_should_be_tagged_with_mandatory_tags", + "policyId": "PacMan_TaggingRule_version-1", + "ruleName": "DatabricksTaggingRule", + "targetType": "databricks", + "assetGroup": "azure", + "alexaKeyword": "DatabricksTaggingRule", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-missing-mandatory-tags\",\"key\":\"ruleKey\"},{\"encrypt\":false,\"value\":\",\",\"key\":\"splitterChar\"},{\"encrypt\":false,\"value\":\"Application,Environment,Stack,Role\",\"key\":\"mandatoryTags\"},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"tagging\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"pacbotdev@T-Mobile.com\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_TaggingRule_version-1_DatabricksTaggingRule_databricks\",\"autofix\":false,\"alexaKeyword\":\"DatabricksTaggingRule\",\"ruleRestUrl\":\"\",\"targetType\":\"databricks\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_TaggingRule_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_databricks_should_be_tagged_with_mandatory_tags\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_databricks_should_be_tagged_with_mandatory_tags", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Databricks should be tagged with mandatory tags", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "tagging" + }, + { + "ruleId": "PacMan_TaggingRule_version-1_LoadbalancerTaggingRule_loadbalancer", + "ruleUUID": "azure_loadbalancer_should_be_tagged_with_mandatory_tags", + "policyId": "PacMan_TaggingRule_version-1", + "ruleName": "LoadbalancerTaggingRule", + "targetType": "loadbalancer", + "assetGroup": "azure", + "alexaKeyword": "LoadbalancerTaggingRule", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-missing-mandatory-tags\",\"encrypt\":false},{\"key\":\"splitterChar\",\"value\":\",\",\"encrypt\":false},{\"key\":\"mandatoryTags\",\"value\":\"Application,Environment,Stack,Role\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"tagging\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_TaggingRule_version-1_LoadbalancerTaggingRule_loadbalancer\",\"autofix\":false,\"alexaKeyword\":\"LoadbalancerTaggingRule\",\"ruleRestUrl\":\"\",\"targetType\":\"loadbalancer\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_TaggingRule_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_loadbalancer_should_be_tagged_with_mandatory_tags\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_loadbalancer_should_be_tagged_with_mandatory_tags", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Loadbalancer should be tagged with mandatory tags", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "tagging" + }, + { + "ruleId": "PacMan_TaggingRule_version-1_MysqlserverTaggingRule_mysqlserver", + "ruleUUID": "azure_mysqlserver_should_be_tagged_with_mandatory_tags", + "policyId": "PacMan_TaggingRule_version-1", + "ruleName": "MysqlserverTaggingRule", + "targetType": "mysqlserver", + "assetGroup": "azure", + "alexaKeyword": "MysqlserverTaggingRule", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-missing-mandatory-tags\",\"key\":\"ruleKey\"},{\"encrypt\":false,\"value\":\",\",\"key\":\"splitterChar\"},{\"encrypt\":false,\"value\":\"Application,Environment,Stack,Role\",\"key\":\"mandatoryTags\"},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"tagging\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_TaggingRule_version-1_MysqlserverTaggingRule_mysqlserver\",\"autofix\":false,\"alexaKeyword\":\"MysqlserverTaggingRule\",\"ruleRestUrl\":\"\",\"targetType\":\"mysqlserver\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_TaggingRule_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_mysqlserver_should_be_tagged_with_mandatory_tags\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_mysqlserver_should_be_tagged_with_mandatory_tags", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Mysqlserver should be tagged with mandatory tags", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "tagging" + }, + { + "ruleId": "PacMan_Azure_Close_management_ports_version-1_SecurityCenter_virtualmachine", + "ruleUUID": "azure_virtualmachine_security_center", + "policyId": "PacMan_Azure_Close_management_ports_version-1", + "ruleName": "Close_management_ports", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "Close_management_ports", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-azure-security-rule\",\"key\":\"ruleKey\"},{\"key\":\"policyName\",\"value\":\"Management@ports@should@be@closed@on@your@virtual@machines\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_Close_management_ports_version-1_SecurityCenter_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"close management port\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_Close_management_ports_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_virtualmachine_security_center\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_virtualmachine_security_center", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Close management ports on your Virtual Machines", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Azure_Close_management_ports_version-1_SecurityCenter_virtualmachine", + "ruleUUID": "azure_virtualmachine_security_center", + "policyId": "PacMan_Azure_Close_management_ports_version-1", + "ruleName": "Close_management_ports", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "Close_management_ports", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-azure-security-rule\",\"key\":\"ruleKey\"},{\"key\":\"policyName\",\"value\":\"Management@ports@should@be@closed@on@your@virtual@machines\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_Close_management_ports_version-1_SecurityCenter_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"close management port\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_Close_management_ports_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_virtualmachine_security_center\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_virtualmachine_security_center", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Close management ports on your Virtual Machines", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Azure_Enable_Adaptive_Application_controls_version-1_SecurityCenter_virtualmachine", + "ruleUUID": "azure_virtualmachine_enable_adaptive_application", + "policyId": "PacMan_Azure_Enable_Adaptive_Application_controls_version-1", + "ruleName": "Enable_Adaptive_Application_controls", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "Enable_Adaptive_Application_controls", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-azure-security-rule\",\"key\":\"ruleKey\"},{\"key\":\"policyName\",\"value\":\"Adaptive@Application@Controls@should@be@enabled@on@virtual@machines\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_Enable_Adaptive_Application_controls_version-1_SecurityCenter_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"enable adaptive controls\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_Enable_Adaptive_Application_controls_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"PacMan_Azure_Enable_Adaptive_Application_controls_version-1\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_virtualmachine_enable_adaptive_application", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Enable Adaptive Application Controls", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Azure_Enable_Network_Security_version-1_SecurityCenter_virtualmachine", + "ruleUUID": "azure_virtualmachine_enable_network_security", + "policyId": "PacMan_Azure_Enable_Network_Security_version-1", + "ruleName": "Enable_Network_Security", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "Enable_Network_Security", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-azure-security-rule\",\"key\":\"ruleKey\"},{\"key\":\"policyName\",\"value\":\"Access@should@be@restricted@for@permissive@Network@Security@Groups@with@Internet-facing@VMs\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_Enable_Network_Security_version-1_SecurityCenter_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"enable network security\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_Enable_Network_Security_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_virtualmachine_enable_network_security\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_virtualmachine_enable_network_security", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Enable Network Security Groups on virtual machines", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Azure_Install_a_vulnerability_version-1_SecurityCenter_virtualmachine", + "ruleUUID": "azure_virtualmachine_install_a_vulnerability", + "policyId": "PacMan_Azure_Install_a_vulnerability_version-1", + "ruleName": "Install_a_vulnerability", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "Install_a_vulnerability", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-azure-security-rule\",\"key\":\"ruleKey\"},{\"key\":\"policyName\",\"value\":\"Vulnerability@assessment@solution@should@be@installed@on@your@virtual@machines\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_Install_a_vulnerability_version-1_SecurityCenter_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"install vulnerability scan tool\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_Install_a_vulnerability_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_virtualmachine_install_a_vulnerability\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_virtualmachine_install_a_vulnerability", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Install a vulnerability assessment solution on your virtual machines", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Azure_Install_monitoring_agent_version-1_SecurityCenter_virtualmachine", + "ruleUUID": "azure_virtualmachine_install_monitoring_agent", + "policyId": "PacMan_Azure_Install_monitoring_agent_version-1", + "ruleName": "Install_Monitoring_Agent", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "Install_Monitoring_Agent", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-azure-security-rule\",\"key\":\"ruleKey\"},{\"key\":\"policyName\",\"value\":\"Monitoring@agent@should@be@installed@on@your@machines\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"governance\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_Install_monitoring_agent_version-1_SecurityCenter_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"install monitoring agent\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_Install_monitoring_agent_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_virtualmachine_install_monitoring_agent\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_virtualmachine_install_monitoring_agent", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Install monitoring agent on your machines", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Azure_Just_in_time_network_access_version-1_SecurityCenter_virtualmachine", + "ruleUUID": "azure_virtualmachine_just_in_time_network_access", + "policyId": "PacMan_Azure_Just_in_time_network_access_version-1", + "ruleName": "Just_in_time_network_access", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "Just_in_time_network_access", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-azure-security-rule\",\"key\":\"ruleKey\"},{\"key\":\"policyName\",\"value\":\"Just-In-Time@network@access@control@should@be@applied@on@virtual@machines\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_Just_in_time_network_access_version-1_SecurityCenter_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"enable just in time access\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_Just_in_time_network_access_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_virtualmachine_just_in_time_network_access\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_virtualmachine_just_in_time_network_access", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Apply a Just-In-Time network access control", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Azure_Remediate_vulnerabilities_version-1_SecurityCenter_virtualmachine", + "ruleUUID": "azure_virtualmachine_remediate_vulnerabilities", + "policyId": "PacMan_Azure_Remediate_vulnerabilities_version-1", + "ruleName": "Remediate_vulnerabilities", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "Remediate_vulnerabilities", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-azure-security-rule\",\"key\":\"ruleKey\"},{\"key\":\"policyName\",\"value\":\"Vulnerabilities@should@be@remediated@by@a@Vulnerability@Assessment@solution\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_Remediate_vulnerabilities_version-1_SecurityCenter_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"remediate vulnerabilities\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_Remediate_vulnerabilities_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_virtualmachine_remediate_vulnerabilities\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_virtualmachine_remediate_vulnerabilities", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Remediate vulnerabilities - by a Vulnerability Assessment solution", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Azure_Resolve_monitoring_agent_version-1_SecurityCenter_virtualmachine", + "ruleUUID": "azure_virtualmachine_resolve_monitoring_agent", + "policyId": "PacMan_Azure_Resolve_monitoring_agent_version-1", + "ruleName": "resolve_monitoring_agent", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "resolve_monitoring_agent", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-azure-security-rule\",\"key\":\"ruleKey\"},{\"key\":\"policyName\",\"value\":\"Monitoring@agent@health@issues@should@be@resolved@on@your@machines\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_Resolve_monitoring_agent_version-1_SecurityCenter_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"resolve monitoring agent findings\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_Resolve_monitoring_agent_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_virtualmachine_resolve_monitoring_agent\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_virtualmachine_resolve_monitoring_agent", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Resolve monitoring agent health issues on your machines", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Azure_harden-NSGs_internet_version-1_SecurityCenter_virtualmachine", + "ruleUUID": "azure_virtualmachine_resolve_monitoring_agent", + "policyId": "PacMan_Azure_harden-NSGs_internet_version-1", + "ruleName": "resolve_monitoring_agent", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "resolve_monitoring_agent", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-azure-security-rule\",\"key\":\"ruleKey\"},{\"key\":\"policyName\",\"value\":\"Monitoring@agent@health@issues@should@be@resolved@on@your@machines\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_Resolve_monitoring_agent_version-1_SecurityCenter_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"resolve monitoring agent findings\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_Resolve_monitoring_agent_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_virtualmachine_resolve_monitoring_agent\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 * * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_virtualmachine_resolve_monitoring_agent", + "status": "ENABLED", + "userId": "ASGC", + "displayName": "Resolve monitoring agent health issues on your machines", + "createdDate": "2019-10-25", + "modifiedDate": "2019-10-25", + "severity": "high", + "category": "security" } ] diff --git a/installer/resources/pacbot_app/files/DB.sql b/installer/resources/pacbot_app/files/DB.sql index 4da0ebc0d..5e03682b1 100644 --- a/installer/resources/pacbot_app/files/DB.sql +++ b/installer/resources/pacbot_app/files/DB.sql @@ -1227,6 +1227,15 @@ INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_CloudWatchEventsForAllAccounts_version-1','CloudWatchEventsForAllAccounts','Events from all AWS account should be routed to a central event bus so that the events and be processed and analyzed centrally.',"Events from all AWS account should be routed to a central event.",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_LowUtilizationAmazonEC2InstancesRule_version-1','LowUtilizationAmazonEC2InstancesRule','Checks the Amazon Elastic Compute Cloud (Amazon EC2) instances that were running at any time during the last 14 days and alerts you if the daily CPU utilization was 10% or less and network I/O was 5 MB or less on 4 or more days. Running instances generate hourly usage charges. Although some scenarios can result in low utilization by design, you can often lower your costs by managing the number and size of your instances. n instance had 10% or less daily average CPU utilization and 5 MB or less network I/O on at least 4 of the previous 14 days',"Consider stopping or terminating instances that have low utilization, or scale the number of instances by using Auto Scaling.",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_ElasticSearchPublicAccess_version-1','ElasticsearchPublicAccessRule','Make necessary changes to the access control policy and security groups to make the ES endpoint private, Allow only a specific list of IP addresses, Once the Elastic Search endpoint is not publicly accessible PacBot will auotmatically close the issue, In case you want this to be public then send a request for exeception to cloudsecops@t-mobile.com, You can also request exception from the policy violation details page, Secops will review and involve DSO if required and grant exception and PacBot will automatically ignore this resource till the expiry of exception.',"Make necessary changes to the access control policy and security groups to make the ES endpoint private, Allow only a specific list of IP addresses, Once the Elastic Search endpoint is not publicly accessible PacBot will auotmatically close the issue, In case you want this to be public then send a request for exeception to cloudsecops@t-mobile.com, You can also request exception from the policy violation details page, Secops will review and involve DSO if required and grant exception and PacBot will automatically ignore this resource till the expiry of exception.",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); +INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_Azure_Close_management_ports_version-1','PacMan_Azure_Close_management_ports_version-1','This is Azure Security Rule',"",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); +INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_Azure_Enable_Adaptive_Application_controls_version-1','Enable Adaptive Application Controls','Application control helps you deal with malicious and/or unauthorized software, by allowing only specific applications to run on your VMs and Computers',"Open the Security Center dashboard.,In the left pane select Adaptive application controls located under Advanced cloud defense and Follow the guidelines.",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); +INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_Azure_Enable_Network_Security_version-1','Enable Network Security','Azure Network Security',"",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); +INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_Azure_Install_a_vulnerability_version-1','Install a vulnerability assessment solution on your virtual machines','The vulnerability assessment in Azure Security Center is part of the Security Center virtual machine (VM) recommendations. If Security Center doesnt find a vulnerability assessment solution installed on your VM, it recommends that you install one. A partner agent, after being deployed, starts reporting vulnerability data to the partner’s management platform. In turn, the partners management platform provides vulnerability and health monitoring data back to Security Center. You can quickly identify vulnerable VMs on the Security Center dashboard. Switch to the partner management console directly from Security Center for additional reports and information.',"",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); +INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_Azure_Install_monitoring_agent_version-1','Install monitoring agent on your machines','Security Center uses the Microsoft Monitoring Agent (MMA) to collect security events from your Azure virtual machines. To make sure your virtual machines are successfully monitored, you need to enable data collection in Security Center and make sure the MMA agent is both installed on the virtual machines and properly collects security events to the configured workspace. Enabling data collection in Security Center enables you to benefit from multiple agent-based features, including OS baselines rules assessments, monitoring for missing system updates, endpoint protection issues and advanced threat detection capabilities.',"Installation of the monitoring agent and enabling data collection in Security Center can be done in several ways: Using Security Center’s automatic provisioning on your subscription(s). This will automatically provision the monitoring agent on current and future-created virtual machines on your subscription(s). You can enable automatic provisioning on multiple subscriptions by clicking on the Getting started menu item, and select 'Install agents'. You can also enable it for specific subscriptions and customize additional settings by clicking on the 'Security policy' menu item, select 'Edit settings' on a subscription and enable auto provisioning in the 'data collection' menu item. Install the Microsoft Monitoring agent on your Virtual machines as a VM extension or directly, by following these instructions. Provision the Microsoft Monitoring agent with Azure Policies.",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); +INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_Azure_Just_in_time_network_access_version-1','Enable Just in time network access on your virutal machines','Just-in-time (JIT) virtual machine (VM) access can be used to lock down inbound traffic to your Azure VMs, reducing exposure to attacks while providing easy access to connect to VMs when needed.',"Open the Security Center dashboard.,In the left pane, select Just-in-time VM access.,The Just-in-time VM access window opens.,Select the Recommended tab.,Under VIRTUAL MACHINE, click the VMs that you want to enable. This puts a checkmark next to a VM.,",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); +INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_Azure_Remediate_vulnerabilities_version-1','Remediate vulnerabilities - by a Vulnerability Assessment solution','This is Azure security rule',"",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); +INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_Azure_Resolve_monitoring_agent_version-1','Resolve monitoring agent health issues on your machines','This is Azure Secuirty Rule',"",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); +INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_Azure_harden-NSGs_internet_version-1','Harden Network Security Group rules of internet facing Virtual Machines','This is Azure Secuirty Rule',"",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); /* Rule Initialisation */ @@ -1348,6 +1357,28 @@ INSERT IGNORE INTO cf_RuleInstance (`ruleId`,`ruleUUID`,`policyId`,`ruleName`,`t INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_VirtualmachineTaggingRule_virtualmachine','azure_virtualmachine_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','VirtualmachineTaggingRule','virtualmachine','azure','VirtualmachineTaggingRule','{"params":[{"encrypt":false,"value":"check-for-missing-mandatory-tags","key":"ruleKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_VirtualmachineTaggingRule_virtualmachine","autofix":false,"alexaKeyword":"VirtualmachineTaggingRule","ruleRestUrl":"","targetType":"virtualmachine","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","assetGroup":"azure","ruleUUID":"azure_virtualmachine_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_virtualmachine_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Virtualmachine should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_SqlserverTaggingRule_sqlserver','azure_sqlserver_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','SqlserverTaggingRule','sqlserver','azure','SqlserverTaggingRule','{"params":[{"encrypt":false,"value":"check-for-missing-mandatory-tags","key":"ruleKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_SqlserverTaggingRule_sqlserver","autofix":false,"alexaKeyword":"SqlserverTaggingRule","ruleRestUrl":"","targetType":"sqlserver","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","assetGroup":"azure","ruleUUID":"azure_sqlserver_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_sqlserver_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Sqlserver should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_SqldatabaseTaggingRule_sqldatabase','azure_sqldatabase_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','SqldatabaseTaggingRule','sqldatabase','azure','SqldatabaseTaggingRule','{"params":[{"encrypt":false,"value":"check-for-missing-mandatory-tags","key":"ruleKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_SqldatabaseTaggingRule_sqldatabase","autofix":false,"alexaKeyword":"SqldatabaseTaggingRule","ruleRestUrl":"","targetType":"sqldatabase","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","assetGroup":"azure","ruleUUID":"azure_sqldatabase_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_sqldatabase_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Sqldatabase should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_BlobcontainerTaggingRule_blobcontainer','azure_blobcontainer_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','BlobcontainerTaggingRule','blobcontainer','azure','BlobcontainerTaggingRule','{"params":[{"encrypt":false,"value":"check-for-missing-mandatory-tags","key":"ruleKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_BlobcontainerTaggingRule_blobcontainer","autofix":false,"alexaKeyword":"BlobcontainerTaggingRule","ruleRestUrl":"","targetType":"blobcontainer","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","assetGroup":"azure","ruleUUID":"azure_blobcontainer_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_blobcontainer_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Blobcontainer should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_SecuritycenterTaggingRule_securitycenter','azure_securitycenter_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','SecuritycenterTaggingRule','securitycenter','azure','SecuritycenterTaggingRule','{"params":[{"encrypt":false,"value":"check-for-missing-mandatory-tags","key":"ruleKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_SecuritycenterTaggingRule_securitycenter","autofix":false,"assetGroup":"azure","alexaKeyword":"SecuritycenterTaggingRule","ruleRestUrl":"","targetType":"securitycenter","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","ruleUUID":"azure_securitycenter_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_securitycenter_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Securitycenter should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_ResourcegroupTaggingRule_resourcegroup','azure_resourcegroup_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','ResourcegroupTaggingRule','resourcegroup','azure','ResourcegroupTaggingRule','{"params":[{"encrypt":false,"value":"check-for-missing-mandatory-tags","key":"ruleKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_ResourcegroupTaggingRule_resourcegroup","autofix":false,"alexaKeyword":"ResourcegroupTaggingRule","ruleRestUrl":"","targetType":"resourcegroup","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","assetGroup":"azure","ruleUUID":"azure_resourcegroup_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_resourcegroup_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Resourcegroup should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_DiskTaggingRule_disk','azure_disk_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','DiskTaggingRule','disk','azure','DiskTaggingRule','{"params":[{"encrypt":false,"value":"check-for-missing-mandatory-tags","key":"ruleKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_DiskTaggingRule_disk","autofix":false,"alexaKeyword":"DiskTaggingRule","ruleRestUrl":"","targetType":"disk","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","assetGroup":"azure","ruleUUID":"azure_disk_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_disk_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Disk should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_StorageaccountTaggingRule_storageaccount','azure_storageaccount_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','StorageaccountTaggingRule','storageaccount','azure','StorageaccountTaggingRule','{"params":[{"encrypt":false,"value":"check-for-missing-mandatory-tags","key":"ruleKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_StorageaccountTaggingRule_storageaccount","autofix":false,"alexaKeyword":"StorageaccountTaggingRule","ruleRestUrl":"","targetType":"storageaccount","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","assetGroup":"azure","ruleUUID":"azure_storageaccount_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_storageaccount_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Storageaccount should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_NetworkinterfaceTaggingRule_networkinterface','azure_networkinterface_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','NetworkinterfaceTaggingRule','networkinterface','azure','NetworkinterfaceTaggingRule','{"params":[{"encrypt":false,"value":"check-for-missing-mandatory-tags","key":"ruleKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_NetworkinterfaceTaggingRule_networkinterface","autofix":false,"alexaKeyword":"NetworkinterfaceTaggingRule","ruleRestUrl":"","targetType":"networkinterface","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","assetGroup":"azure","ruleUUID":"azure_networkinterface_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_networkinterface_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Networkinterface should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_NsgTaggingRule_nsg','azure_nsg_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','NsgTaggingRule','nsg','azure','NsgTaggingRule','{"params":[{"encrypt":false,"value":"check-for-missing-mandatory-tags","key":"ruleKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_NsgTaggingRule_nsg","autofix":false,"alexaKeyword":"NsgTaggingRule","ruleRestUrl":"","targetType":"nsg","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","assetGroup":"azure","ruleUUID":"azure_nsg_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_nsg_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Nsg should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_VnetTaggingRule_vnet','azure_vnet_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','VnetTaggingRule','vnet','azure','VnetTaggingRule','{"params":[{"encrypt":false,"value":"check-for-missing-mandatory-tags","key":"ruleKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_VnetTaggingRule_vnet","autofix":false,"alexaKeyword":"VnetTaggingRule","ruleRestUrl":"","targetType":"vnet","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","assetGroup":"azure","ruleUUID":"azure_vnet_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_vnet_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Vnet should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_DatabricksTaggingRule_databricks','azure_databricks_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','DatabricksTaggingRule','databricks','azure','DatabricksTaggingRule','{"params":[{"encrypt":false,"value":"check-for-missing-mandatory-tags","key":"ruleKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_DatabricksTaggingRule_databricks","autofix":false,"alexaKeyword":"DatabricksTaggingRule","ruleRestUrl":"","targetType":"databricks","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","assetGroup":"azure","ruleUUID":"azure_databricks_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_databricks_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Databricks should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_LoadbalancerTaggingRule_loadbalancer','azure_loadbalancer_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','LoadbalancerTaggingRule','loadbalancer','azure','LoadbalancerTaggingRule','{"params":[{"key":"ruleKey","value":"check-for-missing-mandatory-tags","encrypt":false},{"key":"splitterChar","value":",","encrypt":false},{"key":"mandatoryTags","value":"Application,Environment,Stack,Role","encrypt":false},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_LoadbalancerTaggingRule_loadbalancer","autofix":false,"alexaKeyword":"LoadbalancerTaggingRule","ruleRestUrl":"","targetType":"loadbalancer","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","assetGroup":"azure","ruleUUID":"azure_loadbalancer_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_loadbalancer_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Loadbalancer should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_TaggingRule_version-1_MysqlserverTaggingRule_mysqlserver','azure_mysqlserver_should_be_tagged_with_mandatory_tags','PacMan_TaggingRule_version-1','MysqlserverTaggingRule','mysqlserver','azure','MysqlserverTaggingRule','{"params":[{"encrypt":false,"value":"check-for-missing-mandatory-tags","key":"ruleKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"tagging","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_TaggingRule_version-1_MysqlserverTaggingRule_mysqlserver","autofix":false,"alexaKeyword":"MysqlserverTaggingRule","ruleRestUrl":"","targetType":"mysqlserver","pac_ds":"azure","policyId":"PacMan_TaggingRule_version-1","assetGroup":"azure","ruleUUID":"azure_mysqlserver_should_be_tagged_with_mandatory_tags","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_mysqlserver_should_be_tagged_with_mandatory_tags'),'ENABLED','ASGC','Mysqlserver should be tagged with mandatory tags',{d '2019-10-25'},{d '2019-10-25'},null,null); + +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Azure_Close_management_ports_version-1_SecurityCenter_virtualmachine','azure_virtualmachine_security_center','PacMan_Azure_Close_management_ports_version-1','Close_management_ports','virtualmachine','azure','Close_management_ports','{"params":[{"encrypt":false,"value":"check-for-azure-security-rule","key":"ruleKey"},{"key":"policyName","value":"Management@ports@should@be@closed@on@your@virtual@machines","isValueNew":true,"encrypt":false},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"security","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_Azure_Close_management_ports_version-1_SecurityCenter_virtualmachine","autofix":false,"alexaKeyword":"close management port","ruleRestUrl":"","targetType":"virtualmachine","pac_ds":"azure","policyId":"PacMan_Azure_Close_management_ports_version-1","assetGroup":"azure","ruleUUID":"azure_virtualmachine_security_center","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_virtualmachine_security_center'),'ENABLED','ASGC','Close management ports on your Virtual Machines',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Azure_Enable_Adaptive_Application_controls_version-1_SecurityCenter_virtualmachine','azure_virtualmachine_enable_adaptive_application','PacMan_Azure_Enable_Adaptive_Application_controls_version-1','Enable_Adaptive_Application_controls','virtualmachine','azure','Enable_Adaptive_Application_controls','{"params":[{"encrypt":false,"value":"check-for-azure-security-rule","key":"ruleKey"},{"key":"policyName","value":"Adaptive@Application@Controls@should@be@enabled@on@virtual@machines","isValueNew":true,"encrypt":false},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"security","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_Azure_Enable_Adaptive_Application_controls_version-1_SecurityCenter_virtualmachine","autofix":false,"alexaKeyword":"enable adaptive controls","ruleRestUrl":"","targetType":"virtualmachine","pac_ds":"azure","policyId":"PacMan_Azure_Enable_Adaptive_Application_controls_version-1","assetGroup":"azure","ruleUUID":"PacMan_Azure_Enable_Adaptive_Application_controls_version-1","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_virtualmachine_enable_adaptive_application'),'ENABLED','ASGC','Enable Adaptive Application Controls',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Azure_Enable_Network_Security_version-1_SecurityCenter_virtualmachine','azure_virtualmachine_enable_network_security','PacMan_Azure_Enable_Network_Security_version-1','Enable_Network_Security','virtualmachine','azure','Enable_Network_Security','{"params":[{"encrypt":false,"value":"check-for-azure-security-rule","key":"ruleKey"},{"key":"policyName","value":"Access@should@be@restricted@for@permissive@Network@Security@Groups@with@Internet-facing@VMs","isValueNew":true,"encrypt":false},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"security","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_Azure_Enable_Network_Security_version-1_SecurityCenter_virtualmachine","autofix":false,"alexaKeyword":"enable network security","ruleRestUrl":"","targetType":"virtualmachine","pac_ds":"azure","policyId":"PacMan_Azure_Enable_Network_Security_version-1","assetGroup":"azure","ruleUUID":"azure_virtualmachine_enable_network_security","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_virtualmachine_enable_network_security'),'ENABLED','ASGC','Enable Network Security Groups on virtual machines',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Azure_Install_a_vulnerability_version-1_SecurityCenter_virtualmachine','azure_virtualmachine_install_a_vulnerability','PacMan_Azure_Install_a_vulnerability_version-1','Install_A_Vulnerability','virtualmachine','azure','Install_A_Vulnerability','{"params":[{"encrypt":false,"value":"check-for-azure-security-rule","key":"ruleKey"},{"key":"policyName","value":"Vulnerability@assessment@solution@should@be@installed@on@your@virtual@machines","isValueNew":true,"encrypt":false},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"security","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_Azure_Install_a_vulnerability_version-1_SecurityCenter_virtualmachine","autofix":false,"alexaKeyword":"install vulnerability scan tool","ruleRestUrl":"","targetType":"virtualmachine","pac_ds":"azure","policyId":"PacMan_Azure_Install_a_vulnerability_version-1","assetGroup":"azure","ruleUUID":"azure_virtualmachine_install_a_vulnerability","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_virtualmachine_install_a_vulnerability'),'ENABLED','ASGC','Install a vulnerability assessment solution on your virtual machines',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Azure_Install_monitoring_agent_version-1_SecurityCenter_virtualmachine','azure_virtualmachine_install_monitoring_agent','PacMan_Azure_Install_monitoring_agent_version-1','Install_Monitoring_Agent','virtualmachine','azure','Install_Monitoring_Agent','{"params":[{"encrypt":false,"value":"check-for-azure-security-rule","key":"ruleKey"},{"key":"policyName","value":"Monitoring@agent@should@be@installed@on@your@machines","isValueNew":true,"encrypt":false},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"governance","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_Azure_Install_monitoring_agent_version-1_SecurityCenter_virtualmachine","autofix":false,"alexaKeyword":"install monitoring agent","ruleRestUrl":"","targetType":"virtualmachine","pac_ds":"azure","policyId":"PacMan_Azure_Install_monitoring_agent_version-1","assetGroup":"azure","ruleUUID":"azure_virtualmachine_install_monitoring_agent","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_virtualmachine_install_monitoring_agent'),'ENABLED','ASGC','Install monitoring agent on your machines',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Azure_Just_in_time_network_access_version-1_SecurityCenter_virtualmachine','azure_virtualmachine_just_in_time_network_access','PacMan_Azure_Just_in_time_network_access_version-1','Just_in_time_network_access','virtualmachine','azure','Just_in_time_network_access','{"params":[{"encrypt":false,"value":"check-for-azure-security-rule","key":"ruleKey"},{"key":"policyName","value":"Just-In-Time@network@access@control@should@be@applied@on@virtual@machines","isValueNew":true,"encrypt":false},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"security","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_Azure_Just_in_time_network_access_version-1_SecurityCenter_virtualmachine","autofix":false,"alexaKeyword":"enable just in time access","ruleRestUrl":"","targetType":"virtualmachine","pac_ds":"azure","policyId":"PacMan_Azure_Just_in_time_network_access_version-1","assetGroup":"azure","ruleUUID":"azure_virtualmachine_just_in_time_network_access","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_virtualmachine_just_in_time_network_access'),'ENABLED','ASGC','Apply a Just-In-Time network access control',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Azure_Remediate_vulnerabilities_version-1_SecurityCenter_virtualmachine','azure_virtualmachine_remediate_vulnerabilities','PacMan_Azure_Remediate_vulnerabilities_version-1','Remediate_vulnerabilities','virtualmachine','azure','Remediate_vulnerabilities','{"params":[{"encrypt":false,"value":"check-for-azure-security-rule","key":"ruleKey"},{"key":"policyName","value":"Vulnerabilities@should@be@remediated@by@a@Vulnerability@Assessment@solution","isValueNew":true,"encrypt":false},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"security","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_Azure_Remediate_vulnerabilities_version-1_SecurityCenter_virtualmachine","autofix":false,"alexaKeyword":"remediate vulnerabilities","ruleRestUrl":"","targetType":"virtualmachine","pac_ds":"azure","policyId":"PacMan_Azure_Remediate_vulnerabilities_version-1","assetGroup":"azure","ruleUUID":"azure_virtualmachine_remediate_vulnerabilities","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_virtualmachine_remediate_vulnerabilities'),'ENABLED','ASGC','Remediate vulnerabilities - by a Vulnerability Assessment solution',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Azure_Resolve_monitoring_agent_version-1_SecurityCenter_virtualmachine','azure_virtualmachine_resolve_monitoring_agent','PacMan_Azure_Resolve_monitoring_agent_version-1','resolve_monitoring_agent','virtualmachine','azure','resolve_monitoring_agent','{"params":[{"encrypt":false,"value":"check-for-azure-security-rule","key":"ruleKey"},{"key":"policyName","value":"Monitoring@agent@health@issues@should@be@resolved@on@your@machines","isValueNew":true,"encrypt":false},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"security","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_Azure_Resolve_monitoring_agent_version-1_SecurityCenter_virtualmachine","autofix":false,"alexaKeyword":"resolve monitoring agent findings","ruleRestUrl":"","targetType":"virtualmachine","pac_ds":"azure","policyId":"PacMan_Azure_Resolve_monitoring_agent_version-1","assetGroup":"azure","ruleUUID":"azure_virtualmachine_resolve_monitoring_agent","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_virtualmachine_resolve_monitoring_agent'),'ENABLED','ASGC','Resolve monitoring agent health issues on your machines',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Azure_harden-NSGs_internet_version-1_SecurityCenter_virtualmachine','azure_virtualmachine_harden-nsgs_internet','PacMan_Azure_harden-NSGs_internet_version-1','harden-NSGs_internet','virtualmachine','azure','harden-NSGs_internet','{"params":[{"encrypt":false,"value":"check-for-azure-security-rule","key":"ruleKey"},{"key":"policyName","value":"Harden@Network@Security@Group@rules@of@internet@facing@virtual@machines","isValueNew":true,"encrypt":false},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"security","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_Azure_harden-NSGs_internet_version-1_SecurityCenter_virtualmachine","autofix":false,"alexaKeyword":"harden nsgs for internet facing vms","ruleRestUrl":"","targetType":"virtualmachine","pac_ds":"azure","policyId":"PacMan_Azure_harden-NSGs_internet_version-1","assetGroup":"azure","ruleUUID":"azure_virtualmachine_harden-nsgs_internet","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_virtualmachine_harden-nsgs_internet'),'ENABLED','ASGC','Harden Network Security Group rules of internet facing Virtual Machines',{d '2019-10-25'},{d '2019-10-25'},null,null); + /* Omni Seach Configuration */ From 619029cf4c5bbbef73827c046bf2c62a684bb8ee Mon Sep 17 00:00:00 2001 From: Kanchana Date: Mon, 4 Nov 2019 20:57:28 +0530 Subject: [PATCH 037/107] date changed for testing commit --- .../files/rule_engine_cloudwatch_rules.json | 40 +++++-------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json b/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json index 5b0ae9c89..5caed9008 100644 --- a/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json +++ b/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json @@ -2788,33 +2788,11 @@ "status": "ENABLED", "userId": "ASGC", "displayName": "Close management ports on your Virtual Machines", - "createdDate": "2019-10-25", - "modifiedDate": "2019-10-25", - "severity": "high", - "category": "security" - }, - { - "ruleId": "PacMan_Azure_Close_management_ports_version-1_SecurityCenter_virtualmachine", - "ruleUUID": "azure_virtualmachine_security_center", - "policyId": "PacMan_Azure_Close_management_ports_version-1", - "ruleName": "Close_management_ports", - "targetType": "virtualmachine", - "assetGroup": "azure", - "alexaKeyword": "Close_management_ports", - "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-azure-security-rule\",\"key\":\"ruleKey\"},{\"key\":\"policyName\",\"value\":\"Management@ports@should@be@closed@on@your@virtual@machines\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_Close_management_ports_version-1_SecurityCenter_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"close management port\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_Close_management_ports_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_virtualmachine_security_center\",\"ruleType\":\"ManageRule\"}", - "ruleFrequency": "0 * * * ? *", - "ruleExecutable": "", - "ruleRestUrl": "", - "ruleType": "ManageRule", - "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_virtualmachine_security_center", - "status": "ENABLED", - "userId": "ASGC", - "displayName": "Close management ports on your Virtual Machines", - "createdDate": "2019-10-25", - "modifiedDate": "2019-10-25", + "createdDate": "2019-11-04", + "modifiedDate": "2019-11-04", "severity": "high", "category": "security" - }, + } { "ruleId": "PacMan_Azure_Enable_Adaptive_Application_controls_version-1_SecurityCenter_virtualmachine", "ruleUUID": "azure_virtualmachine_enable_adaptive_application", @@ -2971,21 +2949,21 @@ }, { "ruleId": "PacMan_Azure_harden-NSGs_internet_version-1_SecurityCenter_virtualmachine", - "ruleUUID": "azure_virtualmachine_resolve_monitoring_agent", + "ruleUUID": "azure_virtualmachine_harden-nsgs_internet", "policyId": "PacMan_Azure_harden-NSGs_internet_version-1", - "ruleName": "resolve_monitoring_agent", + "ruleName": "Harden-NSGs_internet", "targetType": "virtualmachine", "assetGroup": "azure", - "alexaKeyword": "resolve_monitoring_agent", - "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-azure-security-rule\",\"key\":\"ruleKey\"},{\"key\":\"policyName\",\"value\":\"Monitoring@agent@health@issues@should@be@resolved@on@your@machines\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_Resolve_monitoring_agent_version-1_SecurityCenter_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"resolve monitoring agent findings\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_Resolve_monitoring_agent_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_virtualmachine_resolve_monitoring_agent\",\"ruleType\":\"ManageRule\"}", + "alexaKeyword": "Harden-NSGs_internet", + "ruleParams": "{\"params\":[{\"encrypt\":false,\"value\":\"check-for-azure-security-rule\",\"key\":\"ruleKey\"},{\"key\":\"policyName\",\"value\":\"Harden@Network@Security@Group@rules@of@internet@facing@virtual@machines\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_harden-NSGs_internet_version-1_SecurityCenter_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"harden nsgs for internet facing vms\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_harden-NSGs_internet_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_virtualmachine_harden-nsgs_internet\",\"ruleType\":\"ManageRule\"}", "ruleFrequency": "0 * * * ? *", "ruleExecutable": "", "ruleRestUrl": "", "ruleType": "ManageRule", - "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_virtualmachine_resolve_monitoring_agent", + "ruleArn": "arn:aws:events:us-east-1:***REMOVED***:rule/azure_virtualmachine_harden-nsgs_internet", "status": "ENABLED", "userId": "ASGC", - "displayName": "Resolve monitoring agent health issues on your machines", + "displayName": "Harden Network Security Group rules of internet facing Virtual Machines", "createdDate": "2019-10-25", "modifiedDate": "2019-10-25", "severity": "high", From d18efbc0132d743db7f3230a12aa7414632b0399 Mon Sep 17 00:00:00 2001 From: Kanchana Date: Mon, 4 Nov 2019 21:07:03 +0530 Subject: [PATCH 038/107] Added back the constants --- .../main/java/com/tmobile/pacman/api/commons/Constants.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/Constants.java b/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/Constants.java index c9be6f349..53f6e4a13 100644 --- a/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/Constants.java +++ b/commons/pac-api-commons/src/main/java/com/tmobile/pacman/api/commons/Constants.java @@ -321,5 +321,10 @@ public interface Constants { String NPE_PATTERN = "(^(npe|non.?prod(uction)?))|(.*:+(npe).*)"; String AZURE_WINDOWS = "Windows"; String CLOUD_KERNEL_COMPLIANCE_POLICY = "PacMan_cloud-kernel-compliance_version-1"; + String RESOURCE_IDS = "resourceIds"; + String RULE_IDS = "ruleIds"; + String TOTAL_VIOLATIONS = "totalViolations"; + String CLOUD_QUALYS_POLICY="PacMan_Ec2InstanceScannedByQualys_version-1"; + String VIRTUALMACHINE_KERNEL_COMPLIANCE_RULE = "PacMan_cloud-kernel-compliance_version-1_Virtualmachine-Kernel-Compliance-Rule_virtualmachine"; } From e4f565b7efaf8bd83d952806ccd8ae413cf13d0b Mon Sep 17 00:00:00 2001 From: Kanchana Date: Mon, 4 Nov 2019 21:36:06 +0530 Subject: [PATCH 039/107] asset fiegn client changes --- .../compliance/client/AssetServiceClient.java | 24 +++++++++++++++++-- .../repository/ComplianceRepositoryImpl.java | 6 ++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/client/AssetServiceClient.java b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/client/AssetServiceClient.java index 7a7aa1d09..242cfb2c1 100644 --- a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/client/AssetServiceClient.java +++ b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/client/AssetServiceClient.java @@ -29,6 +29,7 @@ import com.tmobile.pacman.api.compliance.domain.AssetApi; import com.tmobile.pacman.api.compliance.domain.AssetCount; +import com.tmobile.pacman.api.compliance.domain.ExemptedAssetByPolicy; /** * The Interface AssetServiceClient. @@ -36,18 +37,21 @@ @FeignClient(name = "assetclient", url = "${service.url.asset}") public interface AssetServiceClient { - /** + /** * Gets the total assets count. * * @param assetGroup the asset group * @param targetType the target type * @param domain the domain + * @param application the application * @return AssetCount */ @RequestMapping(method = RequestMethod.GET, value = "/v1/count") AssetCount getTotalAssetsCount(@RequestParam("ag") String assetGroup, @RequestParam("type") String targetType, - @RequestParam("domain") String domain); + @RequestParam("domain") String domain, + @RequestParam("application") String application, + @RequestParam("provider") String provider); /** * Gets the applications list. @@ -129,4 +133,20 @@ AssetCount getTotalAssetsCountByEnvironment( @RequestMapping(method = RequestMethod.GET, value = "/v1/list/targettype") AssetApi getTargetTypeListByDomain(@RequestParam("ag") String assetGroup, @RequestParam("domain") String domain); + + /** + * Gets the total assets exempted by policy. + * + * @param assetGroup the asset group + * @param application the application + * @param targetType the target type + * @param domain the domain + * @return the total assets exempted by policy + */ + @RequestMapping(method = RequestMethod.GET, value = "v1/count/exempted/bypolicy") + ExemptedAssetByPolicy getTotalAssetsExemptedByPolicy( + @RequestParam("ag") String assetGroup, + @RequestParam("application") String application, + @RequestParam("type") String targetType, + @RequestParam("domain") String domain); } diff --git a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/repository/ComplianceRepositoryImpl.java b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/repository/ComplianceRepositoryImpl.java index 3cc80b447..436d29ee1 100644 --- a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/repository/ComplianceRepositoryImpl.java +++ b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/repository/ComplianceRepositoryImpl.java @@ -445,7 +445,7 @@ public Map getTagging(String assetGroup, String targetType) throws ruleIdWithTargetTypeQuery = "SELECT A.targetType FROM cf_RuleInstance A, cf_Policy B WHERE A.policyId = B.policyId AND A.status = 'ENABLED' AND B.policyId = 'PacMan_TaggingRule_version-1'"; ruleIdwithTargetType = rdsepository.getDataFromPacman(ruleIdWithTargetTypeQuery); if (Strings.isNullOrEmpty(targetType)) { - assetCount = assetServiceClient.getTotalAssetsCount(assetGroup, targetType, null,null); + assetCount = assetServiceClient.getTotalAssetsCount(assetGroup, targetType, null,null,""); data = assetCount.getData(); assetcountCount = data.getAssetcount(); @@ -593,7 +593,7 @@ public List> getRecommendations(String assetGroup, String ta */ public Long getTotalAssetCountForAnytargetType(String assetGroup, String targetType) { - AssetCount totalAssets = assetServiceClient.getTotalAssetsCount(assetGroup, targetType, null,null); + AssetCount totalAssets = assetServiceClient.getTotalAssetsCount(assetGroup, targetType, null,null,""); AssetCountData data = totalAssets.getData(); AssetCountByAppEnvDTO[] assetcount = data.getAssetcount(); Long totalAssetsCount = 0l; @@ -2482,7 +2482,7 @@ public List> getDataSourceForTargetTypeForAG(String assetGro * getTotalAssetCount(java.lang.String, java.lang.String) */ public Map getTotalAssetCount(String assetGroup, String domain, String application, String type) { - AssetCount totalAssets = assetServiceClient.getTotalAssetsCount(assetGroup, type, domain, application); + AssetCount totalAssets = assetServiceClient.getTotalAssetsCount(assetGroup, type, domain, application,""); AssetCountData data = totalAssets.getData(); AssetCountByAppEnvDTO[] assetcount = data.getAssetcount(); Map assetCountByType = new HashMap<>(); From 73b234f4ae75334621591e8c759967e5235b36ab Mon Sep 17 00:00:00 2001 From: johnrexj Date: Tue, 5 Nov 2019 16:39:23 +0530 Subject: [PATCH 040/107] Tenant changes for Azure --- .../inventory/AzureFetchOrchestrator.java | 28 +++- .../BatchAccountInventoryCollector.java | 20 +-- .../BlobContainerInventoryCollector.java | 19 ++- .../collector/CosmosDBInventoryCollector.java | 31 ++--- .../DatabricksInventoryCollector.java | 26 ++-- .../collector/DiskInventoryCollector.java | 17 ++- .../LoadBalancerInventoryCollector.java | 20 ++- .../collector/MariaDBInventoryCollector.java | 28 ++-- .../collector/MySQLInventoryCollector.java | 27 ++-- .../collector/NSGInventoryCollector.java | 20 ++- .../NamespaceInventoryCollector.java | 20 ++- .../NetworkInterfaceInventoryCollector.java | 16 ++- .../collector/NetworkInventoryCollector.java | 19 ++- .../PolicyDefinitionInventoryCollector.java | 19 ++- .../PolicyStatesInventoryCollector.java | 23 ++-- .../PostgreSQLInventoryCollector.java | 24 ++-- .../PublicIpAddressInventoryCollector.java | 14 +- .../ResourceGroupInventoryCollector.java | 18 ++- .../RouteTableInventoryCollector.java | 15 ++- .../collector/SCRecommendationsCollector.java | 21 +-- .../SQLDatabaseInventoryCollector.java | 45 ++----- .../SQLServerInventoryCollector.java | 121 ------------------ .../SearchServiceInventoryCollector.java | 21 ++- .../SecurityAlertsInventoryCollector.java | 27 ++-- .../collector/SitesInventoryCollector.java | 19 ++- .../collector/SnapshotInventoryCollector.java | 16 ++- .../StorageAccountInventoryCollector.java | 14 +- .../collector/SubnetInventoryCollector.java | 20 +-- .../collector/VMInventoryCollector.java | 32 +++-- .../collector/VaultInventoryCollector.java | 19 ++- .../collector/WorkflowInventoryCollector.java | 16 +-- .../inventory/file/AssetFileGenerator.java | 36 +++--- .../azure/inventory/file/FileManager.java | 6 - .../azure/inventory/vo/SubscriptionVH.java | 10 +- 34 files changed, 399 insertions(+), 428 deletions(-) delete mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLServerInventoryCollector.java diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureFetchOrchestrator.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureFetchOrchestrator.java index 71cbbd652..8fa28ea69 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureFetchOrchestrator.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureFetchOrchestrator.java @@ -12,6 +12,10 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure.Authenticated; +import com.microsoft.azure.management.resources.Subscription; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.file.AssetFileGenerator; import com.tmobile.pacbot.azure.inventory.file.S3Uploader; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; @@ -22,6 +26,9 @@ public class AzureFetchOrchestrator { @Autowired AssetFileGenerator fileGenerator; + @Autowired + AzureCredentialProvider azureCredentialProvider; + /** The s 3 uploader. */ @Autowired S3Uploader s3Uploader; @@ -32,6 +39,8 @@ public class AzureFetchOrchestrator { /** The target types. */ @Value("${subscriptions:}") private String subscriptions; + @Value("${tenants:}") + private String tenants; @Value("${s3}") private String s3Bucket ; @@ -81,7 +90,7 @@ private List fetchSubscriptions() { List subscriptionList = new ArrayList<>(); - if(subscriptions != null && !"".equals(subscriptions)){ + /*if(subscriptions != null && !"".equals(subscriptions)){ String[] subscriptionsArray = subscriptions.split(","); for(String subcritpionInfo : subscriptionsArray){ SubscriptionVH subscription= new SubscriptionVH(); @@ -90,7 +99,24 @@ private List fetchSubscriptions() { subscription.setSubscriptionName(subIdName.length>1?subIdName[1].trim():""); subscriptionList.add(subscription); } + }*/ + + if(tenants != null && !"".equals(tenants)){ + String[] tenantList = tenants.split(","); + for(String tenant : tenantList){ + Authenticated azure = azureCredentialProvider.authenticate(tenant); + PagedList subscriptions = azure.subscriptions().list(); + for(Subscription subscription : subscriptions) { + SubscriptionVH subscriptionVH= new SubscriptionVH(); + subscriptionVH.setTenant(tenant); + subscriptionVH.setSubscriptionId(subscription.subscriptionId()); + subscriptionVH.setSubscriptionName(subscription.displayName()); + subscriptionList.add(subscriptionVH); + } + } } + log.info("Total Subscription in Scope : {}",subscriptionList.size()); + log.info("Subscriptions : {}",subscriptionList); return subscriptionList; } } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/BatchAccountInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/BatchAccountInventoryCollector.java index ae99f886a..4e0d85a4c 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/BatchAccountInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/BatchAccountInventoryCollector.java @@ -8,6 +8,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.gson.Gson; @@ -15,27 +16,24 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.BatchAccountVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; import com.tmobile.pacman.commons.utils.CommonUtils; @Component public class BatchAccountInventoryCollector { - private static Logger log = LoggerFactory.getLogger(BatchAccountInventoryCollector.class); + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger LOGGER = LoggerFactory.getLogger(BatchAccountInventoryCollector.class); private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.Batch/batchAccounts?api-version=2019-08-01"; public List fetchBatchAccountDetails(SubscriptionVH subscription) throws Exception { List batchAccountList = new ArrayList(); - String accessToken; - try { - accessToken = AzureCredentialManager.getAuthToken(); - - } catch (Exception e1) { - return batchAccountList; - } + String accessToken = azureCredentialProvider.getToken(subscription.getTenant()); String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); try { @@ -80,8 +78,10 @@ public List fetchBatchAccountDetails(SubscriptionVH subscription } } } catch (Exception e) { - e.printStackTrace(); + LOGGER.error("Error fetching BatchAccount",e); } + + LOGGER.info("Target Type : {} Total: {} ","Batch Account",batchAccountList.size()); return batchAccountList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/BlobContainerInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/BlobContainerInventoryCollector.java index 162b7e897..50f593689 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/BlobContainerInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/BlobContainerInventoryCollector.java @@ -8,6 +8,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.gson.Gson; @@ -18,27 +19,25 @@ import com.microsoft.azure.PagedList; import com.microsoft.azure.management.Azure; import com.microsoft.azure.management.storage.StorageAccount; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.BlobContainerVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; import com.tmobile.pacman.commons.utils.CommonUtils; @Component public class BlobContainerInventoryCollector { - + + @Autowired + AzureCredentialProvider azureCredentialProvider; + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Storage/storageAccounts/%s/blobServices/default/containers?api-version=2019-04-01"; private static Logger log = LoggerFactory.getLogger(BlobContainerInventoryCollector.class); public List fetchBlobContainerDetails(SubscriptionVH subscription,Map> tagMap) { List blobContainerList = new ArrayList(); - String accessToken; - try { - accessToken = AzureCredentialManager.getAuthToken(); - } catch (Exception e1) { - return blobContainerList; - } - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + String accessToken = azureCredentialProvider.getToken(subscription.getTenant()); + Azure azure = azureCredentialProvider.getClient(subscription.getTenant(),subscription.getSubscriptionId()); PagedList storageAccounts = azure.storageAccounts().list(); for (StorageAccount storageAccount : storageAccounts) { String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId()), @@ -71,7 +70,7 @@ public List fetchBlobContainerDetails(SubscriptionVH subscripti } } - System.out.println(blobContainerList.size()); + log.info("Target Type : {} Total: {} ","Blob Container",blobContainerList.size()); return blobContainerList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/CosmosDBInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/CosmosDBInventoryCollector.java index 0f0099c97..fbc277dfc 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/CosmosDBInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/CosmosDBInventoryCollector.java @@ -4,25 +4,32 @@ import java.util.List; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.microsoft.azure.PagedList; import com.microsoft.azure.management.Azure; import com.microsoft.azure.management.cosmosdb.CosmosDBAccount; import com.microsoft.azure.management.cosmosdb.VirtualNetworkRule; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.CosmosDBVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; import com.tmobile.pacbot.azure.inventory.vo.VirtualNetworkRuleVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; @Component public class CosmosDBInventoryCollector { + + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger log = LoggerFactory.getLogger(CosmosDBInventoryCollector.class); public List fetchCosmosDBDetails(SubscriptionVH subscription, Map> tagMap) { - List cosmosDBList = new ArrayList(); - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + List cosmosDBList = new ArrayList<>(); + Azure azure = azureCredentialProvider.getClient(subscription.getTenant(),subscription.getSubscriptionId()); PagedList CosmosDB = azure.cosmosDBAccounts().list(); - System.out.println(CosmosDB.size()); for (CosmosDBAccount cosmosDB : CosmosDB) { CosmosDBVH cosmosDBVH = new CosmosDBVH(); cosmosDBVH.setSubscription(subscription.getSubscriptionId()); @@ -38,22 +45,8 @@ public List fetchCosmosDBDetails(SubscriptionVH subscription, Map tagsFinal = new HashMap(); - * - * for (Map.Entry> resourceGroupTag : - * tagMap.entrySet()) { - * - * if (resourceGroupTag.getKey().equalsIgnoreCase(cosmosDB.resourceGroupName())) - * { flag = true; tagsFinal.putAll(resourceGroupTag.getValue()); - * tagsFinal.putAll(cosmosDB.tags()); break; } - * - * } if (flag == true) { cosmosDBVH.setTags(tagsFinal); } else { - * cosmosDBVH.setTags(cosmosDB.tags()); } - */ - } + log.info("Target Type : {} Total: {} ","Cosom DB",cosmosDBList.size()); return cosmosDBList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/DatabricksInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/DatabricksInventoryCollector.java index f752b527d..6b058435c 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/DatabricksInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/DatabricksInventoryCollector.java @@ -5,6 +5,9 @@ import java.util.HashMap; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.gson.Gson; @@ -12,28 +15,25 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import com.microsoft.azure.management.Azure; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.DatabricksVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; import com.tmobile.pacman.commons.utils.CommonUtils; @Component public class DatabricksInventoryCollector { - + + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger log = LoggerFactory.getLogger(DatabricksInventoryCollector.class); private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.Databricks/workspaces?api-version=2018-04-01"; public List fetchDatabricksDetails(SubscriptionVH subscription) { List databricksList = new ArrayList(); - String accessToken; - try { - accessToken = AzureCredentialManager.getAuthToken(); - } catch (Exception e1) { - return databricksList; - } - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); - + String accessToken = azureCredentialProvider.getToken(subscription.getTenant()); + String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); try { String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); @@ -61,10 +61,10 @@ public List fetchDatabricksDetails(SubscriptionVH subscription) { databricksList.add(databricksVH); } } catch (Exception e) { - e.printStackTrace(); + log.info("Error Collecting Databricks",e); } - System.out.println(databricksList.size()); + log.info("Target Type : {} Total: {} ","Databrick",databricksList.size()); return databricksList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/DiskInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/DiskInventoryCollector.java index 42d064845..726d6e0cc 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/DiskInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/DiskInventoryCollector.java @@ -1,26 +1,34 @@ package com.tmobile.pacbot.azure.inventory.collector; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.microsoft.azure.PagedList; import com.microsoft.azure.management.Azure; import com.microsoft.azure.management.compute.Disk; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.DataDiskVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; @Component public class DiskInventoryCollector { + + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger log = LoggerFactory.getLogger(DiskInventoryCollector.class); + public List fetchDataDiskDetails(SubscriptionVH subscription, Map> tagMap) { List dataDiskList = new ArrayList(); - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + Azure azure = azureCredentialProvider.getClient(subscription.getTenant(),subscription.getSubscriptionId()); PagedList dataDisks = azure.disks().list(); - System.out.println(dataDisks.size()); + for (Disk dataDisk : dataDisks) { DataDiskVH dataDiskVH = new DataDiskVH(); dataDiskVH.setId(dataDisk.id()); @@ -38,6 +46,7 @@ public List fetchDataDiskDetails(SubscriptionVH subscription, Map fetchLoadBalancerDetails(SubscriptionVH subscription, Map> tagMap) { - List loadBalancerList = new ArrayList(); + List loadBalancerList = new ArrayList<>(); - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + Azure azure = azureCredentialProvider.getClient(subscription.getTenant(),subscription.getSubscriptionId()); PagedList loadBalancers = azure.loadBalancers().list(); - System.out.println(loadBalancers.size()); for (LoadBalancer loadBalancer : loadBalancers) { LoadBalancerVH loadBalancerVH = new LoadBalancerVH(); loadBalancerVH.setHashCode(loadBalancer.hashCode()); @@ -40,7 +46,7 @@ public List fetchLoadBalancerDetails(SubscriptionVH subscription loadBalancerList.add(loadBalancerVH); } - + log.info("Target Type : {} Total: {} ","LoadBalancer",loadBalancerList.size()); return loadBalancerList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/MariaDBInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/MariaDBInventoryCollector.java index 690957e28..0376ba9c7 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/MariaDBInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/MariaDBInventoryCollector.java @@ -5,6 +5,9 @@ import java.util.HashMap; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.gson.Gson; @@ -12,28 +15,25 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import com.microsoft.azure.management.Azure; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.MariaDBVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; import com.tmobile.pacman.commons.utils.CommonUtils; @Component public class MariaDBInventoryCollector { - + + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger log = LoggerFactory.getLogger(MariaDBInventoryCollector.class); private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.DBforMariaDB/servers?api-version=2018-06-01-preview"; public List fetchMariaDBDetails(SubscriptionVH subscription) { - List mariaDBList = new ArrayList(); - String accessToken; - try { - accessToken = AzureCredentialManager.getAuthToken(); - } catch (Exception e1) { - return mariaDBList; - } - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); - + List mariaDBList = new ArrayList<>(); + String accessToken = azureCredentialProvider.getToken(subscription.getTenant()); + String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); try { String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); @@ -61,10 +61,10 @@ public List fetchMariaDBDetails(SubscriptionVH subscription) { mariaDBList.add(mariaDBVH); } } catch (Exception e) { - e.printStackTrace(); + log.error("Error Collecting MariaDB",e); } - System.out.println(mariaDBList.size()); + log.info("Target Type : {} Total: {} ","MariaDB",mariaDBList.size()); return mariaDBList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/MySQLInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/MySQLInventoryCollector.java index e34c322d2..dd79f6c0d 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/MySQLInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/MySQLInventoryCollector.java @@ -5,6 +5,9 @@ import java.util.HashMap; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.gson.Gson; @@ -12,29 +15,25 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import com.microsoft.azure.PagedList; -import com.microsoft.azure.management.Azure; -import com.microsoft.azure.management.storage.StorageAccount; -import com.tmobile.pacbot.azure.inventory.vo.BlobContainerVH; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.MySQLServerVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; import com.tmobile.pacman.commons.utils.CommonUtils; @Component public class MySQLInventoryCollector { + + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger log = LoggerFactory.getLogger(MySQLInventoryCollector.class); + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.DBforMySQL/servers?api-version=2017-12-01"; public List fetchMySQLServerDetails(SubscriptionVH subscription) { List mySqlServerList = new ArrayList(); - String accessToken; - try { - accessToken = AzureCredentialManager.getAuthToken(); - } catch (Exception e1) { - return mySqlServerList; - } - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + String accessToken = azureCredentialProvider.getToken(subscription.getTenant()); String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); try { @@ -64,10 +63,10 @@ public List fetchMySQLServerDetails(SubscriptionVH subscription) mySqlServerList.add(mySQLServerVH); } } catch (Exception e) { - e.printStackTrace(); + log.error("Error Collecting mysqlserver",e); } - System.out.println(mySqlServerList.size()); + log.info("Target Type : {} Total: {} ","MySQL Server",mySqlServerList.size()); return mySqlServerList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NSGInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NSGInventoryCollector.java index 6bc168e1d..c3cf6fdf0 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NSGInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NSGInventoryCollector.java @@ -1,10 +1,12 @@ package com.tmobile.pacbot.azure.inventory.collector; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.microsoft.azure.PagedList; @@ -12,19 +14,25 @@ import com.microsoft.azure.management.network.NetworkSecurityGroup; import com.microsoft.azure.management.network.NetworkSecurityRule; import com.microsoft.azure.management.network.Subnet; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; +import com.tmobile.pacbot.azure.inventory.vo.NSGSecurityRule; import com.tmobile.pacbot.azure.inventory.vo.NSGSubnet; import com.tmobile.pacbot.azure.inventory.vo.SecurityGroupVH; -import com.tmobile.pacbot.azure.inventory.vo.NSGSecurityRule; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; @Component public class NSGInventoryCollector { + + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger log = LoggerFactory.getLogger(NSGInventoryCollector.class); + public List fetchNetworkSecurityGroupDetails(SubscriptionVH subscription, Map> tagMap) { - List securityGroupsList = new ArrayList(); + List securityGroupsList = new ArrayList<>(); - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + Azure azure = azureCredentialProvider.getClient(subscription.getTenant(),subscription.getSubscriptionId()); PagedList securityGroups = azure.networkSecurityGroups().list(); for (NetworkSecurityGroup securityGroup : securityGroups) { SecurityGroupVH securityGroupVH = new SecurityGroupVH(); @@ -42,7 +50,7 @@ public List fetchNetworkSecurityGroupDetails(SubscriptionVH sub securityGroupsList.add(securityGroupVH); } - + log.info("Target Type : {} Total: {} ","Nsg",securityGroupsList.size()); return securityGroupsList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NamespaceInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NamespaceInventoryCollector.java index 2db13bf02..ac69feaf9 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NamespaceInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NamespaceInventoryCollector.java @@ -7,6 +7,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.gson.Gson; @@ -14,27 +15,24 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.NamespaceVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; import com.tmobile.pacman.commons.utils.CommonUtils; @Component public class NamespaceInventoryCollector { - + + @Autowired + AzureCredentialProvider azureCredentialProvider; + private static Logger log = LoggerFactory.getLogger(NamespaceInventoryCollector.class); private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.EventHub/namespaces?api-version=2017-04-01"; public List fetchNamespaceDetails(SubscriptionVH subscription) throws Exception { List namespaceList = new ArrayList(); - String accessToken; - try { - accessToken = AzureCredentialManager.getAuthToken(); - - } catch (Exception e1) { - return namespaceList; - } + String accessToken = azureCredentialProvider.getToken(subscription.getTenant()); String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); try { @@ -74,10 +72,10 @@ public List fetchNamespaceDetails(SubscriptionVH subscription) thro } } } catch (Exception e) { - e.printStackTrace(); + log.error("Error collecting namespace",e); } - System.out.println(namespaceList.size()); + log.info("Target Type : {} Total: {} ","Namespace",namespaceList.size()); return namespaceList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NetworkInterfaceInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NetworkInterfaceInventoryCollector.java index 5dde5638f..c96ce7552 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NetworkInterfaceInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NetworkInterfaceInventoryCollector.java @@ -1,29 +1,36 @@ package com.tmobile.pacbot.azure.inventory.collector; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.microsoft.azure.PagedList; import com.microsoft.azure.management.Azure; import com.microsoft.azure.management.network.NetworkInterface; import com.microsoft.azure.management.network.NicIPConfiguration; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.NIIPConfigVH; import com.tmobile.pacbot.azure.inventory.vo.NetworkInterfaceVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; @Component public class NetworkInterfaceInventoryCollector { - + + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger log = LoggerFactory.getLogger(NetworkInterfaceInventoryCollector.class); + public List fetchNetworkInterfaceDetails(SubscriptionVH subscription, Map> tagMap) { List networkInterfaceList = new ArrayList(); - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + Azure azure = azureCredentialProvider.getClient(subscription.getTenant(),subscription.getSubscriptionId()); PagedList networkInterfaces = azure.networkInterfaces().list(); for (NetworkInterface networkInterface : networkInterfaces) { @@ -50,6 +57,7 @@ public List fetchNetworkInterfaceDetails(SubscriptionVH subs networkInterfaceList.add(networkInterfaceVH); } + log.info("Target Type : {} Total: {} ","Networkinterface",networkInterfaceList.size()); return networkInterfaceList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NetworkInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NetworkInventoryCollector.java index fc2da4388..5772c20a8 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NetworkInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/NetworkInventoryCollector.java @@ -1,26 +1,33 @@ package com.tmobile.pacbot.azure.inventory.collector; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.microsoft.azure.PagedList; import com.microsoft.azure.management.Azure; import com.microsoft.azure.management.network.Network; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.NetworkVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; @Component public class NetworkInventoryCollector { - + + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger log = LoggerFactory.getLogger(NetworkInventoryCollector.class); + public List fetchNetworkDetails(SubscriptionVH subscription, Map> tagMap) { - List networkList = new ArrayList(); + List networkList = new ArrayList<>(); - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + Azure azure = azureCredentialProvider.getClient(subscription.getTenant(),subscription.getSubscriptionId()); PagedList networks = azure.networks().list(); for (Network network : networks) { @@ -41,7 +48,7 @@ public List fetchNetworkDetails(SubscriptionVH subscription, Map fetchPolicyDefinitionDetails(SubscriptionVH subscription) { - List policyDefinitionList = new ArrayList(); - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + List policyDefinitionList = new ArrayList<>(); + Azure azure = azureCredentialProvider.getClient(subscription.getTenant(),subscription.getSubscriptionId()); PagedList policyDefinitions = azure.policyDefinitions().list(); - System.out.println(policyDefinitions.size()); for (PolicyDefinition policyDefinition : policyDefinitions) { PolicyDefinitionVH policyDefinitionVH = new PolicyDefinitionVH(); policyDefinitionVH.setId(policyDefinition.id()); @@ -33,6 +39,7 @@ public List fetchPolicyDefinitionDetails(SubscriptionVH subs policyDefinitionVH.setSubscriptionName(subscription.getSubscriptionName()); policyDefinitionList.add(policyDefinitionVH); } + log.info("Target Type : {} Total: {} ","Policy Defintion",policyDefinitionList.size()); return policyDefinitionList; } } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PolicyStatesInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PolicyStatesInventoryCollector.java index 1a455e4e4..db38e614b 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PolicyStatesInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PolicyStatesInventoryCollector.java @@ -4,33 +4,36 @@ import java.util.ArrayList; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.PolicyDefinitionVH; import com.tmobile.pacbot.azure.inventory.vo.PolicyStatesVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; import com.tmobile.pacman.commons.utils.CommonUtils; @Component public class PolicyStatesInventoryCollector { + + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger log = LoggerFactory.getLogger(PolicyStatesInventoryCollector.class); + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.PolicyInsights/policyStates/latest/queryResults?api-version=2018-04-04"; public List fetchPolicyStatesDetails(SubscriptionVH subscription, List policyDefinitionList) throws Exception { List policyStatesList = new ArrayList(); - String accessToken; - try { - accessToken = AzureCredentialManager.getAuthToken(); - - } catch (Exception e1) { - return policyStatesList; - } + String accessToken = azureCredentialProvider.getToken(subscription.getTenant()); String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); try { @@ -90,10 +93,10 @@ public List fetchPolicyStatesDetails(SubscriptionVH subscription policyStatesList.add(policyStatesVH); } } catch (Exception e) { - e.printStackTrace(); + log.error("Error collleting Policy States",e); } - System.out.println(policyStatesList.size()); + log.info("Target Type : {} Total: {} ","Policy States",policyStatesList.size()); return policyStatesList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PostgreSQLInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PostgreSQLInventoryCollector.java index 2394b310d..39dafd798 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PostgreSQLInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PostgreSQLInventoryCollector.java @@ -5,6 +5,9 @@ import java.util.HashMap; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.gson.Gson; @@ -12,26 +15,25 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import com.microsoft.azure.management.Azure; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.PostgreSQLServerVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; import com.tmobile.pacman.commons.utils.CommonUtils; @Component public class PostgreSQLInventoryCollector { + + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger log = LoggerFactory.getLogger(PostgreSQLInventoryCollector.class); + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.DBforPostgreSQL/servers?api-version=2017-12-01"; public List fetchPostgreSQLServerDetails(SubscriptionVH subscription) { List postgreSQLServerList = new ArrayList(); - String accessToken; - try { - accessToken = AzureCredentialManager.getAuthToken(); - } catch (Exception e1) { - return postgreSQLServerList; - } - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + String accessToken = azureCredentialProvider.getToken(subscription.getTenant()); String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); try { @@ -60,10 +62,10 @@ public List fetchPostgreSQLServerDetails(SubscriptionVH subs postgreSQLServerList.add(postgreSQLServerVH); } } catch (Exception e) { - e.printStackTrace(); + log.error("Error collectig PostGresDB",e); } - System.out.println(postgreSQLServerList.size()); + log.info("Target Type : {} Total: {} ","Postgres DB",postgreSQLServerList.size()); return postgreSQLServerList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PublicIpAddressInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PublicIpAddressInventoryCollector.java index 63d0016cb..0390c8740 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PublicIpAddressInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PublicIpAddressInventoryCollector.java @@ -4,26 +4,33 @@ import java.util.List; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.microsoft.azure.PagedList; import com.microsoft.azure.management.Azure; import com.microsoft.azure.management.network.PublicIPAddress; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.PublicIpAddressVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; @Component public class PublicIpAddressInventoryCollector { + + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger log = LoggerFactory.getLogger(PublicIpAddressInventoryCollector.class); public List fetchPublicIpAddressDetails(SubscriptionVH subscription, Map> tagMap) { List publicIpAddressList = new ArrayList(); - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + Azure azure = azureCredentialProvider.getClient(subscription.getTenant(),subscription.getSubscriptionId()); PagedList publicIPAddresses = azure.publicIPAddresses().list(); - System.out.println(publicIPAddresses.size()); for (PublicIPAddress publicIPAddress : publicIPAddresses) { PublicIpAddressVH publicIpAddressVH = new PublicIpAddressVH(); publicIpAddressVH.setId(publicIPAddress.id()); @@ -44,6 +51,7 @@ public List fetchPublicIpAddressDetails(SubscriptionVH subscr publicIpAddressList.add(publicIpAddressVH); } + log.info("Target Type : {} Total: {} ","PublicIPAddress",publicIpAddressList.size()); return publicIpAddressList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/ResourceGroupInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/ResourceGroupInventoryCollector.java index 306075a9e..a81a3fd16 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/ResourceGroupInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/ResourceGroupInventoryCollector.java @@ -2,24 +2,31 @@ import java.util.ArrayList; import java.util.List; -import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.microsoft.azure.PagedList; import com.microsoft.azure.management.Azure; import com.microsoft.azure.management.resources.ResourceGroup; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.ResourceGroupVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; @Component public class ResourceGroupInventoryCollector { + + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger log = LoggerFactory.getLogger(ResourceGroupInventoryCollector.class); + public List fetchResourceGroupDetails(SubscriptionVH subscription) { - List resourceGroupList = new ArrayList(); - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + List resourceGroupList = new ArrayList<>(); + Azure azure = azureCredentialProvider.getClient(subscription.getTenant(),subscription.getSubscriptionId()); PagedList resourceGroups = azure.resourceGroups().list(); - System.out.println(resourceGroups.size()); for (ResourceGroup resourceGroup : resourceGroups) { ResourceGroupVH resourceGroupVH = new ResourceGroupVH(); resourceGroupVH.setSubscription(subscription.getSubscriptionId()); @@ -33,6 +40,7 @@ public List fetchResourceGroupDetails(SubscriptionVH subscripti resourceGroupVH.setTags(resourceGroup.tags()); resourceGroupList.add(resourceGroupVH); } + log.info("Target Type : {} Total: {} ","ResourceGroup",resourceGroupList.size()); return resourceGroupList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/RouteTableInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/RouteTableInventoryCollector.java index 258c7e747..08824be2e 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/RouteTableInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/RouteTableInventoryCollector.java @@ -4,6 +4,9 @@ import java.util.List; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.microsoft.azure.PagedList; @@ -11,22 +14,26 @@ import com.microsoft.azure.management.network.Route; import com.microsoft.azure.management.network.RouteTable; import com.microsoft.azure.management.network.Subnet; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.RouteTableSubnet; import com.tmobile.pacbot.azure.inventory.vo.RouteTableVH; import com.tmobile.pacbot.azure.inventory.vo.RouteVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; @Component public class RouteTableInventoryCollector { + + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger log = LoggerFactory.getLogger(RouteTableInventoryCollector.class); public List fetchRouteTableDetails(SubscriptionVH subscription, Map> tagMap) { List routeTableDetailsList = new ArrayList(); - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + Azure azure = azureCredentialProvider.getClient(subscription.getTenant(),subscription.getSubscriptionId()); PagedList routTableList = azure.routeTables().list(); - System.out.println(routTableList.size()); for (RouteTable routTable : routTableList) { RouteTableVH routeTableVH = new RouteTableVH(); routeTableVH.setHashCode(routTable.hashCode()); @@ -44,7 +51,7 @@ public List fetchRouteTableDetails(SubscriptionVH subscription, routeTableDetailsList.add(routeTableVH); } - + log.info("Target Type : {} Total: {} ","Route Table",routeTableDetailsList.size()); return routeTableDetailsList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SCRecommendationsCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SCRecommendationsCollector.java index a1cfe6e67..565a4ed31 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SCRecommendationsCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SCRecommendationsCollector.java @@ -6,6 +6,9 @@ import java.util.Map; import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.gson.Gson; @@ -14,36 +17,34 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.gson.reflect.TypeToken; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.RecommendationVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; import com.tmobile.pacman.commons.utils.CommonUtils; @Component public class SCRecommendationsCollector { + @Autowired + AzureCredentialProvider azureCredentialProvider; + Set policyList = new HashSet<>(); Set nameList = new HashSet<>(); Set baseNameList = new HashSet<>(); - + private static Logger log = LoggerFactory.getLogger(SCRecommendationsCollector.class); private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.Security/tasks?api-version=2015-06-01-preview"; public List fetchSecurityCenterRecommendations(SubscriptionVH subscription) { List recommendations = new ArrayList<>(); - String accessToken; - try { - accessToken = AzureCredentialManager.getAuthToken(); - } catch (Exception e1) { - return recommendations; - } + String accessToken = azureCredentialProvider.getToken(subscription.getTenant()); String url = String.format(apiUrlTemplate, subscription.getSubscriptionId()); try { String response = CommonUtils.doHttpGet(url, "Bearer", accessToken); recommendations = filterRecommendationInfo(response,subscription); } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); + log.error("Error Collecting Security Center Info",e); } + log.info("Target Type : {} Total: {} ","Security Center",recommendations.size()); return recommendations; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLDatabaseInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLDatabaseInventoryCollector.java index ec8a37e7e..81ec46e25 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLDatabaseInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLDatabaseInventoryCollector.java @@ -6,6 +6,9 @@ import java.util.List; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.microsoft.azure.PagedList; @@ -14,21 +17,24 @@ import com.microsoft.azure.management.sql.SqlFirewallRule; import com.microsoft.azure.management.sql.SqlServer; import com.microsoft.azure.management.sql.SqlVirtualNetworkRule; -import com.tmobile.pacbot.azure.inventory.vo.FirewallRules; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.SQLDatabaseVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacbot.azure.inventory.vo.VirtualNetworkRuleVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; @Component public class SQLDatabaseInventoryCollector { + + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger log = LoggerFactory.getLogger(SQLDatabaseInventoryCollector.class); public List fetchSQLDatabaseDetails(SubscriptionVH subscription, Map> tagMap) { List sqlDatabaseList = new ArrayList(); - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + Azure azure = azureCredentialProvider.getClient(subscription.getTenant(),subscription.getSubscriptionId()); PagedList sqlServers = azure.sqlServers().list(); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); for (SqlServer sqlServer : sqlServers) { @@ -69,46 +75,17 @@ public List fetchSQLDatabaseDetails(SubscriptionVH subscription, } firewallRule(sqlServer, sqlDatabaseVH); - // sqlDatabaseVH.setFirewallRuleDetails(getFirewallRuleDetails(sqlServer.firewallRules().list())); - // sqlDatabaseVH.setVirtualNetworkRuleDetails( - // getVirtualNetworkRuleDetails(sqlServer.virtualNetworkRules().list())); sqlDatabaseList.add(sqlDatabaseVH); } } } + log.info("Target Type : {} Total: {} ","Sql Databse",sqlDatabaseList.size()); return sqlDatabaseList; } - /* - * private List getFirewallRuleDetails(List - * sqlFirewallRuleList) { List firewallRulesList = new - * ArrayList<>(); for (SqlFirewallRule sqlFirewallRule : sqlFirewallRuleList) { - * FirewallRules firewallRuleVH = new FirewallRules(); - * firewallRuleVH.setName(sqlFirewallRule.name()); - * firewallRuleVH.setStartIPAddress(sqlFirewallRule.startIPAddress()); - * firewallRuleVH.setEndIPAddress(sqlFirewallRule.endIPAddress()); - * firewallRulesList.add(firewallRuleVH); } return firewallRulesList; - * - * } - * - * private List getVirtualNetworkRuleDetails( - * List sqlVirtualNetworkRuleList) { - * List virtualNetworkRuleList = new ArrayList<>(); for - * (SqlVirtualNetworkRule sqlVirtualNetworkRule : sqlVirtualNetworkRuleList) { - * VirtualNetworkRule virtualNetworkRuleVH = new VirtualNetworkRule(); - * virtualNetworkRuleVH.setName(sqlVirtualNetworkRule.name()); - * virtualNetworkRuleVH.setSubnetId(sqlVirtualNetworkRule.subnetId()); - * virtualNetworkRuleVH.setResourceGroupName(sqlVirtualNetworkRule. - * resourceGroupName()); - * virtualNetworkRuleVH.setState(sqlVirtualNetworkRule.state()); - * virtualNetworkRuleList.add(virtualNetworkRuleVH); } return - * virtualNetworkRuleList; - * - * } - */ private void firewallRule(SqlServer sqlServer, SQLDatabaseVH sqlDatabaseVH) { List> firewallRuleList = new ArrayList<>(); diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLServerInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLServerInventoryCollector.java deleted file mode 100644 index 2ff114552..000000000 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLServerInventoryCollector.java +++ /dev/null @@ -1,121 +0,0 @@ -package com.tmobile.pacbot.azure.inventory.collector; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.springframework.stereotype.Component; - -import com.microsoft.azure.PagedList; -import com.microsoft.azure.management.Azure; -import com.microsoft.azure.management.sql.SqlElasticPool; -import com.microsoft.azure.management.sql.SqlFailoverGroup; -import com.microsoft.azure.management.sql.SqlFirewallRule; -import com.microsoft.azure.management.sql.SqlServer; -import com.microsoft.azure.management.sql.SqlVirtualNetworkRule; -import com.tmobile.pacbot.azure.inventory.vo.ElasticPoolVH; -import com.tmobile.pacbot.azure.inventory.vo.FailoverGroupVH; -import com.tmobile.pacbot.azure.inventory.vo.SQLServerVH; -import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; - -@Component -public class SQLServerInventoryCollector { - public List fetchSQLServerDetails(SubscriptionVH subscription, - Map> tagMap) { - - List sqlServerList = new ArrayList(); - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); - PagedList sqlServers = azure.sqlServers().list(); - for (SqlServer sqlServer : sqlServers) { - SQLServerVH sqlServerVH = new SQLServerVH(); - sqlServerVH.setSubscription(subscription.getSubscriptionId()); - sqlServerVH.setSubscriptionName(subscription.getSubscriptionName()); - sqlServerVH.setId(sqlServer.id()); - sqlServerVH.setKind(sqlServer.kind()); - sqlServerVH.setName(sqlServer.name()); - sqlServerVH.setRegionName(sqlServer.regionName()); - sqlServerVH.setState(sqlServer.state()); - sqlServerVH.setSystemAssignedManagedServiceIdentityPrincipalId( - sqlServer.systemAssignedManagedServiceIdentityPrincipalId()); - sqlServerVH.setSystemAssignedManagedServiceIdentityTenantId( - sqlServer.systemAssignedManagedServiceIdentityTenantId()); - sqlServerVH.setTags(Util.tagsList(tagMap, sqlServer.resourceGroupName(), sqlServer.tags())); - sqlServerVH.setVersion(sqlServer.version()); - sqlServerVH.setAdministratorLogin(sqlServer.administratorLogin()); - firewallRule(sqlServer, sqlServerVH); - getElasticPoolList(sqlServer.elasticPools().list(), sqlServerVH); - getFailoverGroupList(sqlServer.failoverGroups().list(), sqlServerVH); - sqlServerList.add(sqlServerVH); - } - - return sqlServerList; - - } - - private void getElasticPoolList(List sqlElasticPoolList, SQLServerVH sqlServerVH) { - List elasticPoolList = new ArrayList<>(); - for (SqlElasticPool sqlElasticPool : sqlElasticPoolList) { - ElasticPoolVH elasticPoolVH = new ElasticPoolVH(); - elasticPoolVH.setName(sqlElasticPool.name()); - elasticPoolVH.setSize(sqlElasticPool.listDatabases().size()); - elasticPoolVH.setStorageCapacity(sqlElasticPool.storageCapacityInMB()); - elasticPoolVH.setId(sqlElasticPool.id()); - elasticPoolVH.setStorageMB(sqlElasticPool.storageMB()); - elasticPoolVH.setDtu(sqlElasticPool.dtu()); - elasticPoolVH.setEdition(sqlElasticPool.edition().toString()); - elasticPoolList.add(elasticPoolVH); - - } - sqlServerVH.setElasticPoolList(elasticPoolList); - - } - - private void firewallRule(SqlServer sqlServer, SQLServerVH sqlServerVH) { - List> firewallRuleList = new ArrayList<>(); - Map firewallMap; - for (SqlFirewallRule sqlFirewallRule : sqlServer.firewallRules().list()) { - firewallMap = new HashMap<>(); - firewallMap.put("name", sqlFirewallRule.name()); - firewallMap.put("startIPAddress", sqlFirewallRule.startIPAddress()); - firewallMap.put("endIPAddress", sqlFirewallRule.endIPAddress()); - firewallRuleList.add(firewallMap); - - } - for (SqlVirtualNetworkRule sqlVirtualNetworkRule : sqlServer.virtualNetworkRules().list()) { - firewallMap = new HashMap<>(); - - firewallMap.put("virtualNetworkRuleName", - sqlVirtualNetworkRule.name() != null ? sqlVirtualNetworkRule.name() : ""); - firewallMap.put("virtualNetworkSubnetId", - sqlVirtualNetworkRule.subnetId() != null ? sqlVirtualNetworkRule.subnetId() : ""); - firewallMap.put("virtualNetworkResourceGroupName", - sqlVirtualNetworkRule.resourceGroupName() != null ? sqlVirtualNetworkRule.resourceGroupName() : ""); - firewallMap.put("virtualNetworkState", - sqlVirtualNetworkRule.state() != null ? sqlVirtualNetworkRule.state() : ""); - - firewallRuleList.add(firewallMap); - } - sqlServerVH.setFirewallRuleDetails(firewallRuleList); - } - - private void getFailoverGroupList(List sqlFailoverGroupList, SQLServerVH sqlServerVH) { - List failoverGroupList = new ArrayList<>(); - for (SqlFailoverGroup sqlFailoverGroup : sqlFailoverGroupList) { - FailoverGroupVH failoverGroupVH = new FailoverGroupVH(); - failoverGroupVH.setSize(sqlFailoverGroup.databases().size()); - failoverGroupVH.setId(sqlFailoverGroup.id()); - failoverGroupVH.setName(sqlFailoverGroup.name()); - failoverGroupVH.setReplicationState(sqlFailoverGroup.replicationState()); - failoverGroupVH.setReadOnlyEndpointPolicy(sqlFailoverGroup.readOnlyEndpointPolicy().toString()); - failoverGroupVH.setReadWriteEndpointPolicy(sqlFailoverGroup.readWriteEndpointPolicy().toString()); - failoverGroupVH.setGracePeriod(sqlFailoverGroup.readWriteEndpointDataLossGracePeriodMinutes()); - failoverGroupList.add(failoverGroupVH); - - } - sqlServerVH.setFailoverGroupList(failoverGroupList); - - } - -} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SearchServiceInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SearchServiceInventoryCollector.java index be6b9af52..57e525d81 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SearchServiceInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SearchServiceInventoryCollector.java @@ -7,6 +7,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.gson.Gson; @@ -14,28 +15,24 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.SearchServiceVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; import com.tmobile.pacman.commons.utils.CommonUtils; @Component public class SearchServiceInventoryCollector { - + + @Autowired + AzureCredentialProvider azureCredentialProvider; + private static Logger log = LoggerFactory.getLogger(SearchServiceInventoryCollector.class); private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.Search/searchServices?api-version=2015-08-19"; public List fetchSearchServiceDetails(SubscriptionVH subscription) throws Exception { List searchServiceList = new ArrayList(); - String accessToken; - try { - accessToken = AzureCredentialManager.getAuthToken(); - - } catch (Exception e1) { - return searchServiceList; - } - + String accessToken = azureCredentialProvider.getToken(subscription.getTenant()); String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); try { @@ -69,10 +66,10 @@ public List fetchSearchServiceDetails(SubscriptionVH subscripti } } } catch (Exception e) { - e.printStackTrace(); + log.error("Error collecting Search Service",e); } - System.out.println(searchServiceList.size()); + log.info("Target Type : {} Total: {} ","Search Service",searchServiceList.size()); return searchServiceList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SecurityAlertsInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SecurityAlertsInventoryCollector.java index 49fcbd234..08c6f7a3f 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SecurityAlertsInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SecurityAlertsInventoryCollector.java @@ -5,6 +5,9 @@ import java.util.HashMap; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.gson.Gson; @@ -12,28 +15,24 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import com.microsoft.azure.management.Azure; -import com.tmobile.pacbot.azure.inventory.vo.DatabricksVH; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.SecurityAlertsVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; import com.tmobile.pacman.commons.utils.CommonUtils; @Component public class SecurityAlertsInventoryCollector { - + + @Autowired + AzureCredentialProvider azureCredentialProvider; + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.Security/alerts?api-version=2019-01-01"; - + private static Logger log = LoggerFactory.getLogger(SecurityAlertsInventoryCollector.class); + public List fetchSecurityAlertsDetails(SubscriptionVH subscription) throws Exception { List securityAlertsList = new ArrayList(); - String accessToken; - try { - accessToken = AzureCredentialManager.getAuthToken(); - - } catch (Exception e1) { - return securityAlertsList; - } + String accessToken = azureCredentialProvider.getToken(subscription.getTenant()); String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); try { @@ -57,10 +56,10 @@ public List fetchSecurityAlertsDetails(SubscriptionVH subscrip securityAlertsList.add(securityAlertsVH); } } catch (Exception e) { - e.printStackTrace(); + log.error("Error collecting Security Alerts",e); } - System.out.println(securityAlertsList.size()); + log.info("Target Type : {} Total: {} ","Security Alerts",securityAlertsList.size()); return securityAlertsList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SitesInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SitesInventoryCollector.java index f333c023a..f4816340a 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SitesInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SitesInventoryCollector.java @@ -7,6 +7,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.gson.Gson; @@ -14,26 +15,24 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.SitesVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; import com.tmobile.pacman.commons.utils.CommonUtils; @Component public class SitesInventoryCollector { + + @Autowired + AzureCredentialProvider azureCredentialProvider; + private static Logger log = LoggerFactory.getLogger(SitesInventoryCollector.class); private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.Network/vpnSites?api-version=2019-06-01"; public List fetchSitesDetails(SubscriptionVH subscription) throws Exception { List sitesList = new ArrayList(); - String accessToken; - try { - accessToken = AzureCredentialManager.getAuthToken(); - - } catch (Exception e1) { - return sitesList; - } + String accessToken = azureCredentialProvider.getToken(subscription.getTenant()); String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); try { @@ -68,10 +67,10 @@ public List fetchSitesDetails(SubscriptionVH subscription) throws Excep } } } catch (Exception e) { - e.printStackTrace(); + log.error("Error Collecting sites",e); } - System.out.println(sitesList.size()); + log.info("Target Type : {} Total: {} ","Site",sitesList.size()); return sitesList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SnapshotInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SnapshotInventoryCollector.java index 620b92d37..d1bc20860 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SnapshotInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SnapshotInventoryCollector.java @@ -4,24 +4,31 @@ import java.util.List; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.microsoft.azure.PagedList; import com.microsoft.azure.management.Azure; import com.microsoft.azure.management.compute.Snapshot; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.SnapshotVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; @Component public class SnapshotInventoryCollector { - + + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger log = LoggerFactory.getLogger(SnapshotInventoryCollector.class); + public List fetchSnapshotDetails(SubscriptionVH subscription, Map> tagMap) { List snapshotList = new ArrayList(); - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + Azure azure = azureCredentialProvider.getClient(subscription.getTenant(),subscription.getSubscriptionId()); PagedList snapshots = azure.snapshots().list(); - System.out.println(snapshots.size()); for (Snapshot snapshot : snapshots) { SnapshotVH snapshotVH = new SnapshotVH(); snapshotVH.setId(snapshot.id()); @@ -37,6 +44,7 @@ public List fetchSnapshotDetails(SubscriptionVH subscription, Map fetchStorageAccountDetails(SubscriptionVH subscription, Map> tagMap) { List storageAccountList = new ArrayList(); - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + Azure azure = azureCredentialProvider.getClient(subscription.getTenant(),subscription.getSubscriptionId()); PagedList storageAccounts = azure.storageAccounts().list(); for (StorageAccount storageAccount : storageAccounts) { StorageAccountVH storageAccountVH = new StorageAccountVH(); @@ -48,7 +56,7 @@ public List fetchStorageAccountDetails(SubscriptionVH subscrip endPointDetails(storageAccount.endPoints(), storageAccountVH); storageAccountList.add(storageAccountVH); } - + log.info("Target Type : {} Total: {} ","Storage Account",storageAccountList.size()); return storageAccountList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SubnetInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SubnetInventoryCollector.java index 0966f9b9c..32c831c05 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SubnetInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SubnetInventoryCollector.java @@ -8,6 +8,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.gson.Gson; @@ -18,6 +19,7 @@ import com.microsoft.azure.PagedList; import com.microsoft.azure.management.Azure; import com.microsoft.azure.management.network.Network; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.SubnetVH; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; @@ -25,20 +27,18 @@ @Component public class SubnetInventoryCollector { - + + @Autowired + AzureCredentialProvider azureCredentialProvider; + private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets?api-version=2019-07-01"; private static Logger log = LoggerFactory.getLogger(SubnetInventoryCollector.class); public List fetchSubnetDetails(SubscriptionVH subscription) { - List subnetList = new ArrayList(); - String accessToken; - try { - accessToken = AzureCredentialManager.getAuthToken(); - } catch (Exception e1) { - return subnetList; - } - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + List subnetList = new ArrayList<>(); + String accessToken = azureCredentialProvider.getToken(subscription.getTenant()); + Azure azure = azureCredentialProvider.authenticate(subscription.getTenant(),subscription.getSubscriptionId()); PagedList networks = azure.networks().list(); for (Network network : networks) { String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId()), @@ -77,7 +77,7 @@ public List fetchSubnetDetails(SubscriptionVH subscription) { } } - System.out.println(subnetList.size()); + log.info("Target Type : {} Total: {} ","Subnet",subnetList.size()); return subnetList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/VMInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/VMInventoryCollector.java index ea8ab26fb..dc8d71479 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/VMInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/VMInventoryCollector.java @@ -9,6 +9,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.microsoft.azure.PagedList; @@ -19,21 +20,25 @@ import com.microsoft.azure.management.network.NetworkInterface; import com.microsoft.azure.management.network.NicIPConfiguration; import com.microsoft.azure.management.network.Subnet; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; import com.tmobile.pacbot.azure.inventory.vo.VMDiskVH; import com.tmobile.pacbot.azure.inventory.vo.VirtualMachineVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; @Component public class VMInventoryCollector { + @Autowired + AzureCredentialProvider azureCredentialProvider; + private static Logger log = LoggerFactory.getLogger(VMInventoryCollector.class); public List fetchVMDetails(SubscriptionVH subscription, Map> tagMap) { - List vmList = new ArrayList(); + List vmList = new ArrayList<>(); - Azure azure = AzureCredentialManager.authenticate(subscription.getSubscriptionId()); + Azure azure = azureCredentialProvider.getClient(subscription.getTenant(),subscription.getSubscriptionId()); + List networkInterfaces = azure.networkInterfaces().list(); PagedList vms = azure.virtualMachines().list(); @@ -100,7 +105,8 @@ public List fetchVMDetails(SubscriptionVH subscription, Map fetchVaultDetails(SubscriptionVH subscription) throws Exception { List vaultList = new ArrayList(); - String accessToken; - try { - accessToken = AzureCredentialManager.getAuthToken(); - - } catch (Exception e1) { - return vaultList; - } + String accessToken = azureCredentialProvider.getToken(subscription.getTenant()); String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); try { @@ -76,10 +75,10 @@ public List fetchVaultDetails(SubscriptionVH subscription) throws Excep } } } catch (Exception e) { - e.printStackTrace(); + log.error("Error Colectting vaults ",e); } - System.out.println(vaultList.size()); + log.info("Target Type : {} Total: {} ","Vault",vaultList.size()); return vaultList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/WorkflowInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/WorkflowInventoryCollector.java index 2f23261de..6a19d7b37 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/WorkflowInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/WorkflowInventoryCollector.java @@ -7,6 +7,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.gson.Gson; @@ -14,27 +15,24 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; import com.tmobile.pacbot.azure.inventory.vo.WorkflowVH; -import com.tmobile.pacman.commons.azure.clients.AzureCredentialManager; import com.tmobile.pacman.commons.utils.CommonUtils; @Component public class WorkflowInventoryCollector { + @Autowired + AzureCredentialProvider azureCredentialProvider; + private static Logger log = LoggerFactory.getLogger(WorkflowInventoryCollector.class); private String apiUrlTemplate = "https://management.azure.com/subscriptions/%s/providers/Microsoft.Logic/workflows?api-version=2016-06-01"; public List fetchWorkflowDetails(SubscriptionVH subscription) throws Exception { List workflowList = new ArrayList(); - String accessToken; - try { - accessToken = AzureCredentialManager.getAuthToken(); - - } catch (Exception e1) { - return workflowList; - } + String accessToken = azureCredentialProvider.getToken(subscription.getTenant()); String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); try { @@ -71,7 +69,7 @@ public List fetchWorkflowDetails(SubscriptionVH subscription) throws e.printStackTrace(); } - System.out.println(workflowList.size()); + log.info("Target Type : {} Total: {} ","workflow",workflowList.size()); return workflowList; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/AssetFileGenerator.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/AssetFileGenerator.java index 8edeabe8f..146dcca5c 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/AssetFileGenerator.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/AssetFileGenerator.java @@ -15,6 +15,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import com.microsoft.azure.management.Azure; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; import com.tmobile.pacbot.azure.inventory.collector.BatchAccountInventoryCollector; import com.tmobile.pacbot.azure.inventory.collector.BlobContainerInventoryCollector; import com.tmobile.pacbot.azure.inventory.collector.CosmosDBInventoryCollector; @@ -36,7 +38,6 @@ import com.tmobile.pacbot.azure.inventory.collector.RouteTableInventoryCollector; import com.tmobile.pacbot.azure.inventory.collector.SCRecommendationsCollector; import com.tmobile.pacbot.azure.inventory.collector.SQLDatabaseInventoryCollector; -import com.tmobile.pacbot.azure.inventory.collector.SQLServerInventoryCollector; import com.tmobile.pacbot.azure.inventory.collector.SearchServiceInventoryCollector; import com.tmobile.pacbot.azure.inventory.collector.SecurityAlertsInventoryCollector; import com.tmobile.pacbot.azure.inventory.collector.SitesInventoryCollector; @@ -53,8 +54,10 @@ @Component public class AssetFileGenerator { + @Autowired + AzureCredentialProvider azureCredentialProvider; /** The target types. */ - @Value("${targetTypes:virtualmachine}") + @Value("${targetTypes:}") private String targetTypes; /** The log. */ @@ -87,9 +90,6 @@ public class AssetFileGenerator { @Autowired SCRecommendationsCollector scRecommendationsCollector; - @Autowired - SQLServerInventoryCollector sqlServerInventoryCollector; - @Autowired BlobContainerInventoryCollector blobContainerInventoryCollector; @@ -164,6 +164,19 @@ public void generateFiles(List subscriptions, String filePath) { for (SubscriptionVH subscription : subscriptions) { log.info("Started Discovery for sub {}", subscription); + + try { + String accessToken = azureCredentialProvider.getAuthToken(subscription.getTenant()); + Azure azure = azureCredentialProvider.authenticate(subscription.getTenant(),subscription.getSubscriptionId()); + azureCredentialProvider.putClient(subscription.getTenant(),subscription.getSubscriptionId(), azure); + azureCredentialProvider.putToken(subscription.getTenant(), accessToken); + + } catch (Exception e) { + log.error("Error authenticating for {}",subscription,e); + continue; + } + + List resourceGroupList = new ArrayList(); try { resourceGroupList = resourceGroupInventoryCollector.fetchResourceGroupDetails(subscription); @@ -284,19 +297,6 @@ public void generateFiles(List subscriptions, String filePath) { } }); - executor.execute(() -> { - if (!(isTypeInScope("sqlserver"))) { - return; - } - - try { - FileManager.generateSQLServerFiles( - sqlServerInventoryCollector.fetchSQLServerDetails(subscription, tagMap)); - } catch (Exception e) { - e.printStackTrace(); - } - }); - executor.execute(() -> { if (!(isTypeInScope("blobcontainer"))) { return; diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/FileManager.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/FileManager.java index 48c410da6..4edc3c67f 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/FileManager.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/FileManager.java @@ -84,7 +84,6 @@ public static void initialise(String folderName) throws IOException { FileGenerator.writeToFile("azure-vnet.data", "[", false); FileGenerator.writeToFile("azure-loadbalancer.data", "[", false); FileGenerator.writeToFile("azure-securitycenter.data", "[", false); - FileGenerator.writeToFile("azure-sqlserver.data", "[", false); FileGenerator.writeToFile("azure-blobcontainer.data", "[", false); FileGenerator.writeToFile("azure-resourcegroup.data", "[", false); FileGenerator.writeToFile("azure-cosmosdb.data", "[", false); @@ -119,7 +118,6 @@ public static void finalise() throws IOException { FileGenerator.writeToFile("azure-vnet.data", "]", true); FileGenerator.writeToFile("azure-securitycenter.data", "]", true); FileGenerator.writeToFile("azure-loadbalancer.data", "]", true); - FileGenerator.writeToFile("azure-sqlserver.data", "]", true); FileGenerator.writeToFile("azure-blobcontainer.data", "]", true); FileGenerator.writeToFile("azure-resourcegroup.data", "]", true); FileGenerator.writeToFile("azure-cosmosdb.data", "]", true); @@ -199,10 +197,6 @@ public static void generateSecurityCenterFiles(List recommenda } - public static void generateSQLServerFiles(List sqlServerList) throws IOException { - FileGenerator.generateJson(sqlServerList, "azure-sqlserver.data"); - } - public static void generateBlobContainerFiles(List blobDetailsList) throws IOException { FileGenerator.generateJson(blobDetailsList, "azure-blobcontainer.data"); } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SubscriptionVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SubscriptionVH.java index 34560eec8..09789a001 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SubscriptionVH.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SubscriptionVH.java @@ -4,10 +4,18 @@ public class SubscriptionVH { @Override public String toString() { - return "[subscriptionId=" + subscriptionId + ", subscriptionName=" + subscriptionName + "]"; + return "{ subscriptionName=" + subscriptionName + ", subscriptionId=" + subscriptionId +", tenant="+tenant +"}"; } private String subscriptionId; private String subscriptionName; + private String tenant; + + public String getTenant() { + return tenant; + } + public void setTenant(String tenant) { + this.tenant = tenant; + } public String getSubscriptionId() { return subscriptionId; } From af760eb606231b97a49d91d5aa1edb6d5be4af7c Mon Sep 17 00:00:00 2001 From: johnrexj Date: Wed, 6 Nov 2019 12:04:05 +0530 Subject: [PATCH 041/107] Additional azure inventory changes --- .../auth/AzureCredentialProvider.java | 90 +++++++++++ .../PolicyStatesInventoryCollector.java | 8 +- .../SQLServerInventoryCollector.java | 130 +++++++++++++++ .../inventory/file/AssetFileGenerator.java | 17 ++ .../azure/inventory/file/FileManager.java | 6 + .../azure/inventory/vo/BatchAccountVH.java | 9 +- .../azure/inventory/vo/BlobContainerVH.java | 7 +- .../azure/inventory/vo/LoadBalancerVH.java | 1 - .../azure/inventory/vo/NamespaceVH.java | 10 +- .../azure/inventory/vo/PolicyStatesVH.java | 9 ++ .../azure/inventory/vo/SearchServiceVH.java | 8 - .../pacbot/azure/inventory/vo/SitesVH.java | 10 +- .../pacbot/azure/inventory/vo/SubnetVH.java | 10 -- .../pacbot/azure/inventory/vo/VaultVH.java | 9 -- .../pacbot/azure/inventory/vo/WorkflowVH.java | 9 +- jobs/pacman-data-shipper/pom.xml | 2 +- .../datashipper/config/ConfigManager.java | 2 +- .../entity/EntityAssociationManager.java | 7 +- .../datashipper/entity/EntityManager.java | 26 +-- .../datashipper/error/AwsErrorManager.java | 45 ++++++ .../datashipper/error/AzureErrorManager.java | 18 +++ .../datashipper/error/ErrorManager.java | 149 ++++++++++++++++++ .../cso/pacman/datashipper/es/ESManager.java | 8 +- .../cso/pacman/datashipper/util/Util.java | 10 +- .../pacman/datashipper/es/ESManagerTest.java | 8 +- .../cso/pacman/datashipper/util/UtilTest.java | 2 +- 26 files changed, 512 insertions(+), 98 deletions(-) create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/auth/AzureCredentialProvider.java create mode 100644 jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLServerInventoryCollector.java create mode 100644 jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/error/AwsErrorManager.java create mode 100644 jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/error/AzureErrorManager.java create mode 100644 jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/error/ErrorManager.java diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/auth/AzureCredentialProvider.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/auth/AzureCredentialProvider.java new file mode 100644 index 000000000..8798bb3b1 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/auth/AzureCredentialProvider.java @@ -0,0 +1,90 @@ +package com.tmobile.pacbot.azure.inventory.auth; + +import java.util.HashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import com.microsoft.azure.AzureEnvironment; +import com.microsoft.azure.credentials.ApplicationTokenCredentials; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.Azure.Authenticated; +import com.tmobile.pacman.commons.utils.CommonUtils; + +@Component +public class AzureCredentialProvider { + + /** The Constant logger. */ + static final Logger logger = LoggerFactory.getLogger(AzureCredentialProvider.class); + Map azureClients; ; + Map apiTokens; + + AzureCredentialProvider() { + azureClients = new HashMap<>(); + apiTokens = new HashMap<>(); + } + + public Azure getClient(String tenant,String subscription){ + return azureClients.get(tenant+subscription); + } + + public void putClient(String tenant,String subscription,Azure azure){ + azureClients.put(tenant+subscription,azure); + } + + public String getToken(String tenant) { + return apiTokens.get(tenant); + } + + public void putToken(String tenant, String token) { + apiTokens.put(tenant,token); + } + /* Below methods to be moved to Commons */ + + public Azure authenticate(String tenant,String subscription) { + return Azure.authenticate(getCredentials(tenant)).withSubscription(subscription); + + } + + public Authenticated authenticate(String tenant) { + return Azure.authenticate(getCredentials(tenant)); + } + + + private ApplicationTokenCredentials getCredentials(String tenant){ + String clientId = System.getProperty("azure.clientId."+tenant); + String secret = System.getProperty("azure.secret."+tenant); + return new ApplicationTokenCredentials(clientId, + tenant, secret, AzureEnvironment.AZURE); + } + + public String getAuthToken(String tenant) throws Exception { + String url = "https://login.microsoftonline.com/%s/oauth2/token"; + + String clientId = System.getProperty("azure.clientId."+tenant); + String secret = System.getProperty("azure.secret."+tenant); + + + Map params = new HashMap<>(); + params.put("client_id", clientId); + params.put("client_secret", secret); + params.put("resource", "https://management.azure.com"); + params.put("grant_type", "client_credentials"); + url = String.format(url, tenant); + + try { + String jsonResponse = CommonUtils.doHttpPost(url, params); + Map respMap = new Gson().fromJson(jsonResponse, new TypeToken>() {}.getType() ); + return respMap.get("access_token"); + } catch (Exception e) { + logger.error("Error getting mangement API token from Azure",e); + throw e; + } + } + + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PolicyStatesInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PolicyStatesInventoryCollector.java index db38e614b..659ecd0c0 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PolicyStatesInventoryCollector.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/PolicyStatesInventoryCollector.java @@ -32,7 +32,7 @@ public class PolicyStatesInventoryCollector { public List fetchPolicyStatesDetails(SubscriptionVH subscription, List policyDefinitionList) throws Exception { - List policyStatesList = new ArrayList(); + List policyStatesList = new ArrayList<>(); String accessToken = azureCredentialProvider.getToken(subscription.getTenant()); String url = String.format(apiUrlTemplate, URLEncoder.encode(subscription.getSubscriptionId())); @@ -53,8 +53,10 @@ public List fetchPolicyStatesDetails(SubscriptionVH subscription policyStatesVH.setPolicyType(PolicyDefinitionVH.getPolicyType()); policyStatesVH.setPolicyRule(PolicyDefinitionVH.getPolicyRule()); policyStatesVH.setTimestamp(policyStatesObject.get("timestamp").getAsString()); - policyStatesVH.setId(policyStatesObject.get("resourceId").getAsString()); - policyStatesVH.setResourceId(policyStatesObject.get("resourceId").getAsString()); + policyStatesVH.setId(policyStatesObject.get("policyDefinitionName").getAsString()+"_"+policyStatesObject.get("resourceId").getAsString().toLowerCase()); + policyStatesVH.setResourceId(Util.removeFirstSlash(policyStatesObject.get("resourceId").getAsString())); + policyStatesVH.setResourceIdLower(Util.removeFirstSlash(policyStatesObject.get("resourceId").getAsString().toLowerCase())); + policyStatesVH.setPolicyAssignmentId(policyStatesObject.get("policyAssignmentId").getAsString()); policyStatesVH.setPolicyDefinitionId(policyStatesObject.get("policyDefinitionId").getAsString()); policyStatesVH.setEffectiveParameters(policyStatesObject.get("effectiveParameters").getAsString()); diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLServerInventoryCollector.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLServerInventoryCollector.java new file mode 100644 index 000000000..d27ee6723 --- /dev/null +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/collector/SQLServerInventoryCollector.java @@ -0,0 +1,130 @@ +package com.tmobile.pacbot.azure.inventory.collector; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.microsoft.azure.PagedList; +import com.microsoft.azure.management.Azure; +import com.microsoft.azure.management.sql.SqlElasticPool; +import com.microsoft.azure.management.sql.SqlFailoverGroup; +import com.microsoft.azure.management.sql.SqlFirewallRule; +import com.microsoft.azure.management.sql.SqlServer; +import com.microsoft.azure.management.sql.SqlVirtualNetworkRule; +import com.tmobile.pacbot.azure.inventory.auth.AzureCredentialProvider; +import com.tmobile.pacbot.azure.inventory.vo.ElasticPoolVH; +import com.tmobile.pacbot.azure.inventory.vo.FailoverGroupVH; +import com.tmobile.pacbot.azure.inventory.vo.SQLServerVH; +import com.tmobile.pacbot.azure.inventory.vo.SubscriptionVH; + +@Component +public class SQLServerInventoryCollector { + + @Autowired + AzureCredentialProvider azureCredentialProvider; + + private static Logger log = LoggerFactory.getLogger(SQLServerInventoryCollector.class); + + public List fetchSQLServerDetails(SubscriptionVH subscription, + Map> tagMap) { + + List sqlServerList = new ArrayList<>(); + Azure azure = azureCredentialProvider.getClient(subscription.getTenant(),subscription.getSubscriptionId()); + PagedList sqlServers = azure.sqlServers().list(); + for (SqlServer sqlServer : sqlServers) { + SQLServerVH sqlServerVH = new SQLServerVH(); + sqlServerVH.setSubscription(subscription.getSubscriptionId()); + sqlServerVH.setSubscriptionName(subscription.getSubscriptionName()); + sqlServerVH.setId(sqlServer.id()); + sqlServerVH.setKind(sqlServer.kind()); + sqlServerVH.setName(sqlServer.name()); + sqlServerVH.setRegionName(sqlServer.regionName()); + sqlServerVH.setState(sqlServer.state()); + sqlServerVH.setSystemAssignedManagedServiceIdentityPrincipalId( + sqlServer.systemAssignedManagedServiceIdentityPrincipalId()); + sqlServerVH.setSystemAssignedManagedServiceIdentityTenantId( + sqlServer.systemAssignedManagedServiceIdentityTenantId()); + sqlServerVH.setTags(Util.tagsList(tagMap, sqlServer.resourceGroupName(), sqlServer.tags())); + sqlServerVH.setVersion(sqlServer.version()); + sqlServerVH.setAdministratorLogin(sqlServer.administratorLogin()); + firewallRule(sqlServer, sqlServerVH); + getElasticPoolList(sqlServer.elasticPools().list(), sqlServerVH); + getFailoverGroupList(sqlServer.failoverGroups().list(), sqlServerVH); + sqlServerList.add(sqlServerVH); + } + log.info("Target Type : {} Total: {} ","SqlServer",sqlServerList.size()); + return sqlServerList; + + } + + private void getElasticPoolList(List sqlElasticPoolList, SQLServerVH sqlServerVH) { + List elasticPoolList = new ArrayList<>(); + for (SqlElasticPool sqlElasticPool : sqlElasticPoolList) { + ElasticPoolVH elasticPoolVH = new ElasticPoolVH(); + elasticPoolVH.setName(sqlElasticPool.name()); + elasticPoolVH.setSize(sqlElasticPool.listDatabases().size()); + elasticPoolVH.setStorageCapacity(sqlElasticPool.storageCapacityInMB()); + elasticPoolVH.setId(sqlElasticPool.id()); + elasticPoolVH.setStorageMB(sqlElasticPool.storageMB()); + elasticPoolVH.setDtu(sqlElasticPool.dtu()); + elasticPoolVH.setEdition(sqlElasticPool.edition().toString()); + elasticPoolList.add(elasticPoolVH); + + } + sqlServerVH.setElasticPoolList(elasticPoolList); + + } + + private void firewallRule(SqlServer sqlServer, SQLServerVH sqlServerVH) { + List> firewallRuleList = new ArrayList<>(); + Map firewallMap; + for (SqlFirewallRule sqlFirewallRule : sqlServer.firewallRules().list()) { + firewallMap = new HashMap<>(); + firewallMap.put("name", sqlFirewallRule.name()); + firewallMap.put("startIPAddress", sqlFirewallRule.startIPAddress()); + firewallMap.put("endIPAddress", sqlFirewallRule.endIPAddress()); + firewallRuleList.add(firewallMap); + + } + for (SqlVirtualNetworkRule sqlVirtualNetworkRule : sqlServer.virtualNetworkRules().list()) { + firewallMap = new HashMap<>(); + + firewallMap.put("virtualNetworkRuleName", + sqlVirtualNetworkRule.name() != null ? sqlVirtualNetworkRule.name() : ""); + firewallMap.put("virtualNetworkSubnetId", + sqlVirtualNetworkRule.subnetId() != null ? sqlVirtualNetworkRule.subnetId() : ""); + firewallMap.put("virtualNetworkResourceGroupName", + sqlVirtualNetworkRule.resourceGroupName() != null ? sqlVirtualNetworkRule.resourceGroupName() : ""); + firewallMap.put("virtualNetworkState", + sqlVirtualNetworkRule.state() != null ? sqlVirtualNetworkRule.state() : ""); + + firewallRuleList.add(firewallMap); + } + sqlServerVH.setFirewallRuleDetails(firewallRuleList); + } + + private void getFailoverGroupList(List sqlFailoverGroupList, SQLServerVH sqlServerVH) { + List failoverGroupList = new ArrayList<>(); + for (SqlFailoverGroup sqlFailoverGroup : sqlFailoverGroupList) { + FailoverGroupVH failoverGroupVH = new FailoverGroupVH(); + failoverGroupVH.setSize(sqlFailoverGroup.databases().size()); + failoverGroupVH.setId(sqlFailoverGroup.id()); + failoverGroupVH.setName(sqlFailoverGroup.name()); + failoverGroupVH.setReplicationState(sqlFailoverGroup.replicationState()); + failoverGroupVH.setReadOnlyEndpointPolicy(sqlFailoverGroup.readOnlyEndpointPolicy().toString()); + failoverGroupVH.setReadWriteEndpointPolicy(sqlFailoverGroup.readWriteEndpointPolicy().toString()); + failoverGroupVH.setGracePeriod(sqlFailoverGroup.readWriteEndpointDataLossGracePeriodMinutes()); + failoverGroupList.add(failoverGroupVH); + + } + sqlServerVH.setFailoverGroupList(failoverGroupList); + + } + +} diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/AssetFileGenerator.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/AssetFileGenerator.java index 146dcca5c..fd6706e50 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/AssetFileGenerator.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/AssetFileGenerator.java @@ -38,6 +38,7 @@ import com.tmobile.pacbot.azure.inventory.collector.RouteTableInventoryCollector; import com.tmobile.pacbot.azure.inventory.collector.SCRecommendationsCollector; import com.tmobile.pacbot.azure.inventory.collector.SQLDatabaseInventoryCollector; +import com.tmobile.pacbot.azure.inventory.collector.SQLServerInventoryCollector; import com.tmobile.pacbot.azure.inventory.collector.SearchServiceInventoryCollector; import com.tmobile.pacbot.azure.inventory.collector.SecurityAlertsInventoryCollector; import com.tmobile.pacbot.azure.inventory.collector.SitesInventoryCollector; @@ -90,6 +91,9 @@ public class AssetFileGenerator { @Autowired SCRecommendationsCollector scRecommendationsCollector; + @Autowired + SQLServerInventoryCollector sqlServerInventoryCollector; + @Autowired BlobContainerInventoryCollector blobContainerInventoryCollector; @@ -297,6 +301,19 @@ public void generateFiles(List subscriptions, String filePath) { } }); + executor.execute(() -> { + if (!(isTypeInScope("sqlserver"))) { + return; + } + + try { + FileManager.generateSQLServerFiles( + sqlServerInventoryCollector.fetchSQLServerDetails(subscription, tagMap)); + } catch (Exception e) { + e.printStackTrace(); + } + }); + executor.execute(() -> { if (!(isTypeInScope("blobcontainer"))) { return; diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/FileManager.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/FileManager.java index 4edc3c67f..48c410da6 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/FileManager.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/file/FileManager.java @@ -84,6 +84,7 @@ public static void initialise(String folderName) throws IOException { FileGenerator.writeToFile("azure-vnet.data", "[", false); FileGenerator.writeToFile("azure-loadbalancer.data", "[", false); FileGenerator.writeToFile("azure-securitycenter.data", "[", false); + FileGenerator.writeToFile("azure-sqlserver.data", "[", false); FileGenerator.writeToFile("azure-blobcontainer.data", "[", false); FileGenerator.writeToFile("azure-resourcegroup.data", "[", false); FileGenerator.writeToFile("azure-cosmosdb.data", "[", false); @@ -118,6 +119,7 @@ public static void finalise() throws IOException { FileGenerator.writeToFile("azure-vnet.data", "]", true); FileGenerator.writeToFile("azure-securitycenter.data", "]", true); FileGenerator.writeToFile("azure-loadbalancer.data", "]", true); + FileGenerator.writeToFile("azure-sqlserver.data", "]", true); FileGenerator.writeToFile("azure-blobcontainer.data", "]", true); FileGenerator.writeToFile("azure-resourcegroup.data", "]", true); FileGenerator.writeToFile("azure-cosmosdb.data", "]", true); @@ -197,6 +199,10 @@ public static void generateSecurityCenterFiles(List recommenda } + public static void generateSQLServerFiles(List sqlServerList) throws IOException { + FileGenerator.generateJson(sqlServerList, "azure-sqlserver.data"); + } + public static void generateBlobContainerFiles(List blobDetailsList) throws IOException { FileGenerator.generateJson(blobDetailsList, "azure-blobcontainer.data"); } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/BatchAccountVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/BatchAccountVH.java index e1351c68c..7a38b17b0 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/BatchAccountVH.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/BatchAccountVH.java @@ -4,7 +4,6 @@ public class BatchAccountVH extends AzureVH { - private String id; private String name; private String type; private String location; @@ -20,9 +19,7 @@ public class BatchAccountVH extends AzureVH { private boolean dedicatedCoreQuotaPerVMFamilyEnforced; private Map autoStorage; - public String getId() { - return id; - } + public String getName() { return name; @@ -40,9 +37,7 @@ public Map getTags() { return tags; } - public void setId(String id) { - this.id = id; - } + public void setName(String name) { this.name = name; diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/BlobContainerVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/BlobContainerVH.java index 6b910e464..a4be44fa5 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/BlobContainerVH.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/BlobContainerVH.java @@ -1,6 +1,5 @@ package com.tmobile.pacbot.azure.inventory.vo; -import java.util.HashMap; import java.util.Map; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -11,7 +10,7 @@ public class BlobContainerVH extends AzureVH { private String name; private String type; private String tag; - private HashMap propertiesMap; + private Map propertiesMap; private Map tags; public String getName() { @@ -38,11 +37,11 @@ public void setTag(String tag) { this.tag = tag; } - public HashMap getPropertiesMap() { + public Map getPropertiesMap() { return propertiesMap; } - public void setPropertiesMap(HashMap propertiesMap) { + public void setPropertiesMap(Map propertiesMap) { this.propertiesMap = propertiesMap; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/LoadBalancerVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/LoadBalancerVH.java index b2afea344..0bff5add1 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/LoadBalancerVH.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/LoadBalancerVH.java @@ -8,7 +8,6 @@ import com.microsoft.azure.management.network.LoadBalancerPrivateFrontend; import com.microsoft.azure.management.network.LoadBalancerPublicFrontend; import com.microsoft.azure.management.network.LoadBalancingRule; -import com.microsoft.azure.management.resources.fluentcore.arm.Region; @JsonSerialize public class LoadBalancerVH extends AzureVH { diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NamespaceVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NamespaceVH.java index 29da02a72..39574bc19 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NamespaceVH.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/NamespaceVH.java @@ -4,7 +4,6 @@ public class NamespaceVH extends AzureVH { - private String id; private String name; private String type; private String location; @@ -12,10 +11,7 @@ public class NamespaceVH extends AzureVH { private Map properties; private Map sku; - public String getId() { - return id; - } - + public String getName() { return name; } @@ -36,10 +32,6 @@ public Map getProperties() { return properties; } - public void setId(String id) { - this.id = id; - } - public void setName(String name) { this.name = name; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PolicyStatesVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PolicyStatesVH.java index 3b00e037a..fb156e93a 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PolicyStatesVH.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/PolicyStatesVH.java @@ -3,6 +3,7 @@ public class PolicyStatesVH extends AzureVH { private String timestamp; private String resourceId; + private String resourceIdLower; private String policyAssignmentId; private String policyDefinitionId; private String effectiveParameters; @@ -263,4 +264,12 @@ public void setPolicyRule(String policyRule) { this.policyRule = policyRule; } + public String getResourceIdLower() { + return resourceIdLower; + } + + public void setResourceIdLower(String resourceIdLower) { + this.resourceIdLower = resourceIdLower; + } + } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SearchServiceVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SearchServiceVH.java index babf02d52..c39ec4fe7 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SearchServiceVH.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SearchServiceVH.java @@ -4,16 +4,12 @@ public class SearchServiceVH extends AzureVH { - private String id; private String name; private String type; private String location; private Map properties; private Map sku; - public String getId() { - return id; - } public String getName() { return name; @@ -35,10 +31,6 @@ public Map getSku() { return sku; } - public void setId(String id) { - this.id = id; - } - public void setName(String name) { this.name = name; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SitesVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SitesVH.java index b5e231c1d..eba7ec436 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SitesVH.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SitesVH.java @@ -3,7 +3,7 @@ import java.util.Map; public class SitesVH extends AzureVH { - private String id; + private String etag; private String location; private String name; @@ -11,10 +11,6 @@ public class SitesVH extends AzureVH { private Map tags; private Map properties; - - public String getId() { - return id; - } public String getEtag() { return etag; } @@ -28,9 +24,7 @@ public String getType() { return type; } - public void setId(String id) { - this.id = id; - } + public void setEtag(String etag) { this.etag = etag; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SubnetVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SubnetVH.java index 4a3ea024b..5f46cd4f3 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SubnetVH.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/SubnetVH.java @@ -4,8 +4,6 @@ import java.util.Map; public class SubnetVH extends AzureVH { - - private String id; private String etag; private String name; private String type; @@ -55,10 +53,6 @@ public void setPrivateEndpointNetworkPolicies(String privateEndpointNetworkPolic this.privateEndpointNetworkPolicies = privateEndpointNetworkPolicies; } - public String getId() { - return id; - } - public String getEtag() { return etag; } @@ -71,10 +65,6 @@ public String getType() { return type; } - public void setId(String id) { - this.id = id; - } - public void setEtag(String etag) { this.etag = etag; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VaultVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VaultVH.java index ea9f3e987..af5d5c016 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VaultVH.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/VaultVH.java @@ -3,7 +3,6 @@ import java.util.Map; public class VaultVH extends AzureVH { - private String id; private String name; private String type; private String location; @@ -72,10 +71,6 @@ public void setVaultUri(String vaultUri) { this.vaultUri = vaultUri; } - public String getId() { - return id; - } - public String getName() { return name; } @@ -92,10 +87,6 @@ public Map getTags() { return tags; } - public void setId(String id) { - this.id = id; - } - public void setName(String name) { this.name = name; } diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/WorkflowVH.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/WorkflowVH.java index 0e3dd1b63..7e8e02ec1 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/WorkflowVH.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/vo/WorkflowVH.java @@ -4,15 +4,12 @@ public class WorkflowVH extends AzureVH{ - private String id; private String name; private String type; private String location; private Map tags; private Map properties; - public String getId() { - return id; - } + public String getName() { return name; } @@ -28,9 +25,7 @@ public Map getTags() { public Map getProperties() { return properties; } - public void setId(String id) { - this.id = id; - } + public void setName(String name) { this.name = name; } diff --git a/jobs/pacman-data-shipper/pom.xml b/jobs/pacman-data-shipper/pom.xml index a0d5dd557..26ca264ad 100644 --- a/jobs/pacman-data-shipper/pom.xml +++ b/jobs/pacman-data-shipper/pom.xml @@ -144,7 +144,7 @@ com.fasterxml.jackson.core jackson-databind - 2.9.10 + 2.9.10.1 mysql diff --git a/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/config/ConfigManager.java b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/config/ConfigManager.java index d41340b0b..3c434be9d 100644 --- a/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/config/ConfigManager.java +++ b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/config/ConfigManager.java @@ -50,7 +50,7 @@ private static Map> getTypeConfig(String datasoruce) if (typeInfo == null) { typeInfo = new HashMap<>(); - List> typeList = RDSDBManager.executeQuery(System.getProperty(Constants.CONFIG_QUERY)); + List> typeList = RDSDBManager.executeQuery(System.getProperty(Constants.CONFIG_QUERY)+" and dataSourceName ='"+datasoruce+"'"); try{ for (Map _type : typeList) { String typeName = _type.get("targetName"); diff --git a/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/entity/EntityAssociationManager.java b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/entity/EntityAssociationManager.java index 95124a5bc..1994ce5a7 100644 --- a/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/entity/EntityAssociationManager.java +++ b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/entity/EntityAssociationManager.java @@ -38,6 +38,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.tmobile.cso.pacman.datashipper.config.ConfigManager; import com.tmobile.cso.pacman.datashipper.config.CredentialProvider; +import com.tmobile.cso.pacman.datashipper.error.ErrorManager; import com.tmobile.cso.pacman.datashipper.es.ESManager; import com.tmobile.cso.pacman.datashipper.util.Constants; @@ -95,16 +96,16 @@ public List> uploadAssociationInfo(String dataSource,String if (!childType.equalsIgnoreCase("tags")) { ESManager.createType(indexName, childTypeES, type); LOGGER.info("Fetching data for {}", childTypeES); - List> entities = new ArrayList<>(); + List> entities = new ArrayList<>(); S3Object entitiesData = s3Client.getObject(new GetObjectRequest(bucketName, dataPath+"/"+filePrefix+childType+".data")); try (BufferedReader reader = new BufferedReader(new InputStreamReader(entitiesData.getObjectContent()))) { - entities = objectMapper.readValue(reader.lines().collect(Collectors.joining("\n")),new TypeReference>>() {}); + entities = objectMapper.readValue(reader.lines().collect(Collectors.joining("\n")),new TypeReference>>() {}); } String loaddate = new SimpleDateFormat("yyyy-MM-dd HH:mm:00Z").format(new java.util.Date()); entities.parallelStream().forEach(obj -> obj.put("_loaddate", loaddate)); LOGGER.info("Collected : {}", entities.size()); if (!entities.isEmpty()) { - AWSErrorManager.getInstance().handleError(dataSource, indexName, childTypeES, loaddate, errorList,false); + ErrorManager.getInstance(dataSource).handleError(indexName, childTypeES, loaddate, errorList,false); ESManager.uploadData(indexName, childTypeES, entities, key.split(",")); ESManager.deleteOldDocuments(indexName, childTypeES, "_loaddate.keyword", loaddate); diff --git a/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/entity/EntityManager.java b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/entity/EntityManager.java index 9bb5de250..62f393cd0 100644 --- a/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/entity/EntityManager.java +++ b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/entity/EntityManager.java @@ -31,6 +31,7 @@ import com.tmobile.cso.pacman.datashipper.config.ConfigManager; import com.tmobile.cso.pacman.datashipper.dao.RDSDBManager; +import com.tmobile.cso.pacman.datashipper.error.ErrorManager; import com.tmobile.cso.pacman.datashipper.es.ESManager; import com.tmobile.cso.pacman.datashipper.util.Constants; import com.tmobile.cso.pacman.datashipper.util.Util; @@ -74,7 +75,6 @@ public class EntityManager implements Constants { * @return the list */ public List> uploadEntityData(String datasource) { - List> errorList = new ArrayList<>(); Set types = ConfigManager.getTypes(datasource); Iterator itr = types.iterator(); @@ -95,7 +95,7 @@ public List> uploadEntityData(String datasource) { Map> currentInfo = ESManager.getExistingInfo(indexName, type, filters); LOGGER.info("Existing no of docs : {}" , currentInfo.size()); - List> entities = fetchEntitiyInfoFromS3(datasource,type,errorList); + List> entities = fetchEntitiyInfoFromS3(datasource,type,errorList); List> tags = fetchTagsForEntitiesFromS3(datasource, type); LOGGER.info("Fetched from S3"); @@ -113,7 +113,7 @@ public List> uploadEntityData(String datasource) { String[] keysArray = keys.split(","); prepareDocs(currentInfo, entities, tags, overridableInfo, overridesMap, idColumn, keysArray, type); - Map errUpdateInfo = AWSErrorManager.getInstance().handleError(datasource,indexName,type,loaddate,errorList,true); + Map errUpdateInfo = ErrorManager.getInstance(datasource).handleError(indexName,type,loaddate,errorList,true); Map uploadInfo = ESManager.uploadData(indexName, type, entities, loaddate); stats.putAll(uploadInfo); stats.put("errorUpdates", errUpdateInfo); @@ -149,8 +149,8 @@ private List> fetchTagsForEntitiesFromS3(String datasource, return tags; } - private List> fetchEntitiyInfoFromS3(String datasource,String type,List> errorList) { - List> entities = new ArrayList<>() ; + private List> fetchEntitiyInfoFromS3(String datasource,String type,List> errorList) { + List> entities = new ArrayList<>() ; try{ entities = Util.fetchDataFromS3(s3Account,s3Region, s3Role,bucketName, dataPath+"/"+datasource + "-" + type+".data"); } catch (Exception e) { @@ -178,11 +178,11 @@ private List> fetchEntitiyInfoFromS3(String datasource,Strin * @param _keys the keys * @param _type the type */ - private void prepareDocs(Map> currentInfo, List> entities, + private void prepareDocs(Map> currentInfo, List> entities, List> tags, List> overridableInfo, Map>> overridesMap, String idColumn, String[] _keys, String _type) { entities.parallelStream().forEach(entityInfo -> { - String id = entityInfo.get(idColumn); + String id = entityInfo.get(idColumn).toString(); String docId = Util.concatenate(entityInfo, _keys, "_"); entityInfo.put("_resourceid", id); entityInfo.put("_docid", docId); @@ -192,7 +192,7 @@ private void prepareDocs(Map> currentInfo, List _currInfo = currentInfo.get(docId); if (_currInfo != null) { if (_currInfo.get(FIRST_DISCOVERED) == null) { - _currInfo.put(FIRST_DISCOVERED, entityInfo.get(DISCOVERY_DATE)); + _currInfo.put(FIRST_DISCOVERED, entityInfo.get(DISCOVERY_DATE).toString()); } entityInfo.putAll(_currInfo); } else { @@ -224,8 +224,8 @@ private void prepareDocs(Map> currentInfo, List entity) { - entity.put("tags.Application", entity.get("u_business_service").toLowerCase()); + private static void updateOnPremData(Map entity) { + entity.put("tags.Application", entity.get("u_business_service").toString().toLowerCase()); entity.put("tags.Environment", entity.get("used_for")); entity.put("inScope", "true"); } @@ -240,7 +240,7 @@ private static void updateOnPremData(Map entity) { * @param overrideFields * the override fields */ - private static void override(Map entity, List> overrideList, + private static void override(Map entity, List> overrideList, List> overrideFields) { if (overrideList != null && !overrideList.isEmpty()) { @@ -262,7 +262,7 @@ private static void override(Map entity, List entity, List handleError(String index, String type, String loaddate,List> errorList,boolean checkLatest) { + Map>> errorInfo = getErrorInfo(errorList); + String parentType = index.replace(dataSource+"_", ""); + Map errorUpdateInfo = new HashMap<>(); + if(errorInfo.containsKey(parentType) || errorInfo.containsKey("all")) { + List> errorByType = errorInfo.get(parentType); + if(errorByType==null){ + errorByType = errorInfo.get("all"); + } + errorByType.forEach(errorData -> { + String accountId = errorData.get("accountid"); + String region = errorData.get("region"); + long updateCount = ESManager.updateLoadDate(index, type, accountId, region, loaddate,checkLatest); + errorUpdateInfo.put(accountId+":"+region, updateCount); + } + ); + } + return errorUpdateInfo; + } + +} diff --git a/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/error/AzureErrorManager.java b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/error/AzureErrorManager.java new file mode 100644 index 000000000..50ccdc470 --- /dev/null +++ b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/error/AzureErrorManager.java @@ -0,0 +1,18 @@ +package com.tmobile.cso.pacman.datashipper.error; + +import java.util.List; +import java.util.Map; + +public class AzureErrorManager extends ErrorManager { + + protected AzureErrorManager() { + + } + @Override + public Map handleError(String index, String type, String loaddate, + List> errorList, boolean checkLatest) { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/error/ErrorManager.java b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/error/ErrorManager.java new file mode 100644 index 000000000..5a9b82b4a --- /dev/null +++ b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/error/ErrorManager.java @@ -0,0 +1,149 @@ +/******************************************************************************* + * Copyright 2018 T Mobile, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + ******************************************************************************/ +package com.tmobile.cso.pacman.datashipper.error; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.S3Object; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.tmobile.cso.pacman.datashipper.config.CredentialProvider; +import com.tmobile.cso.pacman.datashipper.util.Constants; + +/** + * The Class AWSErrorManager. + */ +public abstract class ErrorManager implements Constants { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(ErrorManager.class); + + /** The s 3 account. */ + private String s3Account = System.getProperty("base.account"); + + /** The s 3 region. */ + private String s3Region = System.getProperty("base.region"); + + /** The s 3 role. */ + private String s3Role = System.getProperty("s3.role"); + + /** The bucket name. */ + private String bucketName = System.getProperty("s3"); + + /** The data path. */ + private String dataPath = System.getProperty("s3.data"); + + /** The error info. */ + private Map>> errorInfo ; + + protected String dataSource; + + /** The error manager. */ + private static ErrorManager errorManager ; + + /** + * Instantiates a new AWS error manager. + */ + + + /** + * Gets the single instance of AWSErrorManager. + * + * @return single instance of AWSErrorManager + */ + public static ErrorManager getInstance(String dataSource){ + if(errorManager==null){ + switch(dataSource) { + case "aws": + errorManager = new AwsErrorManager(); + errorManager.dataSource ="aws"; + break; + case "azure": + errorManager = new AzureErrorManager(); + errorManager.dataSource ="azure"; + break; + default: + } + + } + return errorManager; + } + + /** + * Fetch error info. + * + * @param datasource the datasource + * @param errorList the error list + */ + private void fetchErrorInfo(List> errorList){ + if(errorInfo==null){ + ObjectMapper objectMapper = new ObjectMapper(); + List> inventoryErrors = new ArrayList<>(); + AmazonS3 s3Client = AmazonS3ClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider(new CredentialProvider().getCredentials(s3Account,s3Role))).withRegion(s3Region).build(); + try { + S3Object inventoryErrorData = s3Client.getObject(new GetObjectRequest(bucketName,dataPath+"/"+dataSource+"-loaderror.data")); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inventoryErrorData.getObjectContent()))) { + inventoryErrors = objectMapper.readValue(reader.lines().collect(Collectors.joining("\n")),new TypeReference>>() {}); + } + } catch (IOException e) { + LOGGER.error("Exception in collecting inventory error data",e); + Map errorMap = new HashMap<>(); + errorMap.put(ERROR, "Exception in collecting inventory error data"); + errorMap.put(ERROR_TYPE, WARN); + errorMap.put(EXCEPTION, e.getMessage()); + errorList.add(errorMap); + } + errorInfo = inventoryErrors.parallelStream().collect(Collectors.groupingBy(obj -> obj.get("type"))); + } + } + + /** + * Gets the error info. + * + * @param datasource the datasource + * @param errorList the error list + * @return the error info + */ + public Map>> getErrorInfo(List> errorList){ + if(errorInfo==null){ + fetchErrorInfo(errorList); + } + + return errorInfo; + + } + + public abstract Map handleError(String index, String type, String loaddate,List> errorList,boolean checkLatest) ; + + + + +} diff --git a/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/es/ESManager.java b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/es/ESManager.java index 9cb6155b1..11abf3195 100644 --- a/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/es/ESManager.java +++ b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/es/ESManager.java @@ -98,7 +98,7 @@ private static RestClient getRestClient() { * @param loaddate the loaddate * @return the map */ - public static Map uploadData(String index, String type, List> docs, String loaddate) { + public static Map uploadData(String index, String type, List> docs, String loaddate) { Map status = new LinkedHashMap<>(); List errors = new ArrayList<>(); @@ -112,7 +112,7 @@ public static Map uploadData(String index, String type, List doc : docs) { + for (Map doc : docs) { String id = Util.concatenate(doc, _keys, "_"); StringBuilder _doc = new StringBuilder(createESDoc(doc)); @@ -628,7 +628,7 @@ public static void createType(String index, String type, String parent) { * @param docs the docs * @param parentKey the parent key */ - public static void uploadData(String index, String type, List> docs, String[] parentKey) { + public static void uploadData(String index, String type, List> docs, String[] parentKey) { String actionTemplate = "{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\", \"_parent\" : \"%s\" } }%n"; // added // _parent // node @@ -637,7 +637,7 @@ public static void uploadData(String index, String type, List doc : docs) { + for (Map doc : docs) { StringBuilder _doc = new StringBuilder(new Gson().toJson(doc)); String parent = Util.concatenate(doc, parentKey, "_"); diff --git a/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/util/Util.java b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/util/Util.java index 54c4396f8..1c7588bd7 100644 --- a/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/util/Util.java +++ b/jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/util/Util.java @@ -57,7 +57,7 @@ private Util(){ * the keys * @return true, if successful */ - public static boolean contains(Map x, Map y, String[] keys) { + public static boolean contains(Map x, Map y, String[] keys) { for (String key : keys) { if (!x.get(key).equals(y.get(key))) return false; @@ -76,10 +76,10 @@ public static boolean contains(Map x, Map y, Str * the delimiter * @return the string */ - public static String concatenate(Map map, String[] keys, String delimiter) { + public static String concatenate(Map map, String[] keys, String delimiter) { List values = new ArrayList<>(); for (String key : keys) { - values.add(map.get(key)); + values.add(map.get(key).toString()); } return values.stream().collect(Collectors.joining(delimiter)); } @@ -172,12 +172,12 @@ public static Map getHeader(String base64Creds){ return authToken; } - public static List> fetchDataFromS3(String s3Account,String s3Region,String s3Role, String bucketName,String path) throws IOException{ + public static List> fetchDataFromS3(String s3Account,String s3Region,String s3Role, String bucketName,String path) throws IOException{ AmazonS3 s3Client = AmazonS3ClientBuilder.standard() .withCredentials(new AWSStaticCredentialsProvider(new CredentialProvider().getCredentials(s3Account,s3Role))).withRegion(s3Region).build(); S3Object entitiesData = s3Client.getObject(new GetObjectRequest(bucketName, path)); try (BufferedReader reader = new BufferedReader(new InputStreamReader(entitiesData.getObjectContent()))) { - return new ObjectMapper().readValue(reader.lines().collect(Collectors.joining("\n")),new TypeReference>>() {}); + return new ObjectMapper().readValue(reader.lines().collect(Collectors.joining("\n")),new TypeReference>>() {}); } } diff --git a/jobs/pacman-data-shipper/src/test/java/com/tmobile/cso/pacman/datashipper/es/ESManagerTest.java b/jobs/pacman-data-shipper/src/test/java/com/tmobile/cso/pacman/datashipper/es/ESManagerTest.java index 9fdfc405e..e103a2020 100644 --- a/jobs/pacman-data-shipper/src/test/java/com/tmobile/cso/pacman/datashipper/es/ESManagerTest.java +++ b/jobs/pacman-data-shipper/src/test/java/com/tmobile/cso/pacman/datashipper/es/ESManagerTest.java @@ -70,8 +70,8 @@ public void setUp() throws Exception { @Test public void uploadDataTest() throws Exception{ - List> docs = new ArrayList<>(); - Map doc = new HashMap<>(); + List> docs = new ArrayList<>(); + Map doc = new HashMap<>(); doc.put("id", "id"); docs.add(doc); @@ -170,8 +170,8 @@ public void fetchCurrentCountStatsForAssetGroupsTest() throws Exception{ @Test public void uploadDataWithParentTest() throws Exception{ - List> docs = new ArrayList<>(); - Map doc = new HashMap<>(); + List> docs = new ArrayList<>(); + Map doc = new HashMap<>(); doc.put("id", "id"); docs.add(doc); diff --git a/jobs/pacman-data-shipper/src/test/java/com/tmobile/cso/pacman/datashipper/util/UtilTest.java b/jobs/pacman-data-shipper/src/test/java/com/tmobile/cso/pacman/datashipper/util/UtilTest.java index 1aab26047..295843940 100644 --- a/jobs/pacman-data-shipper/src/test/java/com/tmobile/cso/pacman/datashipper/util/UtilTest.java +++ b/jobs/pacman-data-shipper/src/test/java/com/tmobile/cso/pacman/datashipper/util/UtilTest.java @@ -20,7 +20,7 @@ public void testContains() { @Test public void testConcatenate() { - HashMap hash = new HashMap<>(); + HashMap hash = new HashMap<>(); hash.put("foo", "3"); hash.put("bar", "4"); hash.put("baz", "5"); From 65d379b902aae5a7be9f729717fc6f38e47b74f2 Mon Sep 17 00:00:00 2001 From: johnrexj Date: Wed, 6 Nov 2019 16:54:53 +0530 Subject: [PATCH 042/107] Azure target types scripts added. --- installer/resources/pacbot_app/files/DB.sql | 36 ++++++++++++++++--- .../inventory/AzureFetchOrchestrator.java | 16 +-------- jobs/pacman-data-shipper/pom.xml | 1 + 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/installer/resources/pacbot_app/files/DB.sql b/installer/resources/pacbot_app/files/DB.sql index 5e03682b1..0df3361fa 100644 --- a/installer/resources/pacbot_app/files/DB.sql +++ b/installer/resources/pacbot_app/files/DB.sql @@ -83,8 +83,7 @@ SET @CONFIG_SERVICE_URL='$CONFIG_SERVICE_URL'; SET @PACBOT_AUTOFIX_RESOURCEOWNER_FALLBACK_MAILID='$PACBOT_AUTOFIX_RESOURCEOWNER_FALLBACK_MAILID'; SET @QUALYS_INFO='$QUALYS_INFO'; SET @QUALYS_API_URL='$QUALYS_API_URL'; - - +SET @AZURE_TENANTS='$AZURE_TENANTS'; CREATE TABLE IF NOT EXISTS `OmniSearch_Config` ( `SEARCH_CATEGORY` varchar(100) COLLATE utf8_bin NOT NULL, @@ -1518,6 +1517,7 @@ INSERT IGNORE INTO pac_config_relation (`application`,`parent`) VALUES ('rule',' INSERT IGNORE INTO pac_config_relation (application,parent) VALUES ('rule-engine','rule'); INSERT IGNORE INTO pac_config_relation (application,parent) VALUES ('recommendation-enricher','batch'); INSERT IGNORE INTO pac_config_relation (application,parent) VALUES ('qualys-enricher','batch'); +INSERT IGNORE INTO pac_config_relation (application,parent) VALUES ('azure-discovery','batch'); INSERT IGNORE INTO pac_config_key_metadata (`cfkey`,`description`) VALUES ('admin.api-role','Description PlaceHolder'); INSERT IGNORE INTO pac_config_key_metadata (`cfkey`,`description`) VALUES ('admin.push.notification.pollinterval.milliseconds','description'); @@ -1851,8 +1851,7 @@ INSERT IGNORE INTO pac_config_key_metadata (`cfkey`,`description`) VALUES ('vuln INSERT IGNORE INTO pac_config_key_metadata (`cfkey`,`description`) VALUES ('vulnerability.application.resourcedetailsboth','Description PlaceHolder'); INSERT IGNORE INTO `pac_config_key_metadata` (`cfkey`, `description`) values('qualys_info','Base64 encoded user:password of qualys'); INSERT IGNORE INTO `pac_config_key_metadata` (`cfkey`, `description`) values('qualys_api_url','Qualys api url'); - - +INSERT IGNORE INTO `pac_config_key_metadata` (`cfkey`, `description`) values('tenants','Azure tenants'); INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('logging.config','classpath:spring-logback.xml','application','prd','latest',NULL,NULL,NULL,NULL); @@ -2144,6 +2143,7 @@ INSERT IGNORE INTO pac_config_properties(`cfkey`,`value`,`application`,`profile` INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('server.servlet.context-path','/api/vulnerability','vulnerability-service','prd','latest',NULL,NULL,NULL,NULL); INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('qualys_info',concat(@QUALYS_INFO,''),'qualys-enricher','prd','latest',NULL,NULL,NULL,NULL); INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('qualys_api_url',concat(@QUALYS_API_URL,''),'qualys-enricher','prd','latest',NULL,NULL,NULL,NULL); +INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('tenants',concat(@AZURE_TENANTS,''),'azure-discovery','prd','latest',NULL,NULL,NULL,NULL); INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('vulnerability.application.occurance','severity,_resourceid,pciflag,_vulnage,vulntype,title,classification,_firstFound,_lastFound,qid,patchable,category','vulnerability-service','prd','latest',NULL,NULL,NULL,NULL); @@ -2562,3 +2562,31 @@ DELETE FROM `pac_config_properties` WHERE cfkey='features.vulnerability.enabled' INSERT IGNORE INTO pac_config_properties(`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('features.vulnerability.enabled',concat(@VULNERABILITY_FEATURE_ENABLED,''),'api','prd','latest',NULL,NULL,NULL,NULL); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('batchaccounts','Azure batchaccounts','Compute','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_batchaccounts/batchaccounts'),'2019-09-19','2019-09-19','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('blobcontainer','Azure blobcontainer','Storage','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_blobcontainer/blobcontainer'),'2019-06-27','2019-06-27','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('cosmosdb','Azure cosmosdb)','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_cosmosdb/cosmosdb','2019-06-27','2019-06-27','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('databricks','Azure databricks)','Analytics','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_databricks/databricks','2019-06-27','2019-06-27','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('disk','Azure Disk','Compute','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_disk/disk'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('loadbalancer','Azure Loadbalancer','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_loadbalancer/loadbalancer'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('mariadb','Azure mariadb','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_mariadb/mariadb'),'2019-06-27','2019-06-27','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('mysqlserver','Azure mysqlserver','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_mysqlserver/mysqlserver'),'2019-06-27','2019-06-27','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('namespaces','Azure namespaces','Web','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_namespaces/namespaces'),'2019-09-19','2019-09-19','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('networkinterface','Azure Network Interface','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_networkinterface/networkinterface'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('nsg','Azure Network Security Group','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_nsg/nsg'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('policydefinitions','Azure policydefinitions','Governance','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_policydefinitions/policydefinitions'),'2019-08-08','2019-08-08','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('policyevaluationresults','Azure policyevaluationresults','Governance','azure','{\"key\":\"id,policyDefinitionId\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_policyevaluationresults/policyevaluationresults'),'2019-08-08','2019-08-08','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('postgresql','Azure postgresql','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_postgresql/postgresql'),'2019-06-27','2019-06-27','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('publicipaddress','Azure publicipaddress','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_publicipaddress/publicipaddress'),'2019-07-01','2019-07-01','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('resourcegroup','Azure resourcegroup','General','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_resourcegroup/resourcegroup'),'2019-06-27','2019-06-27','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('searchservices','Azure searchservices','Web','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_searchservices/searchservices'),'2019-09-19','2019-09-19','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('securityalerts','Azure securityalerts','Governance','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_securityalerts/securityalerts'),'2019-08-08','2019-08-08','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('securitycenter','Azure Security Center','Security','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_securitycenter/securitycenter'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('sites','Azure sites','Internet of things','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_sites/sites'),'2019-09-19','2019-09-19','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('sqldatabase','Azure SQL Database','Databases','azure','{\"key\":\"databaseId\",\"id\":\"databaseId\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_sqldatabase/sqldatabase'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('sqlserver','Azure sqlserver','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_sqlserver/sqlserver'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('storageaccount','Azure Object Storage Accounts','Storage','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_storageaccount/storageaccount'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('subnets','Azure subnets','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_subnets/subnets'),'2019-09-19','2019-09-19','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('vaults','Azure vaults','Security','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_vaults/vaults'),'2019-09-19','2019-09-19','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('virtualmachine','Azure Virtual Machines','Compute','azure','{\"key\":\"vmId\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_virtualmachine/virtualmachine'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('vnet','Azure Disk','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_vnet/vnet'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('workflows','Azure workflows','Internet of things','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_workflows/workflows'),'2019-09-19','2019-09-19','Infra & Platforms'); \ No newline at end of file diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureFetchOrchestrator.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureFetchOrchestrator.java index 8fa28ea69..626a70020 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureFetchOrchestrator.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/AzureFetchOrchestrator.java @@ -33,12 +33,9 @@ public class AzureFetchOrchestrator { @Autowired S3Uploader s3Uploader; - @Value("${file.path}") private String filePath ; - /** The target types. */ - @Value("${subscriptions:}") - private String subscriptions; + @Value("${tenants:}") private String tenants; @@ -90,17 +87,6 @@ private List fetchSubscriptions() { List subscriptionList = new ArrayList<>(); - /*if(subscriptions != null && !"".equals(subscriptions)){ - String[] subscriptionsArray = subscriptions.split(","); - for(String subcritpionInfo : subscriptionsArray){ - SubscriptionVH subscription= new SubscriptionVH(); - String[] subIdName = subcritpionInfo.split("~"); - subscription.setSubscriptionId(subIdName[0].trim()); - subscription.setSubscriptionName(subIdName.length>1?subIdName[1].trim():""); - subscriptionList.add(subscription); - } - }*/ - if(tenants != null && !"".equals(tenants)){ String[] tenantList = tenants.split(","); for(String tenant : tenantList){ diff --git a/jobs/pacman-data-shipper/pom.xml b/jobs/pacman-data-shipper/pom.xml index 26ca264ad..98a3a7478 100644 --- a/jobs/pacman-data-shipper/pom.xml +++ b/jobs/pacman-data-shipper/pom.xml @@ -196,6 +196,7 @@ + From 70c04bce8f825e97cbfc027d406fcf1632302bbb Mon Sep 17 00:00:00 2001 From: johnrexj Date: Thu, 7 Nov 2019 10:33:55 +0530 Subject: [PATCH 043/107] updated config properties for azure --- installer/resources/pacbot_app/files/DB.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/installer/resources/pacbot_app/files/DB.sql b/installer/resources/pacbot_app/files/DB.sql index 0df3361fa..ec8489931 100644 --- a/installer/resources/pacbot_app/files/DB.sql +++ b/installer/resources/pacbot_app/files/DB.sql @@ -83,7 +83,6 @@ SET @CONFIG_SERVICE_URL='$CONFIG_SERVICE_URL'; SET @PACBOT_AUTOFIX_RESOURCEOWNER_FALLBACK_MAILID='$PACBOT_AUTOFIX_RESOURCEOWNER_FALLBACK_MAILID'; SET @QUALYS_INFO='$QUALYS_INFO'; SET @QUALYS_API_URL='$QUALYS_API_URL'; -SET @AZURE_TENANTS='$AZURE_TENANTS'; CREATE TABLE IF NOT EXISTS `OmniSearch_Config` ( `SEARCH_CATEGORY` varchar(100) COLLATE utf8_bin NOT NULL, @@ -1851,7 +1850,6 @@ INSERT IGNORE INTO pac_config_key_metadata (`cfkey`,`description`) VALUES ('vuln INSERT IGNORE INTO pac_config_key_metadata (`cfkey`,`description`) VALUES ('vulnerability.application.resourcedetailsboth','Description PlaceHolder'); INSERT IGNORE INTO `pac_config_key_metadata` (`cfkey`, `description`) values('qualys_info','Base64 encoded user:password of qualys'); INSERT IGNORE INTO `pac_config_key_metadata` (`cfkey`, `description`) values('qualys_api_url','Qualys api url'); -INSERT IGNORE INTO `pac_config_key_metadata` (`cfkey`, `description`) values('tenants','Azure tenants'); INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('logging.config','classpath:spring-logback.xml','application','prd','latest',NULL,NULL,NULL,NULL); @@ -2143,7 +2141,8 @@ INSERT IGNORE INTO pac_config_properties(`cfkey`,`value`,`application`,`profile` INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('server.servlet.context-path','/api/vulnerability','vulnerability-service','prd','latest',NULL,NULL,NULL,NULL); INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('qualys_info',concat(@QUALYS_INFO,''),'qualys-enricher','prd','latest',NULL,NULL,NULL,NULL); INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('qualys_api_url',concat(@QUALYS_API_URL,''),'qualys-enricher','prd','latest',NULL,NULL,NULL,NULL); -INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('tenants',concat(@AZURE_TENANTS,''),'azure-discovery','prd','latest',NULL,NULL,NULL,NULL); +INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('s3.data','azure-inventory','azure-discovery','prd','latest',NULL,NULL,NULL,NULL); +INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('s3.processed','backup-azure',''),'azure-discovery','prd','latest',NULL,NULL,NULL,NULL); INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('vulnerability.application.occurance','severity,_resourceid,pciflag,_vulnage,vulntype,title,classification,_firstFound,_lastFound,qid,patchable,category','vulnerability-service','prd','latest',NULL,NULL,NULL,NULL); From c179353dba999a685a1ba3006f12499ae6c94ee8 Mon Sep 17 00:00:00 2001 From: Sajeer Date: Thu, 7 Nov 2019 14:04:55 +0530 Subject: [PATCH 044/107] Azure integration --- installer/resources/lambda_submit/function.py | 83 +++++++++++++++++-- installer/resources/pacbot_app/utils.py | 6 ++ installer/settings/default.local.py | 4 + 3 files changed, 84 insertions(+), 9 deletions(-) diff --git a/installer/resources/lambda_submit/function.py b/installer/resources/lambda_submit/function.py index 4a28430a4..2fab43c51 100644 --- a/installer/resources/lambda_submit/function.py +++ b/installer/resources/lambda_submit/function.py @@ -37,7 +37,6 @@ class SubmitJobLambdaFunction(LambdaFunctionResource): class DataCollectorEventRule(CloudWatchEventRuleResource): name = "AWS-Data-Collector" schedule_expression = "cron(0 * * * ? *)" - DEPENDS_ON = [SubmitJobLambdaFunction] @@ -74,7 +73,6 @@ class DataCollectorCloudWatchEventTarget(CloudWatchEventTargetResource): class DataShipperEventRule(CloudWatchEventRuleResource): name = "aws-redshift-es-data-shipper" schedule_expression = "cron(0 * * * ? *)" - DEPENDS_ON = [SubmitJobLambdaFunction, ESDomainPolicy] @@ -119,7 +117,6 @@ class DataShipperCloudWatchEventTarget(CloudWatchEventTargetResource): class RecommendationsCollectorEventRule(CloudWatchEventRuleResource): name = "AWS-Recommendations-Collector" schedule_expression = "cron(0 * * * ? *)" - DEPENDS_ON = [SubmitJobLambdaFunction] @@ -161,7 +158,6 @@ class RecommendationsCollectorCloudWatchEventTarget(CloudWatchEventTargetResourc class CloudNotificationCollectorEventRule(CloudWatchEventRuleResource): name = "AWS-CloudNotification-Collector" schedule_expression = "cron(0 * * * ? *)" - DEPENDS_ON = [SubmitJobLambdaFunction] @@ -203,7 +199,6 @@ class CloudNotificationCollectorCloudWatchEventTarget(CloudWatchEventTargetResou class QualysKBCollectorEventRule(CloudWatchEventRuleResource): name = "qualys-kb-collector" schedule_expression = "cron(0 0 * * ? *)" - DEPENDS_ON = [SubmitJobLambdaFunction] PROCESS = need_to_deploy_vulnerability_service() @@ -214,7 +209,6 @@ class QualysKBCollectorEventRuleLambdaPermission(LambdaPermission): function_name = SubmitJobLambdaFunction.get_output_attr('function_name') principal = "events.amazonaws.com" source_arn = QualysKBCollectorEventRule.get_output_attr('arn') - PROCESS = need_to_deploy_vulnerability_service() @@ -243,7 +237,6 @@ class QualysKBCollectorCloudWatchEventTarget(CloudWatchEventTargetResource): class QualysAssetDataImporterEventRule(CloudWatchEventRuleResource): name = "qualys-asset-data-importer" schedule_expression = "cron(0 1 * * ? *)" - DEPENDS_ON = [SubmitJobLambdaFunction] PROCESS = need_to_deploy_vulnerability_service() @@ -254,7 +247,6 @@ class QualysAssetDataImporterEventRuleLambdaPermission(LambdaPermission): function_name = SubmitJobLambdaFunction.get_output_attr('function_name') principal = "events.amazonaws.com" source_arn = QualysAssetDataImporterEventRule.get_output_attr('arn') - PROCESS = need_to_deploy_vulnerability_service() @@ -278,5 +270,78 @@ class QualysAssetDataImporterCloudWatchEventTarget(CloudWatchEventTargetResource {'encrypt': False, 'key': "datasource", 'value': "aws"} ] }) - PROCESS = need_to_deploy_vulnerability_service() + + +class AzureDataCollectorEventRule(CloudWatchEventRuleResource): + name = "pacbot-azure-discovery" + schedule_expression = "cron(0 * * * ? *)" + DEPENDS_ON = [SubmitJobLambdaFunction] + PROCESS = need_to_enable_azure() + + +class AzureDataCollectorEventRuleLambdaPermission(LambdaPermission): + statement_id = "AllowExecutionFromAzureDataCollectorEvent" + action = "lambda:InvokeFunction" + function_name = SubmitJobLambdaFunction.get_output_attr('function_name') + principal = "events.amazonaws.com" + source_arn = AzureDataCollectorEventRule.get_output_attr('arn') + PROCESS = need_to_enable_azure() + + +class AzureDataCollectorCloudWatchEventTarget(CloudWatchEventTargetResource): + rule = AzureDataCollectorEventRule.get_output_attr('name') + arn = SubmitJobLambdaFunction.get_output_attr('arn') + target_id = 'AzureDataCollectorTarget' # Unique identifier + target_input = json.dumps({ + 'jobName': "pacbot-azure-discovery", + 'jobUuid': "pacbot-azure-discovery", + 'jobType': "jar", + 'jobDesc': "Collects azure data and upload to S3", + 'environmentVariables': [ + {'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,azure-discovery/prd/latest"}, + ], + 'params': [ + {'encrypt': False, 'key': "package_hint", 'value': "com.tmobile.pacbot"}, + {'encrypt': False, 'key': "file.path", 'value': "/home/ec2-user/azure-data"}, + ] + }) + PROCESS = need_to_enable_azure() + + +class AzureDataShipperEventRule(CloudWatchEventRuleResource): + name = "data-shipper-azure" + schedule_expression = "cron(0 * * * ? *)" + DEPENDS_ON = [SubmitJobLambdaFunction, ESDomainPolicy] + PROCESS = need_to_enable_azure() + + +class AzureDataShipperEventRuleLambdaPermission(LambdaPermission): + statement_id = "AllowExecutionFromAzureDataShipper" + action = "lambda:InvokeFunction" + function_name = SubmitJobLambdaFunction.get_output_attr('function_name') + principal = "events.amazonaws.com" + source_arn = AzureDataShipperEventRule.get_output_attr('arn') + PROCESS = need_to_enable_azure() + + +class AzureDataShipperCloudWatchEventTarget(CloudWatchEventTargetResource): + rule = AzureDataShipperEventRule.get_output_attr('name') + arn = SubmitJobLambdaFunction.get_output_attr('arn') + target_id = 'AzureDataShipperTarget' # Unique identifier + target_input = json.dumps({ + 'jobName': "data-shipper-azure", + 'jobUuid': "data-shipper-azure", + 'jobType': "jar", + 'jobDesc': "Ship Azure Data from S3 to PacBot ES", + 'environmentVariables': [ + {'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,azure-discovery/prd/latest"}, + ], + 'params': [ + {'encrypt': False, 'key': "package_hint", 'value': "com.tmobile.cso.pacman"}, + {'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="}, + {'encrypt': False, 'key': "datasource", 'value': "azure"}, + {'encrypt': False, 'key': "s3.data", 'value': "azure-inventory"} + ] + }) + PROCESS = need_to_enable_azure() \ No newline at end of file diff --git a/installer/resources/pacbot_app/utils.py b/installer/resources/pacbot_app/utils.py index 230047c03..6ce370741 100644 --- a/installer/resources/pacbot_app/utils.py +++ b/installer/resources/pacbot_app/utils.py @@ -5,3 +5,9 @@ def need_to_deploy_vulnerability_service(): feature_status = Settings.get('ENABLE_VULNERABILITY_FEATURE', False) return feature_status + + +def need_to_enable_azure(): + feature_status = Settings.get('ENABLE_AZURE', False) + + return feature_status diff --git a/installer/settings/default.local.py b/installer/settings/default.local.py index 7af122139..d27ba3425 100644 --- a/installer/settings/default.local.py +++ b/installer/settings/default.local.py @@ -52,3 +52,7 @@ ENABLE_VULNERABILITY_FEATURE = False QUALYS_API_URL = "" # Qualys API Url without trailing slash QUALYS_INFO = "" # Base64 encoded user:password of qualys + +# This settings enable Vulnerability feature and servie +ENABLE_AZURE = False +AZURE_TENANTS = "" From a6e66a3e92585d9835b0b1dd0b6a38919b469693 Mon Sep 17 00:00:00 2001 From: Sajeer Date: Thu, 7 Nov 2019 14:38:35 +0530 Subject: [PATCH 045/107] Added mising function import --- installer/resources/lambda_submit/function.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/resources/lambda_submit/function.py b/installer/resources/lambda_submit/function.py index 2fab43c51..6fab06b43 100644 --- a/installer/resources/lambda_submit/function.py +++ b/installer/resources/lambda_submit/function.py @@ -10,7 +10,7 @@ from resources.data.aws_info import AwsAccount, AwsRegion from resources.lambda_submit.s3_upload import UploadLambdaSubmitJobZipFile, BATCH_JOB_FILE_NAME from resources.pacbot_app.alb import ApplicationLoadBalancer -from resources.pacbot_app.utils import need_to_deploy_vulnerability_service +from resources.pacbot_app.utils import need_to_deploy_vulnerability_service, need_to_enable_azure import json From b90b260ff32e2b9b05228c103fd1f6a227719421 Mon Sep 17 00:00:00 2001 From: Sajeer Date: Thu, 7 Nov 2019 14:40:29 +0530 Subject: [PATCH 046/107] Added missing comma in json --- .../lambda_rule_engine/files/rule_engine_cloudwatch_rules.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json b/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json index 5caed9008..2180a6510 100644 --- a/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json +++ b/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json @@ -2792,7 +2792,7 @@ "modifiedDate": "2019-11-04", "severity": "high", "category": "security" - } + }, { "ruleId": "PacMan_Azure_Enable_Adaptive_Application_controls_version-1_SecurityCenter_virtualmachine", "ruleUUID": "azure_virtualmachine_enable_adaptive_application", From b38efe8288603efd449c3fef163c1195e90411d5 Mon Sep 17 00:00:00 2001 From: Sajeer Date: Thu, 7 Nov 2019 14:57:51 +0530 Subject: [PATCH 047/107] After reinstall-destroy part success message is shown --- installer/core/providers/aws/reinstall.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/installer/core/providers/aws/reinstall.py b/installer/core/providers/aws/reinstall.py index 9c4e11f61..7336a5ef0 100644 --- a/installer/core/providers/aws/reinstall.py +++ b/installer/core/providers/aws/reinstall.py @@ -101,3 +101,7 @@ def render_terraform_destroy_progress(self): self.show_step_heading(K.TERRAFORM_REDEPLOY_DESTROY_STARTED, write_log=False) while self.destroy is False and self.terraform_thread.isAlive(): self.show_progress_message(K.TERRAFORM_DESTROY_STARTED, 0.5) + + end_time = datetime.now() + self.show_step_finish(K.TERRAFORM_DESTROY_COMPLETED, write_log=False, color=self.GREEN_ANSI) + self.display_process_duration(start_time, end_time) From c4b276f250b8538f6921d21ca388fcd58e51efe6 Mon Sep 17 00:00:00 2001 From: Sajeer Date: Thu, 7 Nov 2019 15:12:17 +0530 Subject: [PATCH 048/107] DB.sql syntax issue corrected --- .../resources/lambda_rule_engine/utils.py | 4 ++ installer/resources/pacbot_app/files/DB.sql | 56 +++++++++---------- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/installer/resources/lambda_rule_engine/utils.py b/installer/resources/lambda_rule_engine/utils.py index 1a8d935bf..dd2c99b35 100644 --- a/installer/resources/lambda_rule_engine/utils.py +++ b/installer/resources/lambda_rule_engine/utils.py @@ -1,4 +1,5 @@ from resources.iam.base_role import BaseRole +from resources.pacbot_app.utils import need_to_enable_azure import json @@ -15,6 +16,9 @@ def get_rule_engine_cloudwatch_rules_var(): variable_dict_input = json.loads(data) for index in range(len(variable_dict_input)): + if not need_to_enable_azure and variable_dict_input['assetGroup'] == "azure": + continue + mod = index % 20 + 5 item = { 'ruleId': variable_dict_input[index]['ruleUUID'], diff --git a/installer/resources/pacbot_app/files/DB.sql b/installer/resources/pacbot_app/files/DB.sql index 0df3361fa..80c101ac3 100644 --- a/installer/resources/pacbot_app/files/DB.sql +++ b/installer/resources/pacbot_app/files/DB.sql @@ -2562,31 +2562,31 @@ DELETE FROM `pac_config_properties` WHERE cfkey='features.vulnerability.enabled' INSERT IGNORE INTO pac_config_properties(`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('features.vulnerability.enabled',concat(@VULNERABILITY_FEATURE_ENABLED,''),'api','prd','latest',NULL,NULL,NULL,NULL); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('batchaccounts','Azure batchaccounts','Compute','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_batchaccounts/batchaccounts'),'2019-09-19','2019-09-19','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('blobcontainer','Azure blobcontainer','Storage','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_blobcontainer/blobcontainer'),'2019-06-27','2019-06-27','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('cosmosdb','Azure cosmosdb)','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_cosmosdb/cosmosdb','2019-06-27','2019-06-27','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('databricks','Azure databricks)','Analytics','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_databricks/databricks','2019-06-27','2019-06-27','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('disk','Azure Disk','Compute','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_disk/disk'),'2019-11-05','2019-11-05','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('loadbalancer','Azure Loadbalancer','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_loadbalancer/loadbalancer'),'2019-11-05','2019-11-05','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('mariadb','Azure mariadb','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_mariadb/mariadb'),'2019-06-27','2019-06-27','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('mysqlserver','Azure mysqlserver','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_mysqlserver/mysqlserver'),'2019-06-27','2019-06-27','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('namespaces','Azure namespaces','Web','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_namespaces/namespaces'),'2019-09-19','2019-09-19','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('networkinterface','Azure Network Interface','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_networkinterface/networkinterface'),'2019-11-05','2019-11-05','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('nsg','Azure Network Security Group','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_nsg/nsg'),'2019-11-05','2019-11-05','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('policydefinitions','Azure policydefinitions','Governance','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_policydefinitions/policydefinitions'),'2019-08-08','2019-08-08','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('policyevaluationresults','Azure policyevaluationresults','Governance','azure','{\"key\":\"id,policyDefinitionId\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_policyevaluationresults/policyevaluationresults'),'2019-08-08','2019-08-08','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('postgresql','Azure postgresql','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_postgresql/postgresql'),'2019-06-27','2019-06-27','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('publicipaddress','Azure publicipaddress','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_publicipaddress/publicipaddress'),'2019-07-01','2019-07-01','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('resourcegroup','Azure resourcegroup','General','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_resourcegroup/resourcegroup'),'2019-06-27','2019-06-27','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('searchservices','Azure searchservices','Web','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_searchservices/searchservices'),'2019-09-19','2019-09-19','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('securityalerts','Azure securityalerts','Governance','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_securityalerts/securityalerts'),'2019-08-08','2019-08-08','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('securitycenter','Azure Security Center','Security','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_securitycenter/securitycenter'),'2019-11-05','2019-11-05','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('sites','Azure sites','Internet of things','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_sites/sites'),'2019-09-19','2019-09-19','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('sqldatabase','Azure SQL Database','Databases','azure','{\"key\":\"databaseId\",\"id\":\"databaseId\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_sqldatabase/sqldatabase'),'2019-11-05','2019-11-05','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('sqlserver','Azure sqlserver','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_sqlserver/sqlserver'),'2019-11-05','2019-11-05','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('storageaccount','Azure Object Storage Accounts','Storage','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_storageaccount/storageaccount'),'2019-11-05','2019-11-05','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('subnets','Azure subnets','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_subnets/subnets'),'2019-09-19','2019-09-19','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('vaults','Azure vaults','Security','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_vaults/vaults'),'2019-09-19','2019-09-19','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('virtualmachine','Azure Virtual Machines','Compute','azure','{\"key\":\"vmId\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_virtualmachine/virtualmachine'),'2019-11-05','2019-11-05','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('vnet','Azure Disk','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_vnet/vnet'),'2019-11-05','2019-11-05','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('workflows','Azure workflows','Internet of things','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_workflows/workflows'),'2019-09-19','2019-09-19','Infra & Platforms'); \ No newline at end of file +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('batchaccounts','Azure batchaccounts','Compute','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_batchaccounts/batchaccounts'),'2019-09-19','2019-09-19','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('blobcontainer','Azure blobcontainer','Storage','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_blobcontainer/blobcontainer'),'2019-06-27','2019-06-27','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('cosmosdb','Azure cosmosdb)','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_cosmosdb/cosmosdb','2019-06-27','2019-06-27','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('databricks','Azure databricks)','Analytics','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_databricks/databricks','2019-06-27','2019-06-27','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('disk','Azure Disk','Compute','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_disk/disk'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('loadbalancer','Azure Loadbalancer','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_loadbalancer/loadbalancer'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('mariadb','Azure mariadb','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_mariadb/mariadb'),'2019-06-27','2019-06-27','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('mysqlserver','Azure mysqlserver','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_mysqlserver/mysqlserver'),'2019-06-27','2019-06-27','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('namespaces','Azure namespaces','Web','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_namespaces/namespaces'),'2019-09-19','2019-09-19','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('networkinterface','Azure Network Interface','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_networkinterface/networkinterface'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('nsg','Azure Network Security Group','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_nsg/nsg'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('policydefinitions','Azure policydefinitions','Governance','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_policydefinitions/policydefinitions'),'2019-08-08','2019-08-08','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('policyevaluationresults','Azure policyevaluationresults','Governance','azure','{\"key\":\"id,policyDefinitionId\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_policyevaluationresults/policyevaluationresults'),'2019-08-08','2019-08-08','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('postgresql','Azure postgresql','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_postgresql/postgresql'),'2019-06-27','2019-06-27','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('publicipaddress','Azure publicipaddress','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_publicipaddress/publicipaddress'),'2019-07-01','2019-07-01','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('resourcegroup','Azure resourcegroup','General','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_resourcegroup/resourcegroup'),'2019-06-27','2019-06-27','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('searchservices','Azure searchservices','Web','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_searchservices/searchservices'),'2019-09-19','2019-09-19','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('securityalerts','Azure securityalerts','Governance','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_securityalerts/securityalerts'),'2019-08-08','2019-08-08','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('securitycenter','Azure Security Center','Security','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_securitycenter/securitycenter'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('sites','Azure sites','Internet of things','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_sites/sites'),'2019-09-19','2019-09-19','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('sqldatabase','Azure SQL Database','Databases','azure','{\"key\":\"databaseId\",\"id\":\"databaseId\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_sqldatabase/sqldatabase'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('sqlserver','Azure sqlserver','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_sqlserver/sqlserver'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('storageaccount','Azure Object Storage Accounts','Storage','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_storageaccount/storageaccount'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('subnets','Azure subnets','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_subnets/subnets'),'2019-09-19','2019-09-19','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('vaults','Azure vaults','Security','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_vaults/vaults'),'2019-09-19','2019-09-19','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('virtualmachine','Azure Virtual Machines','Compute','azure','{\"key\":\"vmId\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_virtualmachine/virtualmachine'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('vnet','Azure Disk','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_vnet/vnet'),'2019-11-05','2019-11-05','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('workflows','Azure workflows','Internet of things','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_workflows/workflows'),'2019-09-19','2019-09-19','Infra & Platforms'); From 505fe63a1c7b0e885aad73d7258677852cb01cb4 Mon Sep 17 00:00:00 2001 From: Sajeer Date: Thu, 7 Nov 2019 15:31:09 +0530 Subject: [PATCH 049/107] DB SQL syntax issue corrected --- installer/resources/pacbot_app/files/DB.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/resources/pacbot_app/files/DB.sql b/installer/resources/pacbot_app/files/DB.sql index 80c101ac3..52093f4d2 100644 --- a/installer/resources/pacbot_app/files/DB.sql +++ b/installer/resources/pacbot_app/files/DB.sql @@ -2565,7 +2565,7 @@ INSERT IGNORE INTO pac_config_properties(`cfkey`,`value`,`application`,`profile` INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('batchaccounts','Azure batchaccounts','Compute','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_batchaccounts/batchaccounts'),'2019-09-19','2019-09-19','Infra & Platforms'); INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('blobcontainer','Azure blobcontainer','Storage','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_blobcontainer/blobcontainer'),'2019-06-27','2019-06-27','Infra & Platforms'); INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('cosmosdb','Azure cosmosdb)','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_cosmosdb/cosmosdb','2019-06-27','2019-06-27','Infra & Platforms'); -INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('databricks','Azure databricks)','Analytics','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_databricks/databricks','2019-06-27','2019-06-27','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('databricks','Azure databricks)','Analytics','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_databricks/databricks'),'2019-06-27','2019-06-27','Infra & Platforms'); INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('disk','Azure Disk','Compute','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_disk/disk'),'2019-11-05','2019-11-05','Infra & Platforms'); INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('loadbalancer','Azure Loadbalancer','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_loadbalancer/loadbalancer'),'2019-11-05','2019-11-05','Infra & Platforms'); INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('mariadb','Azure mariadb','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_mariadb/mariadb'),'2019-06-27','2019-06-27','Infra & Platforms'); From 721ad12bf37cb74b284e9c9e7a6b7ac9795e9e2c Mon Sep 17 00:00:00 2001 From: Sajeer Date: Thu, 7 Nov 2019 15:37:14 +0530 Subject: [PATCH 050/107] Azure CW rules are created only if it is enabled --- installer/resources/lambda_rule_engine/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/installer/resources/lambda_rule_engine/utils.py b/installer/resources/lambda_rule_engine/utils.py index dd2c99b35..b86562d14 100644 --- a/installer/resources/lambda_rule_engine/utils.py +++ b/installer/resources/lambda_rule_engine/utils.py @@ -16,8 +16,8 @@ def get_rule_engine_cloudwatch_rules_var(): variable_dict_input = json.loads(data) for index in range(len(variable_dict_input)): - if not need_to_enable_azure and variable_dict_input['assetGroup'] == "azure": - continue + if variable_dict_input[index]['assetGroup'] == "azure" and not need_to_enable_azure: + del(variable_dict_input[index]) mod = index % 20 + 5 item = { From 9308bb6fa1deeff65e61ca734ee8aa47b0de3e94 Mon Sep 17 00:00:00 2001 From: Sajeer Date: Thu, 7 Nov 2019 15:53:38 +0530 Subject: [PATCH 051/107] DB.sql synctax corected --- installer/resources/pacbot_app/files/DB.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/resources/pacbot_app/files/DB.sql b/installer/resources/pacbot_app/files/DB.sql index 52093f4d2..48ca1fea4 100644 --- a/installer/resources/pacbot_app/files/DB.sql +++ b/installer/resources/pacbot_app/files/DB.sql @@ -2564,7 +2564,7 @@ INSERT IGNORE INTO pac_config_properties(`cfkey`,`value`,`application`,`profile` INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('batchaccounts','Azure batchaccounts','Compute','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_batchaccounts/batchaccounts'),'2019-09-19','2019-09-19','Infra & Platforms'); INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('blobcontainer','Azure blobcontainer','Storage','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_blobcontainer/blobcontainer'),'2019-06-27','2019-06-27','Infra & Platforms'); -INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('cosmosdb','Azure cosmosdb)','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_cosmosdb/cosmosdb','2019-06-27','2019-06-27','Infra & Platforms'); +INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('cosmosdb','Azure cosmosdb)','Databases','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_cosmosdb/cosmosdb'),'2019-06-27','2019-06-27','Infra & Platforms'); INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('databricks','Azure databricks)','Analytics','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_databricks/databricks'),'2019-06-27','2019-06-27','Infra & Platforms'); INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('disk','Azure Disk','Compute','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_disk/disk'),'2019-11-05','2019-11-05','Infra & Platforms'); INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('loadbalancer','Azure Loadbalancer','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_loadbalancer/loadbalancer'),'2019-11-05','2019-11-05','Infra & Platforms'); From b76cb67ac5e3f1dcf2f85849a477c423fd99ce6b Mon Sep 17 00:00:00 2001 From: Sajeer Date: Thu, 7 Nov 2019 16:26:58 +0530 Subject: [PATCH 052/107] Rules are generated for azure if it is enabled --- installer/resources/lambda_rule_engine/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/installer/resources/lambda_rule_engine/utils.py b/installer/resources/lambda_rule_engine/utils.py index b86562d14..1aad83e23 100644 --- a/installer/resources/lambda_rule_engine/utils.py +++ b/installer/resources/lambda_rule_engine/utils.py @@ -15,10 +15,10 @@ def get_rule_engine_cloudwatch_rules_var(): data = data.replace("role/pacman_ro", "role/" + BaseRole.get_input_attr('name')) variable_dict_input = json.loads(data) + required_rules = [] for index in range(len(variable_dict_input)): - if variable_dict_input[index]['assetGroup'] == "azure" and not need_to_enable_azure: - del(variable_dict_input[index]) - + if variable_dict_input[index]['assetGroup'] == "azure" and not need_to_enable_azure(): + continue mod = index % 20 + 5 item = { 'ruleId': variable_dict_input[index]['ruleUUID'], @@ -26,6 +26,6 @@ def get_rule_engine_cloudwatch_rules_var(): 'schedule': "cron(%s * * * ? *)" % str(mod) } - variable_dict_input[index] = item + required_rules.append(item) - return variable_dict_input + return required_rules From c9f5b6ddc02b7b4e956f529deff254abdf374b80 Mon Sep 17 00:00:00 2001 From: Sajeer Date: Thu, 7 Nov 2019 17:48:06 +0530 Subject: [PATCH 053/107] Comment added for settings input --- installer/settings/default.local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/settings/default.local.py b/installer/settings/default.local.py index d27ba3425..825caab3d 100644 --- a/installer/settings/default.local.py +++ b/installer/settings/default.local.py @@ -55,4 +55,4 @@ # This settings enable Vulnerability feature and servie ENABLE_AZURE = False -AZURE_TENANTS = "" +AZURE_TENANTS = "" # Comma seperated values for tenants if ENABLE_AZURE is True From 03934f695cce5a8a0ffeb950131b9f7edc8cbcf6 Mon Sep 17 00:00:00 2001 From: Anil Chandran Date: Thu, 7 Nov 2019 18:56:08 -0800 Subject: [PATCH 054/107] Update AzureCredentialProvider.java --- .../auth/AzureCredentialProvider.java | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/auth/AzureCredentialProvider.java b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/auth/AzureCredentialProvider.java index 8798bb3b1..c31749482 100644 --- a/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/auth/AzureCredentialProvider.java +++ b/jobs/azure-discovery/src/main/java/com/tmobile/pacbot/azure/inventory/auth/AzureCredentialProvider.java @@ -1,5 +1,6 @@ package com.tmobile.pacbot.azure.inventory.auth; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -56,8 +57,9 @@ public Authenticated authenticate(String tenant) { private ApplicationTokenCredentials getCredentials(String tenant){ - String clientId = System.getProperty("azure.clientId."+tenant); - String secret = System.getProperty("azure.secret."+tenant); + Map creds = decodeCredetials().get(tenant); + String clientId = creds.get("clientId"); + String secret = creds.get("secretId"); return new ApplicationTokenCredentials(clientId, tenant, secret, AzureEnvironment.AZURE); } @@ -65,8 +67,9 @@ private ApplicationTokenCredentials getCredentials(String tenant){ public String getAuthToken(String tenant) throws Exception { String url = "https://login.microsoftonline.com/%s/oauth2/token"; - String clientId = System.getProperty("azure.clientId."+tenant); - String secret = System.getProperty("azure.secret."+tenant); + Map creds = decodeCredetials().get(tenant); + String clientId = creds.get("clientId"); + String secret = creds.get("secretId"); Map params = new HashMap<>(); @@ -85,6 +88,17 @@ public String getAuthToken(String tenant) throws Exception { throw e; } } + + private Map> decodeCredetials() { + Map> credsMap = new HashMap<>(); + String azureCreds = System.getProperty("azure.credentials"); + Arrays.asList(azureCreds.split("##")).stream().forEach(cred-> { + Map credInfoMap = new HashMap<>(); + Arrays.asList(cred.split(",")).stream().forEach(str-> credInfoMap.put(str.split(":")[0],str.split(":")[1])); + credsMap.put(credInfoMap.get("tenant"), credInfoMap); + }); + return credsMap; + } } From 121ae411a10b66ae7b7aaab3bdf1ddc9d4bd6625 Mon Sep 17 00:00:00 2001 From: Anil Chandran Date: Thu, 7 Nov 2019 20:15:09 -0800 Subject: [PATCH 055/107] Update DB.sql Added azure credentials placholder --- installer/resources/pacbot_app/files/DB.sql | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/installer/resources/pacbot_app/files/DB.sql b/installer/resources/pacbot_app/files/DB.sql index ec8489931..9a3e5486c 100644 --- a/installer/resources/pacbot_app/files/DB.sql +++ b/installer/resources/pacbot_app/files/DB.sql @@ -83,6 +83,7 @@ SET @CONFIG_SERVICE_URL='$CONFIG_SERVICE_URL'; SET @PACBOT_AUTOFIX_RESOURCEOWNER_FALLBACK_MAILID='$PACBOT_AUTOFIX_RESOURCEOWNER_FALLBACK_MAILID'; SET @QUALYS_INFO='$QUALYS_INFO'; SET @QUALYS_API_URL='$QUALYS_API_URL'; +SET @AZURE_CREDENTIALS='$AZURE_CREDENTIALS'; CREATE TABLE IF NOT EXISTS `OmniSearch_Config` ( `SEARCH_CATEGORY` varchar(100) COLLATE utf8_bin NOT NULL, @@ -1235,7 +1236,7 @@ INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_Azure_Resolve_monitoring_agent_version-1','Resolve monitoring agent health issues on your machines','This is Azure Secuirty Rule',"",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_Azure_harden-NSGs_internet_version-1','Harden Network Security Group rules of internet facing Virtual Machines','This is Azure Secuirty Rule',"",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); - + /* Rule Initialisation */ INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_VpcFlowLogsEnabled_version-1_VpcFlowLogsEnabled_vpc','aws_account_should_have_vpclogs_enabled','PacMan_VpcFlowLogsEnabled_version-1','VpcFlowLogsEnabled','vpc','aws','VpcFlowLogsEnabled','{"params":[{"encrypt":"false","value":"role/pacbot_ro","key":"roleIdentifyingString"},{"encrypt":"false","value":"check-for-vpc-flowlog-enabled","key":"ruleKey"},{"encrypt":false,"value":"high","key":"severity"},{"isValueNew":true,"encrypt":false,"value":"security","key":"ruleCategory"}],"environmentVariables":[{"encrypt":false,"value":"123","key":"abc"}],"ruleId":"PacMan_VpcFlowLogsEnabled_version-1_VpcFlowLogsEnabled_vpc","autofix":false,"alexaKeyword":"VpcFlowLogsEnabled","ruleRestUrl":"","targetType":"vpc","pac_ds":"aws","policyId":"PacMan_VpcFlowLogsEnabled_version-1","assetGroup":"aws","ruleUUID":"aws_account_should_have_vpclogs_enabled","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/aws_account_should_have_vpclogs_enabled'),'ENABLED','ASGC','VPC flowlogs should be enabled for all VPCs',{d '2017-08-11'},{d '2018-08-31'},null,null); @@ -2142,7 +2143,8 @@ INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('qualys_info',concat(@QUALYS_INFO,''),'qualys-enricher','prd','latest',NULL,NULL,NULL,NULL); INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('qualys_api_url',concat(@QUALYS_API_URL,''),'qualys-enricher','prd','latest',NULL,NULL,NULL,NULL); INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('s3.data','azure-inventory','azure-discovery','prd','latest',NULL,NULL,NULL,NULL); -INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('s3.processed','backup-azure',''),'azure-discovery','prd','latest',NULL,NULL,NULL,NULL); +INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('s3.processed','backup-azure','azure-discovery','prd','latest',NULL,NULL,NULL,NULL); +INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('azure.credentials',concat(@AZURE_CREDENTIALS,''),'azure-discovery','prd','latest',NULL,NULL,NULL,NULL); INSERT IGNORE INTO pac_config_properties (`cfkey`,`value`,`application`,`profile`,`label`,`createdBy`,`createdDate`,`modifiedBy`,`modifiedDate`) VALUES ('vulnerability.application.occurance','severity,_resourceid,pciflag,_vulnage,vulntype,title,classification,_firstFound,_lastFound,qid,patchable,category','vulnerability-service','prd','latest',NULL,NULL,NULL,NULL); @@ -2588,4 +2590,4 @@ INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `da INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('vaults','Azure vaults','Security','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_vaults/vaults'),'2019-09-19','2019-09-19','Infra & Platforms'); INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('virtualmachine','Azure Virtual Machines','Compute','azure','{\"key\":\"vmId\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_virtualmachine/virtualmachine'),'2019-11-05','2019-11-05','Infra & Platforms'); INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('vnet','Azure Disk','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_vnet/vnet'),'2019-11-05','2019-11-05','Infra & Platforms'); -INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('workflows','Azure workflows','Internet of things','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_workflows/workflows'),'2019-09-19','2019-09-19','Infra & Platforms'); \ No newline at end of file +INSERT IGNORE INTO into `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('workflows','Azure workflows','Internet of things','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_workflows/workflows'),'2019-09-19','2019-09-19','Infra & Platforms'); From 9e524be3f183c89c1d374306a3d0f769922b1430 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Fri, 8 Nov 2019 11:48:53 +0530 Subject: [PATCH 056/107] Azure credentials supplied correctly --- installer/resources/pacbot_app/import_db.py | 17 ++++++++++++++++- installer/settings/default.local.py | 14 +++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/installer/resources/pacbot_app/import_db.py b/installer/resources/pacbot_app/import_db.py index f0c05e6e3..eee387c7a 100644 --- a/installer/resources/pacbot_app/import_db.py +++ b/installer/resources/pacbot_app/import_db.py @@ -25,11 +25,25 @@ class ReplaceSQLPlaceHolder(NullResource): DEPENDS_ON = [MySQLDatabase, ESDomain] + def prepare_azure_tenants_credentias(self): + tenants = Settings.get('AZURE_TENANTS', []) + credential_string = "" + + for tenant in tenants: + tenant_id = tenant['tenant'] + client_id = tenant['clientId'] + seccret_id = tenant['secretId'] + credential_string = "" if credential_string == "" else (credential_string + "##") + credential_string += "tenant:%s,clientId:%s,secretId:%s" % (tenant_id, client_id, seccret_id) + + return credential_string + def get_provisioners(self): script = os.path.join(get_terraform_scripts_dir(), 'sql_replace_placeholder.py') db_user_name = MySQLDatabase.get_input_attr('username') db_password = MySQLDatabase.get_input_attr('password') db_host = MySQLDatabase.get_output_attr('endpoint') + azure_credentails = self.prepare_azure_tenants_credentias() local_execs = [ { 'local-exec': { @@ -96,7 +110,8 @@ def get_provisioners(self): 'ENV_CONFIG_SERVICE_URL': ApplicationLoadBalancer.get_http_url() + "/api/config/rule/prd/latest", 'ENV_PACBOT_AUTOFIX_RESOURCEOWNER_FALLBACK_MAILID': Settings.get('USER_EMAIL_ID', ""), 'ENV_QUALYS_INFO': Settings.get('QUALYS_INFO', ""), - 'ENV_QUALYS_API_URL': Settings.get('QUALYS_API_URL', "") + 'ENV_QUALYS_API_URL': Settings.get('QUALYS_API_URL', ""), + 'ENV_AZURE_CREDENTIALS': azure_credentails, }, 'interpreter': [Settings.PYTHON_INTERPRETER] } diff --git a/installer/settings/default.local.py b/installer/settings/default.local.py index 23b7f16a6..1e59d202c 100644 --- a/installer/settings/default.local.py +++ b/installer/settings/default.local.py @@ -58,4 +58,16 @@ # This settings enable Vulnerability feature and servie ENABLE_AZURE = False -AZURE_TENANTS = "" # Comma seperated values for tenants if ENABLE_AZURE is True +# Tenants should be a list of dict containing tenant, clientId and secretId +AZURE_TENANTS = [ + { + 'tenant': "t111", + 'clientId': "c111", + 'secretId': "s111" + }, + { + 'tenant': "t222", + 'clientId': "c222", + 'secretId': "s222" + }, +] From e27ef69535929818259bbc66358bc17d1f2fda27 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Fri, 8 Nov 2019 17:22:51 +0530 Subject: [PATCH 057/107] Destroy progress message issue corrected. Condition added for azure tenants variable initilization --- installer/core/providers/aws/reinstall.py | 10 +++++++--- installer/resources/pacbot_app/import_db.py | 14 ++++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/installer/core/providers/aws/reinstall.py b/installer/core/providers/aws/reinstall.py index 7336a5ef0..94867e127 100644 --- a/installer/core/providers/aws/reinstall.py +++ b/installer/core/providers/aws/reinstall.py @@ -80,6 +80,7 @@ def re_create_resources(self, resources_to_destroy, resources_to_install, terraf except Exception as e: self.executed_with_error = True self.exception = e + self.destroy = True #If there is any error in destroy set destroy to True self._cleanup_installation_process(dry_run) @@ -97,11 +98,14 @@ def show_progress_status_all(self, resources, terraform_with_targets, dry_run): def render_terraform_destroy_progress(self): """Show the status of terraform init command execution""" - start_time = datetime.now() self.show_step_heading(K.TERRAFORM_REDEPLOY_DESTROY_STARTED, write_log=False) + start_time = datetime.now() while self.destroy is False and self.terraform_thread.isAlive(): - self.show_progress_message(K.TERRAFORM_DESTROY_STARTED, 0.5) - + duration = self.CYAN_ANSI + self.get_duration(datetime.now() - start_time) + self.END_ANSI + message = "Time elapsed: %s" % duration + self.show_progress_message(message, 1.5) end_time = datetime.now() + + self.erase_printed_line() self.show_step_finish(K.TERRAFORM_DESTROY_COMPLETED, write_log=False, color=self.GREEN_ANSI) self.display_process_duration(start_time, end_time) diff --git a/installer/resources/pacbot_app/import_db.py b/installer/resources/pacbot_app/import_db.py index eee387c7a..8b0ae4caf 100644 --- a/installer/resources/pacbot_app/import_db.py +++ b/installer/resources/pacbot_app/import_db.py @@ -14,6 +14,7 @@ from resources.lambda_submit.function import SubmitJobLambdaFunction from resources.lambda_rule_engine.function import RuleEngineLambdaFunction from resources.s3.bucket import BucketStorage +from resources.pacbot_app.utils import need_to_enable_azure from shutil import copy2 import os @@ -29,12 +30,13 @@ def prepare_azure_tenants_credentias(self): tenants = Settings.get('AZURE_TENANTS', []) credential_string = "" - for tenant in tenants: - tenant_id = tenant['tenant'] - client_id = tenant['clientId'] - seccret_id = tenant['secretId'] - credential_string = "" if credential_string == "" else (credential_string + "##") - credential_string += "tenant:%s,clientId:%s,secretId:%s" % (tenant_id, client_id, seccret_id) + if need_to_enable_azure(): + for tenant in tenants: + tenant_id = tenant['tenant'] + client_id = tenant['clientId'] + seccret_id = tenant['secretId'] + credential_string = "" if credential_string == "" else (credential_string + "##") + credential_string += "tenant:%s,clientId:%s,secretId:%s" % (tenant_id, client_id, seccret_id) return credential_string From db8946f634e7e4b217b63652a5e10f59d61e7bb5 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Fri, 8 Nov 2019 17:34:46 +0530 Subject: [PATCH 058/107] destroy message changed for redeployment --- installer/core/constants.py | 3 ++- installer/core/providers/aws/reinstall.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/installer/core/constants.py b/installer/core/constants.py index e9c22cca2..573376371 100644 --- a/installer/core/constants.py +++ b/installer/core/constants.py @@ -93,10 +93,11 @@ TERRAFORM_OUTPUT_STORED = "Terraform output is stored" TERRAFORM_DESTROY_STARTED = "Terraform destroy started" -TERRAFORM_REDEPLOY_DESTROY_STARTED = "Terraform destroy started as part of Redeploy" +TERRAFORM_REDEPLOY_DESTROY_STARTED = "Deleting resources for redeployment" TERRAFORM_DESTROY_RUNNING = "Destroying resources" TERRAFORM_DESTROY_ERROR = "Terraform destroy encountered an error" TERRAFORM_DESTROY_COMPLETED = "Terraform destroy executed successfully!!! Please check destroy log for more details" +TERRAFORM_REDEP_DESTROY_COMPLETED = "Successful!!! Resources will be recreated in next steps." TERRAFORM_DESTROY_DRY_RUN = "Terraform destroy is not executed as dry-run is enabled" TERRAFORM_TAINT_STARTED = "Terraform taint(destroy and re-install) started" diff --git a/installer/core/providers/aws/reinstall.py b/installer/core/providers/aws/reinstall.py index 94867e127..efef7d0c9 100644 --- a/installer/core/providers/aws/reinstall.py +++ b/installer/core/providers/aws/reinstall.py @@ -21,6 +21,7 @@ class ReInstall(Install): # Do not inherit Destroy current_install_status (int): Current install status """ destroy = False + exception = None def execute(self, resources_to_destroy, resources_to_install, terraform_with_targets, dry_run): """ @@ -105,7 +106,10 @@ def render_terraform_destroy_progress(self): message = "Time elapsed: %s" % duration self.show_progress_message(message, 1.5) end_time = datetime.now() - self.erase_printed_line() - self.show_step_finish(K.TERRAFORM_DESTROY_COMPLETED, write_log=False, color=self.GREEN_ANSI) + if self.exception: + self.show_step_finish(K.TERRAFORM_DESTROY_ERROR, write_log=False, color=self.ERROR_ANSI) + else: + self.show_step_finish(K.TERRAFORM_REDEP_DESTROY_COMPLETED, write_log=False, color=self.GREEN_ANSI) + self.display_process_duration(start_time, end_time) From 79e75797249b13d330272cc8ed3ab67b401f73e4 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Mon, 11 Nov 2019 10:15:06 +0530 Subject: [PATCH 059/107] Jobs running frequency reduced to run at every 6 hours --- installer/resources/lambda_rule_engine/utils.py | 4 ++-- installer/resources/lambda_submit/function.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/installer/resources/lambda_rule_engine/utils.py b/installer/resources/lambda_rule_engine/utils.py index 1aad83e23..7d2ada15e 100644 --- a/installer/resources/lambda_rule_engine/utils.py +++ b/installer/resources/lambda_rule_engine/utils.py @@ -19,11 +19,11 @@ def get_rule_engine_cloudwatch_rules_var(): for index in range(len(variable_dict_input)): if variable_dict_input[index]['assetGroup'] == "azure" and not need_to_enable_azure(): continue - mod = index % 20 + 5 + mod = int(index % 20 + 5) item = { 'ruleId': variable_dict_input[index]['ruleUUID'], 'ruleParams': variable_dict_input[index]['ruleParams'], - 'schedule': "cron(%s * * * ? *)" % str(mod) + 'schedule': "cron(%s */6 * * ? *)" % str(mod) } required_rules.append(item) diff --git a/installer/resources/lambda_submit/function.py b/installer/resources/lambda_submit/function.py index 6fab06b43..9774f1826 100644 --- a/installer/resources/lambda_submit/function.py +++ b/installer/resources/lambda_submit/function.py @@ -36,7 +36,7 @@ class SubmitJobLambdaFunction(LambdaFunctionResource): class DataCollectorEventRule(CloudWatchEventRuleResource): name = "AWS-Data-Collector" - schedule_expression = "cron(0 * * * ? *)" + schedule_expression = "cron(0 */6 * * ? *)" DEPENDS_ON = [SubmitJobLambdaFunction] @@ -72,7 +72,7 @@ class DataCollectorCloudWatchEventTarget(CloudWatchEventTargetResource): class DataShipperEventRule(CloudWatchEventRuleResource): name = "aws-redshift-es-data-shipper" - schedule_expression = "cron(0 * * * ? *)" + schedule_expression = "cron(5 */6 * * ? *)" DEPENDS_ON = [SubmitJobLambdaFunction, ESDomainPolicy] @@ -116,7 +116,7 @@ class DataShipperCloudWatchEventTarget(CloudWatchEventTargetResource): class RecommendationsCollectorEventRule(CloudWatchEventRuleResource): name = "AWS-Recommendations-Collector" - schedule_expression = "cron(0 * * * ? *)" + schedule_expression = "cron(6 */6 * * ? *)" DEPENDS_ON = [SubmitJobLambdaFunction] @@ -157,7 +157,7 @@ class RecommendationsCollectorCloudWatchEventTarget(CloudWatchEventTargetResourc class CloudNotificationCollectorEventRule(CloudWatchEventRuleResource): name = "AWS-CloudNotification-Collector" - schedule_expression = "cron(0 * * * ? *)" + schedule_expression = "cron(7 */6 * * ? *)" DEPENDS_ON = [SubmitJobLambdaFunction] @@ -275,7 +275,7 @@ class QualysAssetDataImporterCloudWatchEventTarget(CloudWatchEventTargetResource class AzureDataCollectorEventRule(CloudWatchEventRuleResource): name = "pacbot-azure-discovery" - schedule_expression = "cron(0 * * * ? *)" + schedule_expression = "cron(10 */6 * * ? *)" DEPENDS_ON = [SubmitJobLambdaFunction] PROCESS = need_to_enable_azure() @@ -311,7 +311,7 @@ class AzureDataCollectorCloudWatchEventTarget(CloudWatchEventTargetResource): class AzureDataShipperEventRule(CloudWatchEventRuleResource): name = "data-shipper-azure" - schedule_expression = "cron(0 * * * ? *)" + schedule_expression = "cron(11 */6 * * ? *)" DEPENDS_ON = [SubmitJobLambdaFunction, ESDomainPolicy] PROCESS = need_to_enable_azure() From 2f2db40a028b4a78e1111f39ad546e822d6abffd Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Mon, 11 Nov 2019 11:16:30 +0530 Subject: [PATCH 060/107] rule name changed for azure data collector --- installer/resources/lambda_submit/function.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/resources/lambda_submit/function.py b/installer/resources/lambda_submit/function.py index 9774f1826..2dd2486a0 100644 --- a/installer/resources/lambda_submit/function.py +++ b/installer/resources/lambda_submit/function.py @@ -274,7 +274,7 @@ class QualysAssetDataImporterCloudWatchEventTarget(CloudWatchEventTargetResource class AzureDataCollectorEventRule(CloudWatchEventRuleResource): - name = "pacbot-azure-discovery" + name = "azure-discovery" schedule_expression = "cron(10 */6 * * ? *)" DEPENDS_ON = [SubmitJobLambdaFunction] PROCESS = need_to_enable_azure() From 616af4e3365f3ed990cf2d340c05b2b30a9a1696 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Mon, 11 Nov 2019 18:30:43 +0530 Subject: [PATCH 061/107] Instance type made optimal for batch jobs --- installer/resources/batch/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/resources/batch/env.py b/installer/resources/batch/env.py index 7cf9366de..1a46a9ed6 100644 --- a/installer/resources/batch/env.py +++ b/installer/resources/batch/env.py @@ -15,7 +15,7 @@ class RuleEngineBatchJobEnv(BatchComputeEnvironmentResource): compute_environment_name = "" instance_role = ECSRoleInstanceProfile.get_output_attr('arn') - instance_type = ["m4.xlarge"] + instance_type = ["optimal"] max_vcpus = 256 min_vcpus = 0 desired_vcpus = 0 From f1b583f3c5ad4021b440cb065b8ac1901f37f694 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Mon, 11 Nov 2019 18:52:31 +0530 Subject: [PATCH 062/107] reverted back from optimal to m4 large --- installer/resources/batch/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/resources/batch/env.py b/installer/resources/batch/env.py index 1a46a9ed6..7cf9366de 100644 --- a/installer/resources/batch/env.py +++ b/installer/resources/batch/env.py @@ -15,7 +15,7 @@ class RuleEngineBatchJobEnv(BatchComputeEnvironmentResource): compute_environment_name = "" instance_role = ECSRoleInstanceProfile.get_output_attr('arn') - instance_type = ["optimal"] + instance_type = ["m4.xlarge"] max_vcpus = 256 min_vcpus = 0 desired_vcpus = 0 From a447e1aef97dce985c65076e5e497d0746a4464d Mon Sep 17 00:00:00 2001 From: johnrexj Date: Mon, 11 Nov 2019 20:26:38 +0530 Subject: [PATCH 063/107] Modified jar name for azure discovery --- jobs/azure-discovery/pom.xml | 18 ++++++++++++++++++ jobs/pom.xml | 1 + 2 files changed, 19 insertions(+) diff --git a/jobs/azure-discovery/pom.xml b/jobs/azure-discovery/pom.xml index 5d9d25ead..439458bec 100644 --- a/jobs/azure-discovery/pom.xml +++ b/jobs/azure-discovery/pom.xml @@ -67,6 +67,24 @@ + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + install + + + + + + + run + + + + org.jacoco jacoco-maven-plugin diff --git a/jobs/pom.xml b/jobs/pom.xml index af114c155..c483aa98d 100644 --- a/jobs/pom.xml +++ b/jobs/pom.xml @@ -48,6 +48,7 @@ pacman-cloud-notifications recommendation-enricher pacman-qualys-enricher + pacman-azure-discovery From 8d817e3565c150e23632ff521ef71a2402080b7a Mon Sep 17 00:00:00 2001 From: Anil Chandran Date: Mon, 11 Nov 2019 16:20:36 -0800 Subject: [PATCH 064/107] Update pom.xml --- jobs/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/pom.xml b/jobs/pom.xml index c483aa98d..709b7d6af 100644 --- a/jobs/pom.xml +++ b/jobs/pom.xml @@ -48,7 +48,7 @@ pacman-cloud-notifications recommendation-enricher pacman-qualys-enricher - pacman-azure-discovery + azure-discovery From 7df43a4f0e88a472cd6d77d047ed64e612768024 Mon Sep 17 00:00:00 2001 From: Sajeer Date: Tue, 12 Nov 2019 10:59:09 +0530 Subject: [PATCH 065/107] Batch instance type made configurable --- installer/resources/batch/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/resources/batch/env.py b/installer/resources/batch/env.py index 7cf9366de..356d45ce5 100644 --- a/installer/resources/batch/env.py +++ b/installer/resources/batch/env.py @@ -15,7 +15,7 @@ class RuleEngineBatchJobEnv(BatchComputeEnvironmentResource): compute_environment_name = "" instance_role = ECSRoleInstanceProfile.get_output_attr('arn') - instance_type = ["m4.xlarge"] + instance_type = [Settings.get('BATCH_INSTANCE_TYPE', "m4.xlarge")] max_vcpus = 256 min_vcpus = 0 desired_vcpus = 0 From 7aa5a22ce2b92a6ceaa2ad3e1343babfe5eb57ba Mon Sep 17 00:00:00 2001 From: Sajeer Date: Tue, 12 Nov 2019 12:59:00 +0530 Subject: [PATCH 066/107] Added config creds parameter to azure batch job --- installer/resources/lambda_submit/function.py | 1 + 1 file changed, 1 insertion(+) diff --git a/installer/resources/lambda_submit/function.py b/installer/resources/lambda_submit/function.py index 2dd2486a0..936705506 100644 --- a/installer/resources/lambda_submit/function.py +++ b/installer/resources/lambda_submit/function.py @@ -304,6 +304,7 @@ class AzureDataCollectorCloudWatchEventTarget(CloudWatchEventTargetResource): 'params': [ {'encrypt': False, 'key': "package_hint", 'value': "com.tmobile.pacbot"}, {'encrypt': False, 'key': "file.path", 'value': "/home/ec2-user/azure-data"}, + {'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="}, ] }) PROCESS = need_to_enable_azure() From 36aba45baaaed25e5b4fb8cecb9568f84c79692f Mon Sep 17 00:00:00 2001 From: Sajeer Date: Tue, 12 Nov 2019 13:08:48 +0530 Subject: [PATCH 067/107] Silent install option is passed to redeploy command so that no input should be read --- installer/custom/commands/redeploy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/installer/custom/commands/redeploy.py b/installer/custom/commands/redeploy.py index 1755bdfc7..9d51e203a 100644 --- a/installer/custom/commands/redeploy.py +++ b/installer/custom/commands/redeploy.py @@ -50,6 +50,7 @@ def __init__(self, args): self.need_complete_install = self._need_complete_installation() self.dry_run = True if any([x[1] for x in args if x[0] == "dry-run"]) else self.dry_run + self.silent_install = True if any([x[1] for x in args if x[0] == "silent"]) else self.silent_install def _need_complete_installation(self): """ From c1a02ace48d29ca6681892059c9fc1ec7826ac70 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Wed, 13 Nov 2019 10:23:29 +0530 Subject: [PATCH 068/107] tenant values are also passed to job parameter --- installer/resources/lambda_submit/function.py | 3 ++- installer/resources/pacbot_app/import_db.py | 2 +- installer/resources/pacbot_app/utils.py | 9 +++++++++ installer/settings/default.local.py | 6 +++--- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/installer/resources/lambda_submit/function.py b/installer/resources/lambda_submit/function.py index 936705506..566828b52 100644 --- a/installer/resources/lambda_submit/function.py +++ b/installer/resources/lambda_submit/function.py @@ -10,7 +10,7 @@ from resources.data.aws_info import AwsAccount, AwsRegion from resources.lambda_submit.s3_upload import UploadLambdaSubmitJobZipFile, BATCH_JOB_FILE_NAME from resources.pacbot_app.alb import ApplicationLoadBalancer -from resources.pacbot_app.utils import need_to_deploy_vulnerability_service, need_to_enable_azure +from resources.pacbot_app.utils import need_to_deploy_vulnerability_service, need_to_enable_azure, get_azure_tenants import json @@ -305,6 +305,7 @@ class AzureDataCollectorCloudWatchEventTarget(CloudWatchEventTargetResource): {'encrypt': False, 'key': "package_hint", 'value': "com.tmobile.pacbot"}, {'encrypt': False, 'key': "file.path", 'value': "/home/ec2-user/azure-data"}, {'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="}, + {'encrypt': False, 'key': "tenants", 'value': get_azure_tenants()} ] }) PROCESS = need_to_enable_azure() diff --git a/installer/resources/pacbot_app/import_db.py b/installer/resources/pacbot_app/import_db.py index 8b0ae4caf..44ad65cf4 100644 --- a/installer/resources/pacbot_app/import_db.py +++ b/installer/resources/pacbot_app/import_db.py @@ -32,7 +32,7 @@ def prepare_azure_tenants_credentias(self): if need_to_enable_azure(): for tenant in tenants: - tenant_id = tenant['tenant'] + tenant_id = tenant['tenantId'] client_id = tenant['clientId'] seccret_id = tenant['secretId'] credential_string = "" if credential_string == "" else (credential_string + "##") diff --git a/installer/resources/pacbot_app/utils.py b/installer/resources/pacbot_app/utils.py index 6ce370741..3ec68955f 100644 --- a/installer/resources/pacbot_app/utils.py +++ b/installer/resources/pacbot_app/utils.py @@ -11,3 +11,12 @@ def need_to_enable_azure(): feature_status = Settings.get('ENABLE_AZURE', False) return feature_status + +def get_azure_tenants(): + if need_to_enable_azure(): + tenants = Settings.get('AZURE_TENANTS', []) + tenant_ids = [tenant['tenantId'] for tenant in tenants] + + return ",".join(tenant_ids) + else: + return "" diff --git a/installer/settings/default.local.py b/installer/settings/default.local.py index 1e59d202c..1d15c9cf3 100644 --- a/installer/settings/default.local.py +++ b/installer/settings/default.local.py @@ -58,15 +58,15 @@ # This settings enable Vulnerability feature and servie ENABLE_AZURE = False -# Tenants should be a list of dict containing tenant, clientId and secretId +# Tenants should be a list of dict containing tenantId, clientId and secretId AZURE_TENANTS = [ { - 'tenant': "t111", + 'tenantId': "t111", 'clientId': "c111", 'secretId': "s111" }, { - 'tenant': "t222", + 'tenantId': "t222", 'clientId': "c222", 'secretId': "s222" }, From 71c12e207c38d94afd2491d7192f11adcd0fc4ea Mon Sep 17 00:00:00 2001 From: Kanchana Date: Wed, 13 Nov 2019 11:47:33 +0530 Subject: [PATCH 069/107] Sdk version upgraded --- commons/pac-batch-commons/pom.xml | 3 +-- jobs/pacman-cloud-discovery/pom.xml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/commons/pac-batch-commons/pom.xml b/commons/pac-batch-commons/pom.xml index 7f2cae5e5..c4c61568d 100644 --- a/commons/pac-batch-commons/pom.xml +++ b/commons/pac-batch-commons/pom.xml @@ -24,7 +24,7 @@ com.amazonaws aws-java-sdk-bom - 1.11.432 + 1.11.636 pom import @@ -100,7 +100,6 @@ com.amazonaws aws-java-sdk-s3 - 1.11.636 diff --git a/jobs/pacman-cloud-discovery/pom.xml b/jobs/pacman-cloud-discovery/pom.xml index c1aec841b..bdb0161f4 100644 --- a/jobs/pacman-cloud-discovery/pom.xml +++ b/jobs/pacman-cloud-discovery/pom.xml @@ -24,8 +24,7 @@ com.amazonaws aws-java-sdk - - 1.11.432 + 1.11.636 org.powermock From 1dba69f2d3ff85a761b84faf7ff58175d42c5b0e Mon Sep 17 00:00:00 2001 From: Kanchana Date: Wed, 13 Nov 2019 14:38:01 +0530 Subject: [PATCH 070/107] Compliance issue resolved --- .../api/compliance/service/ComplianceServiceImpl.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/ComplianceServiceImpl.java b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/ComplianceServiceImpl.java index edeb9f742..03d433219 100644 --- a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/ComplianceServiceImpl.java +++ b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/service/ComplianceServiceImpl.java @@ -878,19 +878,18 @@ public List> getRuleSevCatDetails(List> ruleParamsJson = (JsonObject) parser.parse(ruleDetail.get(RULE_PARAMS).toString()); paramsList = new Gson().fromJson(ruleParamsJson.get(PARAMS), new TypeToken>() { }.getType()); - + ruleSevCatDetail.put(RULEID, ruleDetail.get(RULEID)); + ruleSevCatDetail.put("autofix", ruleParamsJson.get("autofix").getAsBoolean()); + ruleSevCatDetail.put("targetType", ruleDetail.get("targetType")); + ruleSevCatDetail.put(DISPLAY_NAME, ruleDetail.get(DISPLAY_NAME)); for (Map param : paramsList) { - ruleSevCatDetail.put(RULEID, ruleDetail.get(RULEID)); if (param.get(KEY).equalsIgnoreCase(RULE_CATEGORY)) { - ruleSevCatDetail.put(RULE_CATEGORY, param.get(VALUE)); - } else if (param.get(KEY).equalsIgnoreCase(SEVERITY)) { ruleSevCatDetail.put(SEVERITY, param.get(VALUE)); - } - ruleSevCatDetails.add(ruleSevCatDetail); } + ruleSevCatDetails.add(ruleSevCatDetail); } return ruleSevCatDetails; From 655edb1413680ae8cdbc48865665330c4a816fca Mon Sep 17 00:00:00 2001 From: Anil Chandran Date: Wed, 13 Nov 2019 16:16:30 -0800 Subject: [PATCH 071/107] Update DB.sql Added azure to cf_Datasource table --- installer/resources/pacbot_app/files/DB.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/resources/pacbot_app/files/DB.sql b/installer/resources/pacbot_app/files/DB.sql index c028e796e..844a2221d 100644 --- a/installer/resources/pacbot_app/files/DB.sql +++ b/installer/resources/pacbot_app/files/DB.sql @@ -933,7 +933,7 @@ INSERT IGNORE INTO `task`(`id`,`index`,`mappings`,`data`) values (1,'exceptions' /*Insert Data Source to necessary tables*/ INSERT IGNORE INTO `cf_Datasource`(`dataSourceId`,`dataSourceName`,`dataSourceDesc`,`config`,`createdDate`,`modifiedDate`) VALUES (1,'aws','Amazon WebService','N/A','2017-08-01','2018-03-09'); - +INSERT IGNORE INTO `cf_Datasource` (dataSourceId,dataSourceName,dataSourceDesc,config,createdDate,modifiedDate) VALUES (2,'azure','Azure','N/A',{d '2019-11-13'},{d '2019-11-13'}) /*Insert Data Asset Group to necessary tables*/ INSERT IGNORE INTO cf_AssetGroupDetails (groupId,groupName,dataSource,displayName,groupType,createdBy,createdUser,createdDate,modifiedUser,modifiedDate,description,aliasQuery,isVisible) VALUES ('201','aws','aws','aws all','admin','Cloud Security','','','pacman','03/26/2018 23:00','Asset Group to segregate all data related to aws.','',true); From dd4cf639b109995965aa3c46868befe3c6dad916 Mon Sep 17 00:00:00 2001 From: johnrexj Date: Thu, 14 Nov 2019 11:32:53 +0530 Subject: [PATCH 072/107] asset group creation fix for azure --- .../admin/repository/service/AssetGroupServiceImpl.java | 4 ++-- .../api/admin/repository/service/DatasourceService.java | 6 ++++-- .../admin/repository/service/DatasourceServiceImpl.java | 7 ++++--- .../api/admin/controller/DatasourceControllerTest.java | 8 ++------ .../repository/service/DatasourceServiceImplTest.java | 8 ++------ .../modules/admin/create-rule/create-rule.component.ts | 3 +-- 6 files changed, 15 insertions(+), 21 deletions(-) diff --git a/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/service/AssetGroupServiceImpl.java b/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/service/AssetGroupServiceImpl.java index 572853e8a..3c081a18b 100644 --- a/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/service/AssetGroupServiceImpl.java +++ b/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/service/AssetGroupServiceImpl.java @@ -304,7 +304,7 @@ private UpdateAssetGroupDetails buildAssetGroupDetails(final AssetGroupDetails e targetTypesIndex.put(targetTypeDetails.getTargetType(), idx[0]); targetTypes.setAdded(true); targetTypes.setTargetName(targetTypeDetails.getTargetType()); - targetTypes.setAllAttributesName(commonService.getFieldNames(existingAssetGroupDetails.getDataSource() + "_" + targetTypeDetails.getTargetType(), targetTypeDetails.getTargetType())); + targetTypes.setAllAttributesName(commonService.getFieldNames(targetTypesRepository.findDataSourceByTargetType(targetTypeDetails.getTargetType()) + "_" + targetTypeDetails.getTargetType(), targetTypeDetails.getTargetType())); if(targetTypeDetails.getAttributeName().equalsIgnoreCase("all") && targetTypeDetails.getAttributeValue().equalsIgnoreCase("all")) { targetTypes.setIncludeAll(true); targetTypes.setAttributes(Lists.newArrayList()); @@ -335,7 +335,7 @@ private UpdateAssetGroupDetails buildAssetGroupDetails(final AssetGroupDetails e TargetTypesDetails targetTypeAttribute = new TargetTypesDetails(); targetTypeAttribute.setAttributes(Lists.newArrayList()); targetTypeAttribute.setTargetName(targetName.trim()); - targetTypeAttribute.setAllAttributesName(commonService.getFieldNames(existingAssetGroupDetails.getDataSource() + "_" + targetName, targetName)); + targetTypeAttribute.setAllAttributesName(commonService.getFieldNames(targetTypesRepository.findDataSourceByTargetType(targetName) + "_" + targetName, targetName)); targetTypeAttribute.setIncludeAll(false); attributes.add(targetTypeAttribute); } diff --git a/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/service/DatasourceService.java b/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/service/DatasourceService.java index 4bec3f28e..b2a8411d2 100644 --- a/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/service/DatasourceService.java +++ b/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/service/DatasourceService.java @@ -15,7 +15,9 @@ ******************************************************************************/ package com.tmobile.pacman.api.admin.repository.service; -import java.util.Collection; +import java.util.List; + +import com.tmobile.pacman.api.admin.repository.model.Datasource; /** * DataSource Service Functionalities @@ -28,5 +30,5 @@ public interface DatasourceService { * @author Nidhish * @return All dataSource details list */ - public Collection getAllDatasourceDetails(); + public List getAllDatasourceDetails(); } diff --git a/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/service/DatasourceServiceImpl.java b/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/service/DatasourceServiceImpl.java index 375f16e14..75b8eb927 100644 --- a/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/service/DatasourceServiceImpl.java +++ b/api/pacman-api-admin/src/main/java/com/tmobile/pacman/api/admin/repository/service/DatasourceServiceImpl.java @@ -15,12 +15,13 @@ ******************************************************************************/ package com.tmobile.pacman.api.admin.repository.service; -import java.util.Collection; +import java.util.List; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.tmobile.pacman.api.admin.repository.DatasourceRepository; +import com.tmobile.pacman.api.admin.repository.model.Datasource; import com.tmobile.pacman.api.commons.Constants; /** @@ -33,7 +34,7 @@ public class DatasourceServiceImpl implements DatasourceService, Constants { private DatasourceRepository datasourceRepository; @Override - public Collection getAllDatasourceDetails() { - return datasourceRepository.getAllDatasourceDetails(); + public List getAllDatasourceDetails() { + return datasourceRepository.findAll(); } } diff --git a/api/pacman-api-admin/src/test/java/com/tmobile/pacman/api/admin/controller/DatasourceControllerTest.java b/api/pacman-api-admin/src/test/java/com/tmobile/pacman/api/admin/controller/DatasourceControllerTest.java index f8ea617bd..7bcaa03b1 100644 --- a/api/pacman-api-admin/src/test/java/com/tmobile/pacman/api/admin/controller/DatasourceControllerTest.java +++ b/api/pacman-api-admin/src/test/java/com/tmobile/pacman/api/admin/controller/DatasourceControllerTest.java @@ -25,7 +25,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import java.util.ArrayList; -import java.util.Collection; import org.junit.Before; import org.junit.Test; @@ -60,13 +59,10 @@ public void init() { @Test public void getAllDatasourceDetailsTest() throws Exception { - Collection allDatasources = new ArrayList(); - Object[] datasources = { "aws", "azure" }; - allDatasources.add(datasources); - when(datasourceService.getAllDatasourceDetails()).thenReturn(allDatasources); + when(datasourceService.getAllDatasourceDetails()).thenReturn(new ArrayList<>()); mockMvc.perform(get("/datasource/list")).andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8_VALUE)) - .andExpect(jsonPath("$.data", hasSize(1))); + .andExpect(jsonPath("$.data", hasSize(0))); } @Test diff --git a/api/pacman-api-admin/src/test/java/com/tmobile/pacman/api/admin/repository/service/DatasourceServiceImplTest.java b/api/pacman-api-admin/src/test/java/com/tmobile/pacman/api/admin/repository/service/DatasourceServiceImplTest.java index 03e8aa081..3e9003178 100644 --- a/api/pacman-api-admin/src/test/java/com/tmobile/pacman/api/admin/repository/service/DatasourceServiceImplTest.java +++ b/api/pacman-api-admin/src/test/java/com/tmobile/pacman/api/admin/repository/service/DatasourceServiceImplTest.java @@ -20,7 +20,6 @@ import static org.mockito.Mockito.when; import java.util.ArrayList; -import java.util.Collection; import org.junit.Test; import org.junit.runner.RunWith; @@ -41,10 +40,7 @@ public class DatasourceServiceImplTest { @Test public void getAllDatasourceDetailsTest() throws Exception { - Collection allDatasources = new ArrayList(); - Object[] datasources = { "aws", "azure" }; - allDatasources.add(datasources); - when(datasourceService.getAllDatasourceDetails()).thenReturn(allDatasources); - assertThat(datasourceRepository.getAllDatasourceDetails().size(), is(1)); + when(datasourceRepository.findAll()).thenReturn(new ArrayList<>()); + assertThat(datasourceService.getAllDatasourceDetails().size(), is(0)); } } diff --git a/webapp/src/app/pacman-features/modules/admin/create-rule/create-rule.component.ts b/webapp/src/app/pacman-features/modules/admin/create-rule/create-rule.component.ts index 4d22174b1..3b805093a 100644 --- a/webapp/src/app/pacman-features/modules/admin/create-rule/create-rule.component.ts +++ b/webapp/src/app/pacman-features/modules/admin/create-rule/create-rule.component.ts @@ -246,8 +246,7 @@ export class CreateRuleComponent implements OnInit, OnDestroy { this.adminService.executeHttpAction(url, method, {}, {}).subscribe(reponse => { const fullDatasourceNames = []; for (let index = 0; index < reponse[0].length; index++) { - const datasourceDetail = reponse[0][index]; - fullDatasourceNames.push(datasourceDetail[0]); + fullDatasourceNames.push(reponse[0][index].dataSourceName); } this.datasourceDetails = fullDatasourceNames; this.getAllAssetGroupNames(); From 2aaa3c834e671eefcd2a1af91f578dcf1fb072eb Mon Sep 17 00:00:00 2001 From: Anil Chandran Date: Wed, 13 Nov 2019 22:35:29 -0800 Subject: [PATCH 073/107] Update DB.sql --- installer/resources/pacbot_app/files/DB.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/resources/pacbot_app/files/DB.sql b/installer/resources/pacbot_app/files/DB.sql index 844a2221d..1d92e88d7 100644 --- a/installer/resources/pacbot_app/files/DB.sql +++ b/installer/resources/pacbot_app/files/DB.sql @@ -933,7 +933,7 @@ INSERT IGNORE INTO `task`(`id`,`index`,`mappings`,`data`) values (1,'exceptions' /*Insert Data Source to necessary tables*/ INSERT IGNORE INTO `cf_Datasource`(`dataSourceId`,`dataSourceName`,`dataSourceDesc`,`config`,`createdDate`,`modifiedDate`) VALUES (1,'aws','Amazon WebService','N/A','2017-08-01','2018-03-09'); -INSERT IGNORE INTO `cf_Datasource` (dataSourceId,dataSourceName,dataSourceDesc,config,createdDate,modifiedDate) VALUES (2,'azure','Azure','N/A',{d '2019-11-13'},{d '2019-11-13'}) +INSERT IGNORE INTO `cf_Datasource` (dataSourceId,dataSourceName,dataSourceDesc,config,createdDate,modifiedDate) VALUES (2,'azure','Azure','N/A',{d '2019-11-13'},{d '2019-11-13'}); /*Insert Data Asset Group to necessary tables*/ INSERT IGNORE INTO cf_AssetGroupDetails (groupId,groupName,dataSource,displayName,groupType,createdBy,createdUser,createdDate,modifiedUser,modifiedDate,description,aliasQuery,isVisible) VALUES ('201','aws','aws','aws all','admin','Cloud Security','','','pacman','03/26/2018 23:00','Asset Group to segregate all data related to aws.','',true); From b2030e5dc93b85b3f54073c1923fa27727090d17 Mon Sep 17 00:00:00 2001 From: Kanchana Date: Thu, 14 Nov 2019 17:00:03 +0530 Subject: [PATCH 074/107] Added star in RE while getting exceptions --- .../java/com/tmobile/pacman/service/ExceptionManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/pacman-rule-engine-2.0/src/main/java/com/tmobile/pacman/service/ExceptionManagerImpl.java b/jobs/pacman-rule-engine-2.0/src/main/java/com/tmobile/pacman/service/ExceptionManagerImpl.java index f8b7bd1ec..25b99a1c2 100644 --- a/jobs/pacman-rule-engine-2.0/src/main/java/com/tmobile/pacman/service/ExceptionManagerImpl.java +++ b/jobs/pacman-rule-engine-2.0/src/main/java/com/tmobile/pacman/service/ExceptionManagerImpl.java @@ -64,7 +64,7 @@ public class ExceptionManagerImpl implements ExceptionManager { */ @Override public Map getIndividualExceptions(String resourceType) throws Exception { - String indexName = "aws_" + resourceType; + String indexName = "*_" + resourceType; String type = "issue_" + resourceType + "_exception"; Map mustFilter = new HashMap<>(); //mustFilter.put("exemptedStatus", "active"); From 7bbe6b9e4890d2ca6a4e47153e299ba44c3cd830 Mon Sep 17 00:00:00 2001 From: Kanchana Date: Thu, 14 Nov 2019 18:28:09 +0530 Subject: [PATCH 075/107] issue id getting by match query --- .../repository/ComplianceRepositoryImpl.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/repository/ComplianceRepositoryImpl.java b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/repository/ComplianceRepositoryImpl.java index 436d29ee1..e6defa785 100644 --- a/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/repository/ComplianceRepositoryImpl.java +++ b/api/pacman-api-compliance/src/main/java/com/tmobile/pacman/api/compliance/repository/ComplianceRepositoryImpl.java @@ -1723,12 +1723,13 @@ public Map getTaggingByAG(String assetGroup, String targetTypes, @SuppressWarnings("rawtypes") public Map getPolicyViolationDetailsByIssueId(String assetGroup, String issueId) throws DataException { - Map issueDetails = new HashMap<>(); - StringBuilder urlToQuery = new StringBuilder(esUrl).append("/").append(assetGroup); - urlToQuery.append("/").append(SEARCH).append("?").append("q").append("=").append("_id").append(":") - .append(issueId); + Map issueDetails = new HashMap<>(); + StringBuilder urlToQuery = new StringBuilder(esUrl).append("/").append(assetGroup); + urlToQuery.append("/").append(SEARCH); - StringBuilder requestBody = new StringBuilder(); + StringBuilder requestBody = new StringBuilder(); + requestBody.append("{\"query\":{\"bool\":{\"must\":[{\"match\":{\"type\":\"issue\"}},{\"match\":{\"_id\":\"") + .append(issueId).append("\"}}]}}}"); String responseJson = ""; try { responseJson = PacHttpUtils.doHttpPost(urlToQuery.toString(), requestBody.toString()); From e336ed5c7f21a073fe58c62ffb7f9322ed8b7dc4 Mon Sep 17 00:00:00 2001 From: johnrexj Date: Thu, 14 Nov 2019 20:31:21 +0530 Subject: [PATCH 076/107] recommendation fix for azure --- .../repository/RecommendationsRepository.java | 59 +++++++++++++------ 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/repository/RecommendationsRepository.java b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/repository/RecommendationsRepository.java index b17675024..0390d29ff 100644 --- a/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/repository/RecommendationsRepository.java +++ b/api/pacman-api-asset/src/main/java/com/tmobile/pacman/api/asset/repository/RecommendationsRepository.java @@ -38,6 +38,9 @@ public class RecommendationsRepository { private static final String PROTOCOL = "http"; private String esUrl; + @Value("${recommendation.categories}") + private String recommendationCategories; + /** The elastic search repository. */ @Autowired private ElasticSearchRepository elasticSearchRepository; @@ -69,31 +72,51 @@ public List> getRecommendationSummary(String assetGroup, Stri try { responseDetails = PacHttpUtils.doHttpPost(urlToQuery.toString(), requestBody.toString()); } catch (Exception e) { - LOGGER.error("Error in getRecommendationSummary "+e); + LOGGER.error("Error while fetching recommendation summary from ES ", e); throw new DataException(e); } JsonParser parser = new JsonParser(); JsonObject responseDetailsjson = parser.parse(responseDetails).getAsJsonObject(); JsonObject aggregations = responseDetailsjson.get(Constants.AGGREGATIONS).getAsJsonObject(); - JsonArray categoryBuckets = aggregations.get("recommendations").getAsJsonObject().get("latest").getAsJsonObject().get("category").getAsJsonObject().get(Constants.BUCKETS).getAsJsonArray(); - if (categoryBuckets.size() > 0) { - for (int i=0; i category = new HashMap<>(); - category.put("category", categoryObj.get("key").getAsString()); - category.put("recommendations", categoryObj.get("doc_count").getAsLong()); - JsonObject savingsObj = categoryObj.get("savings").getAsJsonObject(); - if(savingsObj.size() != 0) { - long potentialMonthlySavings = Math.round(savingsObj.get("value").getAsDouble()); - if(potentialMonthlySavings > 0) { - category.put("potentialMonthlySavings", potentialMonthlySavings); - } - } - recommendationSummary.add(category); + boolean dataAvailable = false; + if(aggregations != null) { + JsonObject recommendations = aggregations.get("recommendations").getAsJsonObject(); + if(recommendations.has("latest") && recommendations.get("latest").getAsJsonObject().has("category")) { + JsonArray categoryBuckets = recommendations.get("latest").getAsJsonObject().get("category") + .getAsJsonObject().get(Constants.BUCKETS).getAsJsonArray(); + if (categoryBuckets.size() > 0) { + dataAvailable = true; + for (int i=0; i category = new HashMap<>(); + category.put("category", categoryObj.get("key").getAsString()); + category.put("recommendations", categoryObj.get("doc_count").getAsLong()); + JsonObject savingsObj = categoryObj.get("savings").getAsJsonObject(); + if(savingsObj.size() != 0) { + long potentialMonthlySavings = Math.round(savingsObj.get("value").getAsDouble()); + if(potentialMonthlySavings > 0) { + category.put("potentialMonthlySavings", potentialMonthlySavings); + } + } + recommendationSummary.add(category); + } + } } - } + } + } + if(!dataAvailable) { + //for azure there are no recommendations currently, so passing 0 values for azure asset group + String[] categories = recommendationCategories.split(","); + for(int i=0; i < categories.length; i++) { + Map category = new HashMap<>(); + category.put("category", categories[i]); + category.put("recommendations", 0L); + recommendationSummary.add(category); + } + } + return recommendationSummary; } From e1d76da74431299ab2224e028b68ff7298c56500 Mon Sep 17 00:00:00 2001 From: johnrexj Date: Thu, 14 Nov 2019 20:39:45 +0530 Subject: [PATCH 077/107] Adding recommendation category script --- installer/resources/pacbot_app/files/DB.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/installer/resources/pacbot_app/files/DB.sql b/installer/resources/pacbot_app/files/DB.sql index 1d92e88d7..1697e5b1d 100644 --- a/installer/resources/pacbot_app/files/DB.sql +++ b/installer/resources/pacbot_app/files/DB.sql @@ -2591,3 +2591,5 @@ INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSou INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('virtualmachine','Azure Virtual Machines','Compute','azure','{\"key\":\"vmId\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_virtualmachine/virtualmachine'),'2019-11-05','2019-11-05','Infra & Platforms'); INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('vnet','Azure Disk','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_vnet/vnet'),'2019-11-05','2019-11-05','Infra & Platforms'); INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('workflows','Azure workflows','Internet of things','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_workflows/workflows'),'2019-09-19','2019-09-19','Infra & Platforms'); + +INSERT IGNORE INTO `pac_config_properties` (`cfkey`, `value`, `application`, `profile`, `label`, `createdBy`, `createdDate`, `modifiedBy`, `modifiedDate`) values('recommendation.categories','fault_tolerance, cost_optimizing, security, performance','application','prd','latest','santoshjayakar.diyyala1@t-mobile.com','09/06/2019 06:07:43','',''); From 5707205a06c7dbe7b1627ff0a99112cebfa664c6 Mon Sep 17 00:00:00 2001 From: johnrexj Date: Fri, 15 Nov 2019 10:26:35 +0530 Subject: [PATCH 078/107] updated the config --- installer/resources/pacbot_app/files/DB.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/resources/pacbot_app/files/DB.sql b/installer/resources/pacbot_app/files/DB.sql index 1697e5b1d..289807e85 100644 --- a/installer/resources/pacbot_app/files/DB.sql +++ b/installer/resources/pacbot_app/files/DB.sql @@ -2592,4 +2592,4 @@ INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSou INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('vnet','Azure Disk','Networking','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled',NULL,concat(@eshost,':',@esport,'/azure_vnet/vnet'),'2019-11-05','2019-11-05','Infra & Platforms'); INSERT IGNORE INTO `cf_Target` (`targetName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('workflows','Azure workflows','Internet of things','azure','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/azure_workflows/workflows'),'2019-09-19','2019-09-19','Infra & Platforms'); -INSERT IGNORE INTO `pac_config_properties` (`cfkey`, `value`, `application`, `profile`, `label`, `createdBy`, `createdDate`, `modifiedBy`, `modifiedDate`) values('recommendation.categories','fault_tolerance, cost_optimizing, security, performance','application','prd','latest','santoshjayakar.diyyala1@t-mobile.com','09/06/2019 06:07:43','',''); +INSERT IGNORE INTO `pac_config_properties` (`cfkey`, `value`, `application`, `profile`, `label`, `createdBy`, `createdDate`, `modifiedBy`, `modifiedDate`) values('recommendation.categories','fault_tolerance, cost_optimizing, security, performance','application','prd','latest','admin@pacbot.org','09/06/2019 06:07:43','',''); From 609cb315d20f272151a50b57ae725d7cbd4b084a Mon Sep 17 00:00:00 2001 From: Kanchana Date: Fri, 15 Nov 2019 17:24:45 +0530 Subject: [PATCH 079/107] Azure ag is added & targetTypes added --- installer/resources/pacbot_app/files/DB.sql | 34 +++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/installer/resources/pacbot_app/files/DB.sql b/installer/resources/pacbot_app/files/DB.sql index 289807e85..110f77ed0 100644 --- a/installer/resources/pacbot_app/files/DB.sql +++ b/installer/resources/pacbot_app/files/DB.sql @@ -937,6 +937,7 @@ INSERT IGNORE INTO `cf_Datasource` (dataSourceId,dataSourceName,dataSourceDesc,c /*Insert Data Asset Group to necessary tables*/ INSERT IGNORE INTO cf_AssetGroupDetails (groupId,groupName,dataSource,displayName,groupType,createdBy,createdUser,createdDate,modifiedUser,modifiedDate,description,aliasQuery,isVisible) VALUES ('201','aws','aws','aws all','admin','Cloud Security','','','pacman','03/26/2018 23:00','Asset Group to segregate all data related to aws.','',true); +INSERT IGNORE INTO `cf_AssetGroupDetails` (`groupId`, `groupName`, `dataSource`, `displayName`, `groupType`, `createdBy`, `createdUser`, `createdDate`, `modifiedUser`, `modifiedDate`, `description`, `aliasQuery`, `isVisible`) values('cdffb9cd-71de-4e29-9cae-783c2aa211ac','azure','aws','Azure','Admin','Sree','admin@pacbot.org','11/13/2019 10:43','admin@pacbot.org','11/15/2019 11:13','All Azure','{\"actions\":[{\"add\":{\"filter\":{\"bool\":{\"should\":[{\"has_parent\":{\"query\":{\"match_all\":{}},\"parent_type\":\"blobcontainer\"}},{\"bool\":{\"must\":[{\"term\":{\"_type\":{\"value\":\"blobcontainer\"}}}]}}]}},\"index\":\"azure_blobcontainer\",\"alias\":\"azure\"}},{\"add\":{\"filter\":{\"bool\":{\"should\":[{\"has_parent\":{\"query\":{\"match_all\":{}},\"parent_type\":\"workflows\"}},{\"bool\":{\"must\":[{\"term\":{\"_type\":{\"value\":\"workflows\"}}}]}}]}},\"index\":\"azure_workflows\",\"alias\":\"azure\"}},{\"add\":{\"filter\":{\"bool\":{\"should\":[{\"has_parent\":{\"query\":{\"match_all\":{}},\"parent_type\":\"virtualmachine\"}},{\"bool\":{\"must\":[{\"term\":{\"_type\":{\"value\":\"virtualmachine\"}}}]}}]}},\"index\":\"azure_virtualmachine\",\"alias\":\"azure\"}},{\"add\":{\"filter\":{\"bool\":{\"should\":[{\"has_parent\":{\"query\":{\"match_all\":{}},\"parent_type\":\"cosmosdb\"}},{\"bool\":{\"must\":[{\"term\":{\"_type\":{\"value\":\"cosmosdb\"}}}]}}]}},\"index\":\"azure_cosmosdb\",\"alias\":\"azure\"}},{\"add\":{\"filter\":{\"bool\":{\"should\":[{\"has_parent\":{\"query\":{\"match_all\":{}},\"parent_type\":\"securitycenter\"}},{\"bool\":{\"must\":[{\"term\":{\"_type\":{\"value\":\"securitycenter\"}}}]}}]}},\"index\":\"azure_securitycenter\",\"alias\":\"azure\"}},{\"add\":{\"filter\":{\"bool\":{\"should\":[{\"has_parent\":{\"query\":{\"match_all\":{}},\"parent_type\":\"sites\"}},{\"bool\":{\"must\":[{\"term\":{\"_type\":{\"value\":\"sites\"}}}]}}]}},\"index\":\"azure_sites\",\"alias\":\"azure\"}}]}','1'); INSERT IGNORE INTO cf_AssetGroupTargetDetails (id_,groupId,targetType,attributeName,attributeValue) VALUES ('11501','201','ec2','all','all'); INSERT IGNORE INTO cf_AssetGroupTargetDetails (id_,groupId,targetType,attributeName,attributeValue) VALUES ('11502','201','s3','all','all'); INSERT IGNORE INTO cf_AssetGroupTargetDetails (id_,groupId,targetType,attributeName,attributeValue) VALUES ('11503','201','appelb','all','all'); @@ -987,6 +988,39 @@ INSERT IGNORE INTO cf_AssetGroupTargetDetails (id_,groupId,targetType,attributeN INSERT IGNORE INTO cf_AssetGroupTargetDetails (id_,groupId,targetType,attributeName,attributeValue) VALUES ('67713','201','elasticsearch','all','all'); INSERT IGNORE INTO cf_AssetGroupTargetDetails (id_,groupId,targetType,attributeName,attributeValue) VALUES ('67714','201','elasticache','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('00021aac-d0e6-4481-a1e7-8460154482ca','cdffb9cd-71de-4e29-9cae-783c2aa211ac','virtualmachine','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('ad076972-5c61-4e02-8c4b-7619db880f7f','cdffb9cd-71de-4e29-9cae-783c2aa211ac','blobcontainer','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a448c3a1-02c2-471d-a4b5-ea870eacbd12','cdffb9cd-71de-4e29-9cae-783c2aa211ac','cosmosdb','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('ac57da97-ad1b-4cd0-9add-e8d23d5eca03','cdffb9cd-71de-4e29-9cae-783c2aa211ac','databricks','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('abcf3e8a-9d11-42b3-9008-d548f1958d42','cdffb9cd-71de-4e29-9cae-783c2aa211ac','disk','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('9beb0437-3571-4732-ac97-6b6d8cc050e4','cdffb9cd-71de-4e29-9cae-783c2aa211ac','mariadb','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('abc56fc7-159d-4984-883e-bd3025b645b9','cdffb9cd-71de-4e29-9cae-783c2aa211ac','mysqlserver','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('abc4c00c-5fd4-4367-a899-62d7399d86ac','cdffb9cd-71de-4e29-9cae-783c2aa211ac','networkinterface','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a4293ded-951e-4b01-8633-6a10ec4b9457','cdffb9cd-71de-4e29-9cae-783c2aa211ac','nsg','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a4293ded-951e-4b01-8633-6a10ec4b9458','cdffb9cd-71de-4e29-9cae-783c2aa211ac','namespace','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a4293ded-951e-4b01-8633-6a10ec4b9459','cdffb9cd-71de-4e29-9cae-783c2aa211ac','policydefinitions','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a4293ded-951e-4b01-8633-6a10ec4b9460','cdffb9cd-71de-4e29-9cae-783c2aa211ac','policyevaluationresults','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a32495ca-ffc5-48af-ba26-316e7cb90012','cdffb9cd-71de-4e29-9cae-783c2aa211ac','postgresql','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a32495ca-ffc5-48af-ba26-316e7cb90013','cdffb9cd-71de-4e29-9cae-783c2aa211ac','publicipaddress','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('9d115e83-2821-4eeb-8224-ba2bbba1a5fa','cdffb9cd-71de-4e29-9cae-783c2aa211ac','resourcegroup','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('9d115e83-2821-4eeb-8224-ba2bbba1a5fb','cdffb9cd-71de-4e29-9cae-783c2aa211ac','searchservices','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('9d115e83-2821-4eeb-8224-ba2bbba1a5fc','cdffb9cd-71de-4e29-9cae-783c2aa211ac','securityalerts','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a5eecbfc-4a0e-4113-8301-13a44e3522d7','cdffb9cd-71de-4e29-9cae-783c2aa211ac','securitycenter','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a5eecbfc-4a0e-4113-8301-13a44e3522d8','cdffb9cd-71de-4e29-9cae-783c2aa211ac','sites','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a61e23d5-7453-4bfe-b97c-27c706674e60','cdffb9cd-71de-4e29-9cae-783c2aa211ac','sqldatabase','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('aad7068e-e5d2-4171-8e65-634aedfba6b2','cdffb9cd-71de-4e29-9cae-783c2aa211ac','sqlserver','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a1480aa8-7239-4604-9ab7-916621792fd6','cdffb9cd-71de-4e29-9cae-783c2aa211ac','storageaccount','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a1480aa8-7239-4604-9ab7-916621792fd7','cdffb9cd-71de-4e29-9cae-783c2aa211ac','subnets','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a1480aa8-7239-4604-9ab7-916621792fd8','cdffb9cd-71de-4e29-9cae-783c2aa211ac','vaults','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a1480aa8-7239-4604-9ab7-916621792fd9','cdffb9cd-71de-4e29-9cae-783c2aa211ac','vnet','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a1480aa8-7239-4604-9ab7-916621792f10','cdffb9cd-71de-4e29-9cae-783c2aa211ac','workflows','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a1480aa8-7239-4604-9ab7-916621792f11','cdffb9cd-71de-4e29-9cae-783c2aa211ac','batchaccounts','all','all'); +INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('a1480aa8-7239-4604-9ab7-916621792f12','cdffb9cd-71de-4e29-9cae-783c2aa211ac','loadbalancer','all','all'); + + + + + /*Insert Domain in required table*/ INSERT IGNORE INTO cf_Domain (domainName,domainDesc,config,createdDate,modifiedDate,userId) VALUES ('Infra & Platforms','Domain for Infra & Platforms','{}',{d '2018-04-09'},{d '2018-08-03'},'user123'); From bf5639344f03704f3f5bf7059384e842138d684f Mon Sep 17 00:00:00 2001 From: Anil Chandran Date: Fri, 15 Nov 2019 10:50:27 -0800 Subject: [PATCH 080/107] Added CloudType -Aws Added CloudType -Aws --- .../tmobile/cso/pacman/inventory/file/FileGenerator.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jobs/pacman-cloud-discovery/src/main/java/com/tmobile/cso/pacman/inventory/file/FileGenerator.java b/jobs/pacman-cloud-discovery/src/main/java/com/tmobile/cso/pacman/inventory/file/FileGenerator.java index ec040ff5d..9d38b4982 100644 --- a/jobs/pacman-cloud-discovery/src/main/java/com/tmobile/cso/pacman/inventory/file/FileGenerator.java +++ b/jobs/pacman-cloud-discovery/src/main/java/com/tmobile/cso/pacman/inventory/file/FileGenerator.java @@ -61,6 +61,9 @@ private FileGenerator() { /** The current date. */ protected static String currentDate = new SimpleDateFormat("yyyy-MM-dd HH:00:00Z").format(new java.util.Date()); + public static final String CLOUD_TYPE = "_cloudType"; + public static final String AWS = "Aws"; + /** The log. */ private static Logger log = LoggerFactory.getLogger(FileGenerator.class); @@ -339,7 +342,8 @@ protected static boolean generateJson( Map> fileInfoMap,String f lineDataMap.put(keysList[i], ""); } } - + lineDataMap.put(CLOUD_TYPE,AWS);// Added _cloudType as AWS + try { if(sb.length() == 0 && new File(folderName+File.separator+fileName).length() < 2) { sb.append(objectMapper.writeValueAsString(lineDataMap)); From 30d3a59afa314210a1cf435f26822ffaa4e35af9 Mon Sep 17 00:00:00 2001 From: ritesh Date: Tue, 12 Nov 2019 12:09:33 +0530 Subject: [PATCH 081/107] azure implemented --- webapp/src/app/core/core.module.ts | 2 + .../app/core/services/data-cache.service.ts | 10 + .../core/services/domain-mapping.service.ts | 33 ++- .../recently-viewed-observable.service.ts | 37 +++ .../home-page/home-page.component.html | 8 + .../policy-knowledgebase.component.css | 49 ++-- .../policy-knowledgebase.component.html | 70 +++-- .../policy-knowledgebase.component.ts | 162 ++++++------ .../asset-group-details.component.css | 55 ++-- .../asset-group-details.component.html | 19 +- .../asset-group-details.component.ts | 34 ++- .../post-login-header.component.html | 62 +++-- .../post-login-header.component.ts | 241 +++++++++++++----- .../default-asset-group.component.css | 21 +- .../default-asset-group.component.html | 8 +- .../default-asset-group.component.ts | 9 +- .../contextual-menu.component.css | 63 ++++- .../contextual-menu.component.html | 14 +- .../contextual-menu.component.ts | 112 +++++++- webapp/src/assets/icons/aws-color.svg | 23 ++ webapp/src/assets/icons/aws-grey.svg | 25 ++ webapp/src/assets/icons/aws-magenta.svg | 25 ++ webapp/src/assets/icons/aws-white.svg | 15 ++ webapp/src/assets/icons/azure-color.svg | 28 ++ webapp/src/assets/icons/azure-grey.svg | 19 ++ webapp/src/assets/icons/azure-magenta.svg | 19 ++ webapp/src/assets/icons/azure-white.svg | 19 ++ webapp/src/assets/icons/multi-cloud-white.svg | 16 ++ webapp/src/assets/icons/multi-cloud.svg | 16 ++ webapp/src/config/static-content.ts | 2 + webapp/src/environments/environment.ts | 2 +- webapp/src/styles.css | 27 +- 32 files changed, 972 insertions(+), 273 deletions(-) create mode 100644 webapp/src/app/core/services/recently-viewed-observable.service.ts create mode 100644 webapp/src/assets/icons/aws-color.svg create mode 100644 webapp/src/assets/icons/aws-grey.svg create mode 100644 webapp/src/assets/icons/aws-magenta.svg create mode 100644 webapp/src/assets/icons/aws-white.svg create mode 100644 webapp/src/assets/icons/azure-color.svg create mode 100644 webapp/src/assets/icons/azure-grey.svg create mode 100644 webapp/src/assets/icons/azure-magenta.svg create mode 100644 webapp/src/assets/icons/azure-white.svg create mode 100644 webapp/src/assets/icons/multi-cloud-white.svg create mode 100644 webapp/src/assets/icons/multi-cloud.svg diff --git a/webapp/src/app/core/core.module.ts b/webapp/src/app/core/core.module.ts index a86ff431f..92fc83f64 100644 --- a/webapp/src/app/core/core.module.ts +++ b/webapp/src/app/core/core.module.ts @@ -30,6 +30,7 @@ import {AdalService} from './services/adal.service'; import { HTTP_INTERCEPTORS } from '@angular/common/http'; import { RequestInterceptorService } from './services/request-interceptor.service'; import { AuthSessionStorageService } from './services/auth-session-storage.service'; +import {RecentlyViewedObservableService} from './services/recently-viewed-observable.service'; @NgModule({ imports: [ @@ -52,6 +53,7 @@ import { AuthSessionStorageService } from './services/auth-session-storage.servi AuthService, AdalService, AuthSessionStorageService, + RecentlyViewedObservableService, { provide: HTTP_INTERCEPTORS, useClass: RequestInterceptorService, diff --git a/webapp/src/app/core/services/data-cache.service.ts b/webapp/src/app/core/services/data-cache.service.ts index 59624ea61..a163443dd 100644 --- a/webapp/src/app/core/services/data-cache.service.ts +++ b/webapp/src/app/core/services/data-cache.service.ts @@ -170,6 +170,16 @@ export class DataCacheService { if (domainList) { this.set(key, domainList); } } + public getRecentlyViewedAssetGroups() { + const key = 'recentlyViewedAssetGroups'; + return this.get(key); + } + public setRecentlyViewedAssetGroups(recentlyViewedAssetGroups) { + const key = 'recentlyViewedAssetGroups'; + if (recentlyViewedAssetGroups) { this.set(key, recentlyViewedAssetGroups); } + } + + public getCurrentSelectedDomainList() { const key = 'domainList'; return this.get(key); diff --git a/webapp/src/app/core/services/domain-mapping.service.ts b/webapp/src/app/core/services/domain-mapping.service.ts index 665d3dd14..542b87666 100644 --- a/webapp/src/app/core/services/domain-mapping.service.ts +++ b/webapp/src/app/core/services/domain-mapping.service.ts @@ -18,11 +18,12 @@ import { COMPLIANCE_ROUTES, TOOLS_ROUTES, ADMIN_ROUTES, OMNISEARCH_ROUTES } from import { ASSETS_ROUTES } from '../../shared/constants/routes'; import { DataCacheService } from './data-cache.service'; import * as _ from 'lodash'; +import { Router } from '@angular/router'; import { CONFIGURATIONS } from '../../../config/configurations'; @Injectable() export class DomainMappingService { - constructor(private dataCacheService: DataCacheService) {} + constructor(private dataCacheService: DataCacheService, private router: Router) {} getDomainInfoForSelectedDomain(key) { /* @@ -54,6 +55,26 @@ export class DomainMappingService { ListOfDashboards = ListOfDashboards.concat(dashboardsObj.dashboards); }); + const currentSelectedAg = this.dataCacheService.getCurrentSelectedAssetGroup(); + let recentList = ''; + let provider = []; + recentList = this.dataCacheService.getRecentlyViewedAssetGroups(); + if (recentList) { + const currentAGDetails = JSON.parse(recentList).filter(element => element.ag === currentSelectedAg); + provider = this.fetchprovider(currentAGDetails); + } + if (currentSelectedAg.includes('azure') || (provider.length === 1 && provider[0] === 'azure')) { + ListOfDashboards = ListOfDashboards.filter(element => { + if (window.location.pathname.includes(element.route) && element.cloudSpecific) { + this.router.navigate(['pl/compliance/compliance-dashboard'], { + queryParams: { domain: domainName }, + queryParamsHandling: 'merge' + }); + } + return element.cloudSpecific !== true; + }); + } + // check qualys enabled or not if (!CONFIGURATIONS.optional.general.qualysEnabled) { ListOfDashboards = ListOfDashboards.filter(item => !(item.name === 'Vulnerabilities' && item.route === 'vulnerabilities-compliance')); @@ -128,6 +149,16 @@ export class DomainMappingService { } + fetchprovider(assetGroupObject) { + const provider = []; + if (assetGroupObject.length && assetGroupObject[0].providers) { + assetGroupObject[0].providers.forEach(element => { + provider.push(element.provider); + }); + } + return provider; + } + getDashboardsPathForADomain(dashboards, moduleName) { return dashboards.find(eachModule => eachModule.moduleName === moduleName) || {'dashboards': []}; } diff --git a/webapp/src/app/core/services/recently-viewed-observable.service.ts b/webapp/src/app/core/services/recently-viewed-observable.service.ts new file mode 100644 index 000000000..3ce125500 --- /dev/null +++ b/webapp/src/app/core/services/recently-viewed-observable.service.ts @@ -0,0 +1,37 @@ +/* + *Copyright 2018 T Mobile, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or + * implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import { Injectable } from '@angular/core'; +import { Observable } from 'rxjs'; +import { ReplaySubject } from 'rxjs/ReplaySubject'; + +@Injectable() + +export class RecentlyViewedObservableService { + + private subject = new ReplaySubject(0); + + constructor() { + } + + updateRecentAssets (recentList) { + this.subject.next(recentList); + } + + getRecentAssets(): Observable { + return this.subject.asObservable(); + } + +} diff --git a/webapp/src/app/landing-page/home-page/home-page.component.html b/webapp/src/app/landing-page/home-page/home-page.component.html index 6f30a6c00..18c96f901 100644 --- a/webapp/src/app/landing-page/home-page/home-page.component.html +++ b/webapp/src/app/landing-page/home-page/home-page.component.html @@ -28,6 +28,14 @@

{{content.homePage.productBrief}}
+
+
+ Azure +
+
+ AWS +
+
Multi-Cloud Supported
+
diff --git a/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.css b/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.css index 40cf84b8e..aee794974 100644 --- a/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.css +++ b/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.css @@ -3,9 +3,9 @@ * * Licensed under the Apache License, Version 2.0 (the "License"); You may not use * this file except in compliance with the License. A copy of the License is located at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * or in the "license" file accompanying this file. This file is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or * implied. See the License for the specific language governing permissions and @@ -13,7 +13,7 @@ */ .policy-knowledgebase-wrapper { - max-height: 100%; + height: 100%; justify-content: flex-start; overflow-y: hidden; padding-bottom: 2px; @@ -24,19 +24,25 @@ border-radius: 3px; box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.15); overflow: hidden; + height: 100%; +} + +.pk-header { + padding-top: 1em; + padding-bottom: 1em; } .pk-top-content { - padding: 2em 3em 1.8em; + padding: 1em 1em 0.2em; background-color: #fff; - min-height: 12em; + min-height: 5em; + -ms-flex-negative: 0; flex-shrink: 0; } .pk-top-header { - text-transform: capitalize; - padding-bottom: 0.66em; - font-size: 1.5em; + padding-bottom: 1.66em; + padding-left: 2.3em; font-family: ex2-medium; color: #555555; } @@ -177,16 +183,16 @@ transition-delay: 0.05s; .pk-main-content { padding: 2.66em 2.5em 2em; - /* overflow-y: hidden; */ overflow-y: auto; overflow-y: overlay; - min-height: calc(100vh - 34em); align-content: flex-start; position: relative; + height: 100%; + background: #fff; } .each-card { width: calc(33.33% - 1em); - height: 12em; + height: 13em; margin: 0.5em; border-radius: 3px; background-color: rgba(0, 0, 0, 0.02); @@ -212,23 +218,30 @@ transition-delay: 0.05s; -webkit-line-clamp: 2; max-height: 4em; display: -webkit-box; + /* autoprefixer: ignore next */ -webkit-box-orient: vertical; overflow: hidden; } .pk-desc { - /* padding-top: 3.5em; */ position: absolute; left: 2em; right: 2em; bottom: 2em; } +.auto-fix-img { + padding-left: 6px; + height: 2em; + padding-top: 4px; +} .pk-date { font-size: 0.92em; letter-spacing: 0.1px; font-family: ex2-light; + justify-content: space-evenly; color: #9b9b9b; + padding-bottom: 0.5em; } .pk-btn { @@ -249,15 +262,17 @@ transition-delay: 0.05s; color: #5a616b; } -.copy-object { +.copy-icon { + line-height: 1.3; pointer-events: none; transition: 0.2s ease; opacity: 0; } -.each-cell-list-row { - align-items: baseline; -} -.each-cell-list-row:hover .copy-object { +.hover-element:hover .copy-icon { opacity: 1; pointer-events: auto; } + +.cloud-img { + height: 1.5em; +} diff --git a/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.html b/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.html index bac2dd743..28d12b7fb 100644 --- a/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.html +++ b/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.html @@ -3,60 +3,74 @@ * * Licensed under the Apache License, Version 2.0 (the "License"); You may not use * this file except in compliance with the License. A copy of the License is located at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * or in the "license" file accompanying this file. This file is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or * implied. See the License for the specific language governing permissions and * limitations under the License. --> -
-
- -
+
+
+ +
- -

{{pageTitle}}

+
+
Repository of all the policies currently enforced.
-
-
Policies
-
Repository of all the policies currently enforced.
+
-
{{tabs}} ({{currentLength}})
+
{{tabs}} + ({{typeObj[tabs]}}) +
-
+
- +
By:
-
+
-
- Total of {{currentLength}} Policies +
+ Total of {{typeObj['All']}} Policies
- -
-
-
{{cards.name}}
- -
- -
-
{{cards.lastScan | date:'mediumDate'}}
-
{{cards.severity}}
+ +
+
+
{{cards.name}}
+ +
+
+
+ {{cards.provider}} +
+
+
+
Auto Fix Enabled
+
+
+
{{cards.severity}} +
+
@@ -68,4 +82,4 @@

{{pageTitle}}

-
\ No newline at end of file +
diff --git a/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.ts b/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.ts index c0ab6c245..ad5ffcc75 100644 --- a/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.ts +++ b/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.ts @@ -3,25 +3,26 @@ * * Licensed under the Apache License, Version 2.0 (the "License"); You may not use * this file except in compliance with the License. A copy of the License is located at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * or in the "license" file accompanying this file. This file is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or * implied. See the License for the specific language governing permissions and * limitations under the License. */ -import { Component, OnInit, OnDestroy } from '@angular/core'; +import { Component, OnDestroy, ViewChild, ElementRef, AfterViewInit, Renderer2 } from '@angular/core'; import { AssetGroupObservableService } from '../../../../core/services/asset-group-observable.service'; -import { Subscription } from 'rxjs/Subscription'; +import { Subscription } from 'rxjs'; import { CommonResponseService } from '../../../../shared/services/common-response.service'; import { environment } from './../../../../../environments/environment'; -import { Router, ActivatedRoute} from '@angular/router'; +import { Router } from '@angular/router'; import { LoggerService } from '../../../../shared/services/logger.service'; import { ErrorHandlingService } from '../../../../shared/services/error-handling.service'; import { WorkflowService } from '../../../../core/services/workflow.service'; import { DomainTypeObservableService } from '../../../../core/services/domain-type-observable.service'; +import { RouterUtilityService } from '../../../../shared/services/router-utility.service'; @Component({ selector: 'app-policy-knowledgebase', @@ -29,12 +30,8 @@ import { DomainTypeObservableService } from '../../../../core/services/domain-ty styleUrls: ['./policy-knowledgebase.component.css'], providers: [CommonResponseService, LoggerService, ErrorHandlingService] }) -export class PolicyKnowledgebaseComponent implements OnInit, OnDestroy { - - pageTitle = 'Policy Knowledgebase'; - breadcrumbArray: any= ['Compliance']; - breadcrumbLinks: any= ['compliance-dashboard']; - breadcrumbPresent: any; +export class PolicyKnowledgebaseComponent implements AfterViewInit, OnDestroy { + pageTitle = 'Policies'; selectedAssetGroup: string; selectedDomain: string; subscriptionToAssetGroup: Subscription; @@ -47,71 +44,91 @@ export class PolicyKnowledgebaseComponent implements OnInit, OnDestroy { searchTxt = ''; knowledgebaseData: any = []; tabName: any = []; + count = []; + num = 0; selName: any = []; selectedTab = 0; selectedFilter = 0; selectedFilterName = ''; + typeObj; searchQuery = ''; loaded = false; - currentLength = 0; datacoming = false; seekdata = false; errorMessage: any; urlToRedirect: any = ''; - private pageLevel = 0; - public backButtonRequired; + public agAndDomain = {}; + currentPageLevel = 0; + + @ViewChild('pkInp') pkInp: ElementRef; constructor(private assetGroupObservableService: AssetGroupObservableService, + private renderer: Renderer2, private router: Router, - private activatedRoute: ActivatedRoute, private commonResponseService: CommonResponseService, private logger: LoggerService, private errorHandling: ErrorHandlingService, private workflowService: WorkflowService, - private domainObservableService: DomainTypeObservableService) { - this.subscriptionToAssetGroup = this.assetGroupObservableService.getAssetGroup().subscribe(assetGroupName => { - this.backButtonRequired = this.workflowService.checkIfFlowExistsCurrently(this.pageLevel); - this.selectedAssetGroup = assetGroupName; - }); - this.domainSubscription = this.domainObservableService.getDomainType().subscribe(domain => { - this.selectedDomain = domain; - this.updateComponent(); + private domainObservableService: DomainTypeObservableService, + private routerUtilityService: RouterUtilityService) { + this.subscriptionToAssetGroup = this.assetGroupObservableService.getAssetGroup().subscribe(assetGroupName => { + this.selectedAssetGroup = assetGroupName; + this.agAndDomain['ag'] = this.selectedAssetGroup; + }); + this.domainSubscription = this.domainObservableService.getDomainType().subscribe(domain => { + this.selectedDomain = domain; + this.agAndDomain['domain'] = this.selectedDomain; + this.updateComponent(); }); + this.currentPageLevel = this.routerUtilityService.getpageLevel(this.router.routerState.snapshot.root); } - ngOnInit() { - this.breadcrumbPresent = 'Policy Knowledgebase'; + ngAfterViewInit() { + } updateComponent() { - this.getData(); + this.loaded = false; + this.datacoming = false; + this.seekdata = false; + this.knowledgebaseData = []; + this.typeObj = undefined; + this.getData(); } processData(data) { try { const getData = data; - const typeObj = { - 'All': 'typeVal' - }; - for (let i = 0 ; i < getData.length; i++) { - typeObj[getData[i].ruleCategory] = 'typeVal'; - } - typeObj[`critical`] = 'selVal'; - typeObj[`high`] = 'selVal'; - typeObj[`medium`] = 'selVal'; - typeObj[`low`] = 'selVal'; - for (let i = 0 ; i < getData.length; i++) { - typeObj[getData[i].severity] = 'selVal'; + this.typeObj = { + 'All': 0 + }; + for (let i = 0; i < getData.length; i++) { + this.typeObj[getData[i].ruleCategory] = 0; + } + this.typeObj[`critical`] = 0; + this.typeObj[`high`] = 0; + this.typeObj[`medium`] = 0; + this.typeObj[`low`] = 0; + for (let i = 0; i < getData.length; i++) { + this.typeObj[getData[i].severity] = 0; + } + this.typeObj[`Auto Fix`] = 0; + delete this.typeObj['']; + for (let i = 0; i < getData.length; i++) { + this.typeObj['All']++; + this.typeObj[getData[i].ruleCategory]++; + this.typeObj[getData[i].severity]++; + if (getData[i].autoFixEnabled === true) { + this.typeObj['Auto Fix']++; } - delete typeObj['']; - let typeArr = []; - const selArr = []; - typeArr = Object.keys(typeObj); - this.tabName = typeArr; - this.selectedTabName = this.tabName[this.selectedTab]; - } catch (error) { - this.logger.log('error', error); } + let typeArr = []; + typeArr = Object.keys(this.typeObj); + this.tabName = typeArr; + this.selectedTabName = this.tabName[this.selectedTab]; + } catch (error) { + this.logger.log('error', error); + } } getData() { @@ -124,26 +141,28 @@ export class PolicyKnowledgebaseComponent implements OnInit, OnDestroy { 'ag': this.selectedAssetGroup, 'searchtext': this.searchTxt, 'filter': { - 'domain': this.selectedDomain + 'domain': this.selectedDomain }, 'from': 0, 'size': 10 }; - const queryParams = {}; - const complianceTableUrl = environment.complianceTable.url; - const complianceTableMethod = environment.complianceTable.method; - this.complianceTableSubscription = this.commonResponseService.getData( - complianceTableUrl, complianceTableMethod, payload, queryParams).subscribe( - response => { + const queryParams = {}; + const complianceTableUrl = environment.complianceTable.url; + const complianceTableMethod = environment.complianceTable.method; + this.complianceTableSubscription = this.commonResponseService.getData( + complianceTableUrl, complianceTableMethod, payload, queryParams).subscribe( + response => { if (response.data.response.length !== 0) { this.datacoming = true; this.knowledgebaseData = response.data.response; - this.currentLength = this.knowledgebaseData.length; this.dataLoaded = true; const x = this; - setTimeout(function () { + setTimeout(function () { x.loaded = true; + if (x.pkInp) { + x.pkInp.nativeElement.focus(); + } }, 200); this.processData(this.knowledgebaseData); } else { @@ -152,39 +171,36 @@ export class PolicyKnowledgebaseComponent implements OnInit, OnDestroy { this.seekdata = true; this.errorMessage = 'noDataAvailable'; } - }, - error => { + }, + error => { this.datacoming = false; this.dataLoaded = true; this.seekdata = true; this.errorMessage = 'apiResponseError'; - }); - } - - getLength() { - setTimeout(() => { - const data = document.getElementsByClassName('mr-pkb-cards'); - this.currentLength = data.length; - }, 10); + }); } /* * this function is used to fetch the rule id and to navigate to the next page */ - gotoNextPage(ruleId) { + gotoNextPage(tileData) { + let autofixEnabled = false; + if ( tileData.autoFixEnabled) { + autofixEnabled = true; + } + const ruleId = tileData.ruleId; try { this.workflowService.addRouterSnapshotToLevel(this.router.routerState.snapshot.root); - this.router.navigate(['../policy-knowledgebase-details', ruleId], - {relativeTo: this.activatedRoute, queryParamsHandling: 'merge'}); + this.router.navigate( + ['pl', { outlets: { details: ['policy-knowledgebase-details', ruleId , autofixEnabled]}}], + { queryParams: this.agAndDomain, + queryParamsHandling: 'merge' }); } catch (error) { - this.errorMessage = this.errorHandling.handleJavascriptError(error); - this.logger.log('error', error); + this.errorMessage = this.errorHandling.handleJavascriptError(error); + this.logger.log('error', error); } } - navigateBack() { - this.workflowService.goBackToLastOpenedPageAndUpdateLevel(this.router.routerState.snapshot.root); - } ngOnDestroy() { try { diff --git a/webapp/src/app/post-login-app/common/asset-group-details/asset-group-details.component.css b/webapp/src/app/post-login-app/common/asset-group-details/asset-group-details.component.css index ecf94c8ca..cf43ee834 100644 --- a/webapp/src/app/post-login-app/common/asset-group-details/asset-group-details.component.css +++ b/webapp/src/app/post-login-app/common/asset-group-details/asset-group-details.component.css @@ -13,30 +13,20 @@ */ .asset-details-wrapper { - margin: 0.2em 1.7em; - border-radius: 0.5em; - box-shadow: 0px 1px 7px 4px rgba(0, 0, 0, 0.2); - color: #fff; - padding: 3.3em 1.5em 3em 1.9em; - min-height: 26em; + /* padding: 2.3em; */ min-width: 10em; - width: 88%; - background-color: #5a616b; - min-width: 17.8em; -} - -.details-section-content { - width: 13.5em; } .desc-detail { - margin-bottom: 1.3em; + margin-bottom: 1.5em; + font-size: 1.1em; } .keyword { font-family: ex2-regular; padding-right: 0.3em; - animation: fadein 0.5s; + animation: fadein 0.3s; + color: #333; } .key-value { @@ -47,27 +37,29 @@ .dotted-line { flex-grow: 2; - border-bottom: 1px dotted #aaa; - animation: fadein 0.5s; + border-bottom: 1px dotted #d9d9d9; + animation: fadein 0.3s; } .name-title { /*white-space: nowrap;*/ font-family: ex2-bold; - font-size: 1.3em; + font-size: 1.4em; margin-bottom: 1em; line-height: 1.3; - animation: fadein 0.5s; + animation: fadein 0.3s; text-transform: capitalize; } .desc-title { white-space: normal; - font-size: 1em; - padding-bottom: 2.7em; + font-size: 1.05em; + padding-bottom: 2.4em; line-height: 1.25; border-bottom: 1px solid #aaa; - animation: fadein 0.5s; + animation: fadein 0.3s; + font-family: ex2-light; + color: #111; } @keyframes fadein { @@ -80,13 +72,28 @@ } article.detail-desc { - padding-top: 2em; + padding: 2em 6px 1.3em; text-transform: capitalize; + max-width: 55%; + min-width: 20em; + border-bottom: 1px solid #aaa; } .error-msg { - color: #fff; + /* color: #fff; */ text-transform: initial; line-height: 1.3; text-align: center; +} + +.link-text-secondary { + text-decoration: underline; +} + +.height100 { + height: 100%; +} + +.each-cloud { + margin: 0 1em; } \ No newline at end of file diff --git a/webapp/src/app/post-login-app/common/asset-group-details/asset-group-details.component.html b/webapp/src/app/post-login-app/common/asset-group-details/asset-group-details.component.html index fbacabb8a..bfd316e46 100644 --- a/webapp/src/app/post-login-app/common/asset-group-details/asset-group-details.component.html +++ b/webapp/src/app/post-login-app/common/asset-group-details/asset-group-details.component.html @@ -12,10 +12,14 @@ * limitations under the License. --> -
+
-
{{detailsVal.displayname}}
+
+
{{detailsVal.displayname}}
+
Multi-Cloud
+
{{provider}}
+
{{detailsVal.description}}
Select this asset group to fetch data under this asset group
@@ -38,7 +42,16 @@
Asset Count
-
{{detailsVal.assetcount | number}}
+
{{detailsVal.assetcount | number}}
+
+
+
Cloud Types
+
+
+ {{val}} + , + +
diff --git a/webapp/src/app/post-login-app/common/asset-group-details/asset-group-details.component.ts b/webapp/src/app/post-login-app/common/asset-group-details/asset-group-details.component.ts index 48ca6d017..e6d8ff564 100644 --- a/webapp/src/app/post-login-app/common/asset-group-details/asset-group-details.component.ts +++ b/webapp/src/app/post-login-app/common/asset-group-details/asset-group-details.component.ts @@ -3,16 +3,16 @@ * * Licensed under the Apache License, Version 2.0 (the "License"); You may not use * this file except in compliance with the License. A copy of the License is located at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * or in the "license" file accompanying this file. This file is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or * implied. See the License for the specific language governing permissions and * limitations under the License. */ -import { Component, Input, EventEmitter, Output } from '@angular/core'; +import { Component, Input, EventEmitter, Output, OnChanges } from '@angular/core'; import { AssetTilesService } from '../../../core/services/asset-tiles.service'; @Component({ @@ -22,7 +22,7 @@ import { AssetTilesService } from '../../../core/services/asset-tiles.service'; providers: [AssetTilesService] }) -export class AssetGroupDetailsComponent { +export class AssetGroupDetailsComponent implements OnChanges { @Input() selectedValue: any; @Input() detailsVal: any = {}; @@ -30,15 +30,33 @@ export class AssetGroupDetailsComponent { public errorMessage: any; @Output() navigatePage: EventEmitter = new EventEmitter(); + provider = []; + constructor () { + } - constructor( - ) { } + ngOnChanges() { + this.createProviderArray(); + } capitalizeFirstLetter(string): any { return string.charAt(0).toUpperCase() + string.slice(1); } - instructParentToNavigate (data) { - this.navigatePage.emit(data); + createProviderArray() { + this.provider = []; + if (this.detailsVal && this.detailsVal.providers) { + this.detailsVal.providers.forEach(element => { + this.provider.push(element.provider); + }); + } + } + + instructParentToNavigate (data, agDetails) { + const obj = { + data: data, + agDetails: agDetails + }; + this.navigatePage.emit(obj); } + } diff --git a/webapp/src/app/post-login-app/common/post-login-header/post-login-header.component.html b/webapp/src/app/post-login-app/common/post-login-header/post-login-header.component.html index 5fe5b7137..0177f3f38 100644 --- a/webapp/src/app/post-login-app/common/post-login-header/post-login-header.component.html +++ b/webapp/src/app/post-login-app/common/post-login-header/post-login-header.component.html @@ -30,18 +30,25 @@
-
+
diff --git a/webapp/src/app/post-login-app/common/post-login-header/post-login-header.component.css b/webapp/src/app/post-login-app/common/post-login-header/post-login-header.component.css index 4dae9fed8..c469c7ef1 100644 --- a/webapp/src/app/post-login-app/common/post-login-header/post-login-header.component.css +++ b/webapp/src/app/post-login-app/common/post-login-header/post-login-header.component.css @@ -385,4 +385,7 @@ nav .nav-icon:last-child { .user-profile-wrapper:hover > .profile { transform: scale(1.2); +} +.cloud-icon-wrapper { + padding-right: .6em; } \ No newline at end of file diff --git a/webapp/src/app/post-login-app/common/post-login-header/post-login-header.component.html b/webapp/src/app/post-login-app/common/post-login-header/post-login-header.component.html index 0177f3f38..3b5d8a325 100644 --- a/webapp/src/app/post-login-app/common/post-login-header/post-login-header.component.html +++ b/webapp/src/app/post-login-app/common/post-login-header/post-login-header.component.html @@ -78,7 +78,7 @@

Recently Viewed

- {{row.name}} + {{row.name}}
diff --git a/webapp/src/app/post-login-app/common/post-login-header/post-login-header.component.ts b/webapp/src/app/post-login-app/common/post-login-header/post-login-header.component.ts index f5a8dc228..14818e9f1 100644 --- a/webapp/src/app/post-login-app/common/post-login-header/post-login-header.component.ts +++ b/webapp/src/app/post-login-app/common/post-login-header/post-login-header.component.ts @@ -190,9 +190,14 @@ export class PostLoginHeaderComponent implements OnInit, OnDestroy { getModuleLinks() { const complianceLinks = this.domainMappingService.getDashboardsApplicableForADomain(this.selectedDomainName, 'compliance'); - const toolsLinks = this.domainMappingService.getDashboardsApplicableForADomain(this.selectedDomainName, 'tools'); - const reportsLinks = this.domainMappingService.getDashboardsApplicableForADomain(this.selectedDomainName, 'reports'); - const myboardLinks = this.domainMappingService.getDashboardsApplicableForADomain(this.selectedDomainName, 'myboard'); + const assetsLinks = this.domainMappingService.getDashboardsApplicableForADomain(this.selectedDomainName, 'assets'); + const statisticsLinks = [ + { + route: 'stats-overlay', + name: 'Statistics', + overlay: true + } + ]; let complianceLinksUpdated = JSON.parse(JSON.stringify(complianceLinks)); @@ -201,48 +206,30 @@ export class PostLoginHeaderComponent implements OnInit, OnDestroy { return eachRoute; }); - let toolsLinksUpdated = JSON.parse(JSON.stringify(toolsLinks)); - toolsLinksUpdated = toolsLinksUpdated.map(eachRoute => { - eachRoute.route = 'tools/' + eachRoute.route; - return eachRoute; - }); - - let myboardLinksUpdated = JSON.parse(JSON.stringify(myboardLinks)); - myboardLinksUpdated = myboardLinksUpdated.map(eachRoute => { - eachRoute.route = 'myboard/' + eachRoute.route; - return eachRoute; - }); - - let reportsLinksUpdated = JSON.parse(JSON.stringify(reportsLinks)); - reportsLinksUpdated = reportsLinksUpdated.map(eachRoute => { - eachRoute.route = 'reports/' + eachRoute.route; + let assetsLinksUpdated = JSON.parse(JSON.stringify(assetsLinks)); + assetsLinksUpdated = assetsLinksUpdated.map(eachRoute => { + eachRoute.route = 'assets/' + eachRoute.route; return eachRoute; }); this.burgerMenuModuleLinks = [ { img: '../assets/icons/compliance.svg', - title: 'PacBoard', + title: 'compliance', rows: complianceLinksUpdated, shown: this.config.required.featureModules.COMPLIANCE_MODULE }, { - img: '../assets/icons/reports.svg', - title: 'reports', - rows: reportsLinksUpdated, - shown: this.config.required.featureModules.REPORTS_MODULE - }, - { - img: '../assets/icons/tools.svg', - title: 'tools', - rows: toolsLinksUpdated, - shown: this.config.required.featureModules.TOOLS_MODULE + img: '../assets/icons/assets.svg', + title: 'assets', + rows: assetsLinksUpdated, + shown: this.config.required.featureModules.ASSETS_MODULE }, { - img: '../assets/icons/myboard.svg', - title: 'my board', - rows: myboardLinksUpdated, - shown: this.config.required.featureModules.MYBOARD_MODULE + img: '../assets/icons/Statistics.svg', + title: 'Statistics', + rows: statisticsLinks, + shown: true } ]; this.footerData = []; From f4c1143a17e91be36749033697ae3027ee15d632 Mon Sep 17 00:00:00 2001 From: Anil Chandran Date: Tue, 19 Nov 2019 10:23:00 -0800 Subject: [PATCH 093/107] Update pom.xml --- jobs/pacman-data-shipper/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/pacman-data-shipper/pom.xml b/jobs/pacman-data-shipper/pom.xml index 98a3a7478..5adf707e0 100644 --- a/jobs/pacman-data-shipper/pom.xml +++ b/jobs/pacman-data-shipper/pom.xml @@ -144,7 +144,7 @@ com.fasterxml.jackson.core jackson-databind - 2.9.10.1 + 2.10.1 mysql From d6a2d8427fa554541857528a01a90c65b2ba0c65 Mon Sep 17 00:00:00 2001 From: dipesh agarwal Date: Thu, 21 Nov 2019 15:06:52 +0530 Subject: [PATCH 094/107] bug fixes --- webapp/src/app/core/services/routing.service.ts | 17 +++++++++++++---- .../recommendations.component.ts | 5 +++-- .../asset-groups/asset-groups.component.ts | 5 ++++- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/webapp/src/app/core/services/routing.service.ts b/webapp/src/app/core/services/routing.service.ts index 293ea7101..36572e352 100644 --- a/webapp/src/app/core/services/routing.service.ts +++ b/webapp/src/app/core/services/routing.service.ts @@ -36,9 +36,7 @@ export class RoutingService { assetGroupAndDomain['domain'] = agAndDomain['domain']; const savedPath = this.workflowService.getPreviouslyOpenedPageInModule(moduleName); - if (savedPath) { - const queryParams = this.workflowService.getPreviouslyOpenedPageQueryParamsInModule(moduleName) ? JSON.parse(this.workflowService.getPreviouslyOpenedPageQueryParamsInModule(moduleName)) : {}; @@ -47,6 +45,18 @@ export class RoutingService { url = savedPath; queryParamsToBePassed = queryParams; + const level = this.workflowService.getDetailsFromStorage(); + const newLevel = []; + if (level['level0'] && level['level0'].length > 0) { + for (let i = 0; i < level['level0'].length; i++) { + if (level['level0'][i]['url'] === savedPath) { + break; + } else { + newLevel.push(level['level0'][i]); + } + } + this.workflowService.saveToStorage({level0: newLevel}); + } } else { const listOfContextualMenuItems = this.domainMappingService.getDashboardsApplicableForADomain(agAndDomain['domain'], moduleName); @@ -56,11 +66,10 @@ export class RoutingService { url = 'pl' + '/' + moduleName + '/'; } queryParamsToBePassed = assetGroupAndDomain; + this.clearPageLevel(); } - this.router.navigate([url], {queryParams: queryParamsToBePassed}).then(response => { // Clearig page levels. - this.clearPageLevel(); }); } catch (error) { this.loggerService.log('error', 'js error - ' + error); diff --git a/webapp/src/app/pacman-features/modules/compliance/recommendations/recommendations.component.ts b/webapp/src/app/pacman-features/modules/compliance/recommendations/recommendations.component.ts index 1ea95a71d..99dc26d79 100644 --- a/webapp/src/app/pacman-features/modules/compliance/recommendations/recommendations.component.ts +++ b/webapp/src/app/pacman-features/modules/compliance/recommendations/recommendations.component.ts @@ -359,8 +359,9 @@ export class RecommendationsComponent implements OnInit, OnChanges, OnDestroy { 'order': 1 }; response.forEach((element) => { - element['displayName'] = this.refactorFieldsService.getDisplayNameForAKey(element['category'].trim().toLocaleLowerCase()) || element['category']; - element['icon'] = '../../../../../assets/icons/recommand_' + element.category.trim() + '.svg'; + element['category'] = element['category'].trim(); + element['displayName'] = this.refactorFieldsService.getDisplayNameForAKey(element['category'].toLocaleLowerCase()) || element['category']; + element['icon'] = '../../../../../assets/icons/recommand_' + element.category + '.svg'; element['color'] = summaryTabsColor[element.category]; element['order'] = displayOrder[element.category]; this.tabName.push(element); diff --git a/webapp/src/app/post-login-app/asset-groups/asset-groups.component.ts b/webapp/src/app/post-login-app/asset-groups/asset-groups.component.ts index b98740b3c..0da87d0a5 100644 --- a/webapp/src/app/post-login-app/asset-groups/asset-groups.component.ts +++ b/webapp/src/app/post-login-app/asset-groups/asset-groups.component.ts @@ -272,8 +272,11 @@ export class AssetGroupsComponent implements AfterViewInit, OnDestroy { if (!name || !this.selectedTabName) { return false; } + const tiles = this.recentTiles.map(item => { + return item['ag']; + }); if (this.selectedTabName.toLowerCase() === 'recently viewed') { - if (this.recentTiles.includes(name.name.toLowerCase())) { + if (tiles.includes(name.name.toLowerCase())) { return true; } else { return false; From c166447445575b5ee9ac43757cccbc3bd3ed7ae6 Mon Sep 17 00:00:00 2001 From: dipesh agarwal Date: Thu, 21 Nov 2019 19:01:22 +0530 Subject: [PATCH 095/107] resource id breaking in policy violation details page --- .../policy-violation-desc.component.css | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/webapp/src/app/pacman-features/secondary-components/policy-violation-desc/policy-violation-desc.component.css b/webapp/src/app/pacman-features/secondary-components/policy-violation-desc/policy-violation-desc.component.css index cf731d5a0..58faf0f32 100644 --- a/webapp/src/app/pacman-features/secondary-components/policy-violation-desc/policy-violation-desc.component.css +++ b/webapp/src/app/pacman-features/secondary-components/policy-violation-desc/policy-violation-desc.component.css @@ -15,7 +15,6 @@ .policy-violation-label { font-size: 1.2em; padding-bottom: 1.5em; - } .policy-violation-label-header { @@ -23,6 +22,10 @@ font-family: ex2-bold; } +.policy-violation-desc-wrapper { + word-break: break-word; +} + .policy-violation-label-content { font-family: ex2-light; line-height: 1.2; From a3b96337810f682a321a346a43d04a7e8c135a78 Mon Sep 17 00:00:00 2001 From: dipesh agarwal Date: Thu, 21 Nov 2019 19:38:34 +0530 Subject: [PATCH 096/107] minor asset group change ui bugfix --- .../post-login-app/asset-groups/asset-groups.component.css | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/webapp/src/app/post-login-app/asset-groups/asset-groups.component.css b/webapp/src/app/post-login-app/asset-groups/asset-groups.component.css index 464abc81f..fc0697a07 100644 --- a/webapp/src/app/post-login-app/asset-groups/asset-groups.component.css +++ b/webapp/src/app/post-login-app/asset-groups/asset-groups.component.css @@ -265,4 +265,10 @@ img.delete-icon { .asset-tile-container { overflow: auto; } +} + +.asset-details-content { + display: flex; + flex-direction: column; + justify-content: space-between; } \ No newline at end of file From 506b46ce9db713cbfe9f13a9519826206aed4a25 Mon Sep 17 00:00:00 2001 From: dipesh agarwal Date: Thu, 21 Nov 2019 22:17:00 +0530 Subject: [PATCH 097/107] merge conflict --- .../post-login-app/asset-groups/asset-groups.component.css | 6 ------ 1 file changed, 6 deletions(-) diff --git a/webapp/src/app/post-login-app/asset-groups/asset-groups.component.css b/webapp/src/app/post-login-app/asset-groups/asset-groups.component.css index fc0697a07..4444bfc23 100644 --- a/webapp/src/app/post-login-app/asset-groups/asset-groups.component.css +++ b/webapp/src/app/post-login-app/asset-groups/asset-groups.component.css @@ -266,9 +266,3 @@ img.delete-icon { overflow: auto; } } - -.asset-details-content { - display: flex; - flex-direction: column; - justify-content: space-between; -} \ No newline at end of file From dabecfefd299527b4d6da531bd194359fe6da9ec Mon Sep 17 00:00:00 2001 From: Kamal Kumar Date: Thu, 21 Nov 2019 12:12:38 -0800 Subject: [PATCH 098/107] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 473f43762..35f72ef3d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![GitHub contributors](https://img.shields.io/github/contributors/tmobile/pacbot.svg)](https://github.com/tmobile/pacbot/graphs/contributors) [![Gitter](https://github.com/tmobile/pacbot/blob/master/wiki/images/chat.svg)](https://gitter.im/TMO-OSS/PacBot) - + # Introduction From c589e78b218cac3e0bcdc0ab4e944ad822165045 Mon Sep 17 00:00:00 2001 From: dipesh agarwal Date: Fri, 22 Nov 2019 15:13:34 +0530 Subject: [PATCH 099/107] Fixed bug related to navigating from policy knowledgebase to policy details page --- .../cloud-notifications.component.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/webapp/src/app/pacman-features/modules/compliance/cloud-notifications/cloud-notifications.component.ts b/webapp/src/app/pacman-features/modules/compliance/cloud-notifications/cloud-notifications.component.ts index c41c334b7..916b4318e 100644 --- a/webapp/src/app/pacman-features/modules/compliance/cloud-notifications/cloud-notifications.component.ts +++ b/webapp/src/app/pacman-features/modules/compliance/cloud-notifications/cloud-notifications.component.ts @@ -121,9 +121,17 @@ export class CloudNotificationsComponent implements OnInit, OnDestroy { .getAssetGroup() .subscribe(assetGroupName => { this.selectedAssetGroup = assetGroupName; - this.calibrateFilter(); - this.getSummary(); - this.updateComponent(); + if (this.selectedAssetGroup.match('azure')) { + setTimeout(() => { + this.router.navigate(['pl', 'compliance/compliance-dashboard'], { + queryParamsHandling: 'merge' + }); + }, 10); + } else { + this.calibrateFilter(); + this.getSummary(); + this.updateComponent(); + } }); } From 219d779707282a40bdb82e4b938d224b94182cbf Mon Sep 17 00:00:00 2001 From: dipesh agarwal Date: Fri, 22 Nov 2019 22:14:40 +0530 Subject: [PATCH 100/107] Fixed policy knowledge base to policy details page routing bug --- .../policy-knowledgebase/policy-knowledgebase.component.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.ts b/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.ts index ad5ffcc75..52f433250 100644 --- a/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.ts +++ b/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.ts @@ -184,7 +184,7 @@ export class PolicyKnowledgebaseComponent implements AfterViewInit, OnDestroy { * this function is used to fetch the rule id and to navigate to the next page */ - gotoNextPage(tileData) { + gotoNextPage(tileData){ let autofixEnabled = false; if ( tileData.autoFixEnabled) { autofixEnabled = true; @@ -193,7 +193,7 @@ export class PolicyKnowledgebaseComponent implements AfterViewInit, OnDestroy { try { this.workflowService.addRouterSnapshotToLevel(this.router.routerState.snapshot.root); this.router.navigate( - ['pl', { outlets: { details: ['policy-knowledgebase-details', ruleId , autofixEnabled]}}], + ['pl', 'compliance', 'policy-knowledgebase-details', ruleId], { queryParams: this.agAndDomain, queryParamsHandling: 'merge' }); } catch (error) { From dd37f23ea5b830a48bb262a3ba4172c4ce253ef4 Mon Sep 17 00:00:00 2001 From: dipesh agarwal Date: Fri, 22 Nov 2019 22:32:22 +0530 Subject: [PATCH 101/107] minor change in code style --- .../cloud-notifications/cloud-notifications.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/src/app/pacman-features/modules/compliance/cloud-notifications/cloud-notifications.component.ts b/webapp/src/app/pacman-features/modules/compliance/cloud-notifications/cloud-notifications.component.ts index 916b4318e..c377bb395 100644 --- a/webapp/src/app/pacman-features/modules/compliance/cloud-notifications/cloud-notifications.component.ts +++ b/webapp/src/app/pacman-features/modules/compliance/cloud-notifications/cloud-notifications.component.ts @@ -123,7 +123,7 @@ export class CloudNotificationsComponent implements OnInit, OnDestroy { this.selectedAssetGroup = assetGroupName; if (this.selectedAssetGroup.match('azure')) { setTimeout(() => { - this.router.navigate(['pl', 'compliance/compliance-dashboard'], { + this.router.navigate(['pl', 'compliance', 'compliance-dashboard'], { queryParamsHandling: 'merge' }); }, 10); From 0a67dc76c3dc5949e964dfee8e1ffd9795b22c6b Mon Sep 17 00:00:00 2001 From: dipesh agarwal Date: Mon, 25 Nov 2019 13:09:30 +0530 Subject: [PATCH 102/107] fixed the resource Id clicking error --- .../modules/assets/asset-list/asset-list.component.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/webapp/src/app/pacman-features/modules/assets/asset-list/asset-list.component.ts b/webapp/src/app/pacman-features/modules/assets/asset-list/asset-list.component.ts index 0d2df3a43..163d42aa4 100644 --- a/webapp/src/app/pacman-features/modules/assets/asset-list/asset-list.component.ts +++ b/webapp/src/app/pacman-features/modules/assets/asset-list/asset-list.component.ts @@ -473,13 +473,20 @@ export class AssetListComponent implements OnInit, OnDestroy { data.map(function(responseData){ const KeysTobeChanged = Object.keys(responseData); let newObj = {}; + let entityType; KeysTobeChanged.forEach(element => { + if ( element === '_entitytype') { + entityType = responseData['_entitytype']; + } const elementnew = refactoredService.getDisplayNameForAKey( element.toLocaleLowerCase() ) || element; newObj = Object.assign(newObj, { [elementnew]: responseData[element] }); }); + if (entityType) { + newObj['Asset Type'] = entityType; + } newData.push(newObj); }); return newData; From 440f0833236ebfa7dae86896a01bcbf2d4000a09 Mon Sep 17 00:00:00 2001 From: dipesh agarwal Date: Mon, 25 Nov 2019 17:55:25 +0530 Subject: [PATCH 103/107] autoFix enabled not shown for the policies having autofix enabled in policy details page --- .../policy-knowledgebase-details.component.html | 10 +++++++++- .../policy-knowledgebase-details.component.ts | 3 ++- .../policy-knowledgebase.component.ts | 2 +- webapp/src/app/shared/constants/routes.ts | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase-details/policy-knowledgebase-details.component.html b/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase-details/policy-knowledgebase-details.component.html index 13d87a76b..86c8f622d 100644 --- a/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase-details/policy-knowledgebase-details.component.html +++ b/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase-details/policy-knowledgebase-details.component.html @@ -24,7 +24,15 @@

{{pageTitle}}

-
{{displayName}}
+
+
{{displayName}}
+
+
+
+ Auto fix Enabled +
+
+
Description
{{ruleDescription}}
diff --git a/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase-details/policy-knowledgebase-details.component.ts b/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase-details/policy-knowledgebase-details.component.ts index d7f58531d..66f6e81d4 100644 --- a/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase-details/policy-knowledgebase-details.component.ts +++ b/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase-details/policy-knowledgebase-details.component.ts @@ -37,7 +37,7 @@ export class PolicyKnowledgebaseDetailsComponent implements OnInit, OnDestroy { breadcrumbPresent: any; selectedAssetGroup: string; subscriptionToAssetGroup: Subscription; - + public autoFix = false; public ruleID: any = ''; public setRuleIdObtained = false; public dataComing = true; @@ -106,6 +106,7 @@ export class PolicyKnowledgebaseDetailsComponent implements OnInit, OnDestroy { /* TODO:Trinanjan Wrong way of doing it */ this.routeSubscription = this.activatedRoute.params.subscribe(params => { this.ruleID = params['ruleID']; + this.autoFix = (params['autoFix'] === 'true'); }); if (this.ruleID !== undefined) { this.setRuleIdObtained = true; diff --git a/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.ts b/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.ts index 52f433250..b3d279aad 100644 --- a/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.ts +++ b/webapp/src/app/pacman-features/modules/compliance/policy-knowledgebase/policy-knowledgebase.component.ts @@ -193,7 +193,7 @@ export class PolicyKnowledgebaseComponent implements AfterViewInit, OnDestroy { try { this.workflowService.addRouterSnapshotToLevel(this.router.routerState.snapshot.root); this.router.navigate( - ['pl', 'compliance', 'policy-knowledgebase-details', ruleId], + ['pl', 'compliance', 'policy-knowledgebase-details', ruleId, autofixEnabled], { queryParams: this.agAndDomain, queryParamsHandling: 'merge' }); } catch (error) { diff --git a/webapp/src/app/shared/constants/routes.ts b/webapp/src/app/shared/constants/routes.ts index c4d654760..c211715b1 100644 --- a/webapp/src/app/shared/constants/routes.ts +++ b/webapp/src/app/shared/constants/routes.ts @@ -152,7 +152,7 @@ export const COMPLIANCE_ROUTES = [ canActivate: [AuthGuardService] }, { - path: 'policy-knowledgebase-details/:ruleID', + path: 'policy-knowledgebase-details/:ruleID/:autoFix', component: PolicyKnowledgebaseDetailsComponent, data: { title: 'Policy Details' From 73aad23e4f1de4bb151a0a55772ef083ec496fe8 Mon Sep 17 00:00:00 2001 From: Sajeer N Date: Tue, 26 Nov 2019 17:51:59 +0530 Subject: [PATCH 104/107] Pep8 compliance: corrections made --- installer/core/constants.py | 8 ++++---- installer/core/providers/aws/input.py | 3 +-- installer/core/providers/aws/reinstall.py | 2 +- installer/files/scripts/utils.py | 2 -- installer/resources/lambda_submit/function.py | 2 +- installer/resources/pacbot_app/files/DB.sql | 1 - installer/resources/pacbot_app/import_db.py | 1 - installer/resources/pacbot_app/utils.py | 1 + 8 files changed, 8 insertions(+), 12 deletions(-) diff --git a/installer/core/constants.py b/installer/core/constants.py index 573376371..d855dbc17 100644 --- a/installer/core/constants.py +++ b/installer/core/constants.py @@ -33,10 +33,10 @@ AWS_CHOOSE_AUTH_OPTION = "Type 1 or 2 or 3 to continue to create services in AWS: " AWS_INCORRECT_MECHANISM = "Entered an incorrect value!!!" AWS_AUTH_MECHANISM_NOT_SUPPLIED = "Please add value 1 or 2 or 3 for AWS_AUTH_MECHANISM in settings/local.py" -AWS_ACCESS_KEY_NOT_SUPPLIED ="Please enter value for AWS_ACCESS_KEY in settings/local.py file" -AWS_SECRET_KEY_NOT_SUPPLIED ="Please enter value for AWS_SECRET_KEY in settings/local.py file" -AWS_REGION_NOT_SUPPLIED ="Please enter value for AWS_REGION in settings/local.py file" -AWS_ASSUME_ROLE_NOT_SUPPLIED ="Please enter value for AWS_ASSUME_ROLE_ARN in settings/local.py file" +AWS_ACCESS_KEY_NOT_SUPPLIED = "Please enter value for AWS_ACCESS_KEY in settings/local.py file" +AWS_SECRET_KEY_NOT_SUPPLIED = "Please enter value for AWS_SECRET_KEY in settings/local.py file" +AWS_REGION_NOT_SUPPLIED = "Please enter value for AWS_REGION in settings/local.py file" +AWS_ASSUME_ROLE_NOT_SUPPLIED = "Please enter value for AWS_ASSUME_ROLE_ARN in settings/local.py file" AWS_ACCESS_KEY_INPUT = "Please enter AWS access key: " AWS_SECRET_KEY_INPUT = "Please enter AWS secret key: " diff --git a/installer/core/providers/aws/input.py b/installer/core/providers/aws/input.py index 6808110c5..27d280f7c 100644 --- a/installer/core/providers/aws/input.py +++ b/installer/core/providers/aws/input.py @@ -11,7 +11,7 @@ class SystemInput(MsgMixin, metaclass=ABCMeta): """Base input class for installation/destruction/status commands. This class reads required input from user for the process to start""" AWS_AUTH_CRED = {} - def __init__(self, silent_install = False): + def __init__(self, silent_install=False): self.silent_install = silent_install def read_input(self): @@ -31,7 +31,6 @@ def read_input(self): self.AWS_AUTH_CRED['aws_region'] ) - Settings.set('AWS_AUTH_CRED', self.AWS_AUTH_CRED) self.load_aws_account_details() diff --git a/installer/core/providers/aws/reinstall.py b/installer/core/providers/aws/reinstall.py index efef7d0c9..eab7f2c90 100644 --- a/installer/core/providers/aws/reinstall.py +++ b/installer/core/providers/aws/reinstall.py @@ -81,7 +81,7 @@ def re_create_resources(self, resources_to_destroy, resources_to_install, terraf except Exception as e: self.executed_with_error = True self.exception = e - self.destroy = True #If there is any error in destroy set destroy to True + self.destroy = True # If there is any error in destroy set destroy to True self._cleanup_installation_process(dry_run) diff --git a/installer/files/scripts/utils.py b/installer/files/scripts/utils.py index 1aab5cf0b..c73c6092a 100644 --- a/installer/files/scripts/utils.py +++ b/installer/files/scripts/utils.py @@ -49,7 +49,6 @@ def prepare_aws_client_with_given_aws_details(service_name, aws_details): auth_data['aws_secret_access_key'] = temp_cred['SecretAccessKey'] auth_data['aws_session_token'] = temp_cred['SessionToken'] - return boto3.client(service_name, **auth_data) @@ -66,7 +65,6 @@ def prepare_aws_resource_with_given_aws_details(service_name, aws_details): auth_data['aws_secret_access_key'] = temp_cred['SecretAccessKey'] auth_data['aws_session_token'] = temp_cred['SessionToken'] - return boto3.resource(service_name, **auth_data) diff --git a/installer/resources/lambda_submit/function.py b/installer/resources/lambda_submit/function.py index 566828b52..1cb6e5514 100644 --- a/installer/resources/lambda_submit/function.py +++ b/installer/resources/lambda_submit/function.py @@ -346,4 +346,4 @@ class AzureDataShipperCloudWatchEventTarget(CloudWatchEventTargetResource): {'encrypt': False, 'key': "s3.data", 'value': "azure-inventory"} ] }) - PROCESS = need_to_enable_azure() \ No newline at end of file + PROCESS = need_to_enable_azure() diff --git a/installer/resources/pacbot_app/files/DB.sql b/installer/resources/pacbot_app/files/DB.sql index c1442978f..6e1eaf25f 100644 --- a/installer/resources/pacbot_app/files/DB.sql +++ b/installer/resources/pacbot_app/files/DB.sql @@ -49,7 +49,6 @@ SET @JOB_LAMBDA_REGION='$JOB_LAMBDA_REGION'; SET @JOB_FUNCTION_NAME='$JOB_FUNCTION_NAME'; SET @JOB_FUNCTION_ARN='$JOB_FUNCTION_ARN'; SET @RULE_BUCKET_REGION='$RULE_BUCKET_REGION'; -SET @RULE_JOB_BUCKET_NAME='$RULE_JOB_BUCKET_NAME'; SET @RULE_LAMBDA_REGION='$RULE_LAMBDA_REGION'; SET @RULE_FUNCTION_NAME='$RULE_FUNCTION_NAME'; SET @RULE_FUNCTION_ARN='$RULE_FUNCTION_ARN'; diff --git a/installer/resources/pacbot_app/import_db.py b/installer/resources/pacbot_app/import_db.py index 44ad65cf4..451727f76 100644 --- a/installer/resources/pacbot_app/import_db.py +++ b/installer/resources/pacbot_app/import_db.py @@ -79,7 +79,6 @@ def get_provisioners(self): 'ENV_JOB_FUNCTION_NAME': SubmitJobLambdaFunction.get_input_attr('function_name'), 'ENV_JOB_FUNCTION_ARN': SubmitJobLambdaFunction.get_output_attr('arn'), 'ENV_RULE_BUCKET_REGION': AwsRegion.get_output_attr('name'), - 'ENV_RULE_JOB_BUCKET_NAME': BucketStorage.get_output_attr('bucket'), 'ENV_RULE_LAMBDA_REGION': AwsRegion.get_output_attr('name'), 'ENV_RULE_FUNCTION_NAME': RuleEngineLambdaFunction.get_input_attr('function_name'), 'ENV_RULE_FUNCTION_ARN': RuleEngineLambdaFunction.get_output_attr('arn'), diff --git a/installer/resources/pacbot_app/utils.py b/installer/resources/pacbot_app/utils.py index 3ec68955f..4617ade37 100644 --- a/installer/resources/pacbot_app/utils.py +++ b/installer/resources/pacbot_app/utils.py @@ -12,6 +12,7 @@ def need_to_enable_azure(): return feature_status + def get_azure_tenants(): if need_to_enable_azure(): tenants = Settings.get('AZURE_TENANTS', []) From 1728c879c6461c318b4238c1a16bf823f595e5ed Mon Sep 17 00:00:00 2001 From: Kanchana Date: Tue, 3 Dec 2019 19:32:12 +0530 Subject: [PATCH 105/107] Added policy Evaluation rule --- .../cloud/awsrules/utils/PacmanUtils.java | 45 +++++++++ .../policies/AzurePolicyEvaluationRule.java | 94 +++++++++++++++++++ .../cloud/constants/PacmanRuleConstants.java | 2 + 3 files changed, 141 insertions(+) create mode 100644 jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/azurerules/policies/AzurePolicyEvaluationRule.java diff --git a/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/awsrules/utils/PacmanUtils.java b/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/awsrules/utils/PacmanUtils.java index 73bf004b2..e029c5526 100644 --- a/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/awsrules/utils/PacmanUtils.java +++ b/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/awsrules/utils/PacmanUtils.java @@ -2956,5 +2956,50 @@ public static Map checkResourceIdBypolicyName(String esUrl, Map< } return secMap; } + + /** + * Function for creating the rule list of a particular virtual machine with + * resource id + * + * @param esUrl + * @param resourceId + * @param policyDefinitionName + * @return + * @throws Exception + */ + public static Map getAzurePolicyEvaluationResults(String esUrl, String resourceId, + String policyDefinitionName) throws Exception { + + JsonParser jsonParser = new JsonParser(); + Map policyEvaluationResultsMap = new HashMap<>(); + Map mustFilter = new HashMap(); + mustFilter.put(convertAttributetoKeyword("resourceIdLower"), resourceId); + mustFilter.put(convertAttributetoKeyword("policyDefinitionName"), policyDefinitionName); + mustFilter.put(PacmanRuleConstants.LATEST, "true"); + JsonObject resultJson = RulesElasticSearchRepositoryUtil.getQueryDetailsFromES(esUrl, mustFilter, null, null, + null, 0, null, null, null); + if (resultJson != null && resultJson.has(PacmanRuleConstants.HITS)) { + String hitsJsonString = resultJson.get(PacmanRuleConstants.HITS).toString(); + JsonObject hitsJson = (JsonObject) jsonParser.parse(hitsJsonString); + JsonArray jsonArray = hitsJson.getAsJsonObject().get(PacmanRuleConstants.HITS).getAsJsonArray(); + if (jsonArray.size() > 0) { + for (int i = 0; i < jsonArray.size(); i++) { + JsonObject firstObject = (JsonObject) jsonArray.get(i); + JsonObject sourceJson = (JsonObject) firstObject.get(PacmanRuleConstants.SOURCE); + if (null != sourceJson) { + boolean isCompliant = sourceJson.get("isCompliant").getAsBoolean(); + policyEvaluationResultsMap.put("isCompliant", isCompliant); + policyEvaluationResultsMap.put("policyName", sourceJson.get("policyName").getAsString()); + policyEvaluationResultsMap.put("policyDescription", + sourceJson.get("policyDescription")); + + } + + } + } + } + return policyEvaluationResultsMap; + } + } diff --git a/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/azurerules/policies/AzurePolicyEvaluationRule.java b/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/azurerules/policies/AzurePolicyEvaluationRule.java new file mode 100644 index 000000000..951396b0c --- /dev/null +++ b/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/azurerules/policies/AzurePolicyEvaluationRule.java @@ -0,0 +1,94 @@ +package com.tmobile.cloud.azurerules.policies; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; + +import com.tmobile.cloud.awsrules.utils.PacmanUtils; +import com.tmobile.cloud.constants.PacmanRuleConstants; +import com.tmobile.pacman.commons.PacmanSdkConstants; +import com.tmobile.pacman.commons.exception.RuleExecutionFailedExeption; +import com.tmobile.pacman.commons.rule.Annotation; +import com.tmobile.pacman.commons.rule.BaseRule; +import com.tmobile.pacman.commons.rule.PacmanRule; +import com.tmobile.pacman.commons.rule.RuleResult; + +/** + * Possible network Just In Time (JIT) access will be monitored by Azure + * Security Center as recommendations + */ + +@PacmanRule(key = "check-for-azure-policy-evaluation-results", desc = "Azure policy evaluation results for different target types", severity = PacmanSdkConstants.SEV_HIGH, category = PacmanSdkConstants.SECURITY) +public class AzurePolicyEvaluationRule extends BaseRule { + + private static final Logger logger = LoggerFactory.getLogger(AzurePolicyEvaluationRule.class); + + + @Override + public RuleResult execute(Map ruleParam, Map resourceAttributes) { + logger.debug("======== Azure Policy Evaluation Rule started ========="); + + MDC.put("executionId", ruleParam.get("executionId")); + MDC.put("ruleId", ruleParam.get(PacmanSdkConstants.RULE_ID)); + + String severity = ruleParam.get(PacmanRuleConstants.SEVERITY); + String category = ruleParam.get(PacmanRuleConstants.CATEGORY); + + String resourceId = resourceAttributes.get(PacmanRuleConstants.RESOURCE_ID).toLowerCase(); + String pacmanHost = PacmanUtils.getPacmanHost(PacmanRuleConstants.ES_URI); + String policyDefinitionName = ruleParam.get("policyDefinitionName"); + String azurePolicyEvaluationResultsURl = ruleParam.get("azurePolicyEvaluationResults"); + + Map policyEvaluationResultsMap = new HashMap<>(); + try { + + policyEvaluationResultsMap = PacmanUtils.getAzurePolicyEvaluationResults( + pacmanHost + azurePolicyEvaluationResultsURl, resourceId, policyDefinitionName); + if (!policyEvaluationResultsMap.isEmpty()) { + boolean isCompliant = (boolean) policyEvaluationResultsMap.get("isCompliant"); + if (!isCompliant == true) { + List> issueList = new ArrayList<>(); + LinkedHashMap issue = new LinkedHashMap<>(); + Annotation annotation = null; + annotation = Annotation.buildAnnotation(ruleParam, Annotation.Type.ISSUE); + annotation.put(PacmanSdkConstants.DESCRIPTION, + policyEvaluationResultsMap.get("policyDescription").toString()); + annotation.put(PacmanRuleConstants.SEVERITY, severity); + annotation.put(PacmanRuleConstants.CATEGORY, category); + annotation.put(PacmanRuleConstants.AZURE_SUBSCRIPTION, resourceAttributes.get(PacmanRuleConstants.AZURE_SUBSCRIPTION)); + annotation.put(PacmanRuleConstants.AZURE_SUBSCRIPTION_NAME, resourceAttributes.get(PacmanRuleConstants.AZURE_SUBSCRIPTION_NAME)); + issue.put("resourceId", resourceId); + issue.put("policyDescription", policyEvaluationResultsMap.get("policyDescription").toString()); + issue.put("policyName", policyEvaluationResultsMap.get("policyName").toString()); + issueList.add(issue); + annotation.put(PacmanRuleConstants.ISSUE_DETAILS, issueList.toString()); + logger.debug( + "======== Azure Policy Evaluation Rule ended with annotation {} : =========", + annotation); + return new RuleResult(PacmanSdkConstants.STATUS_FAILURE, PacmanRuleConstants.FAILURE_MESSAGE, + annotation); + + } + } + + } catch (Exception exception) { + logger.error("error: ", exception); + throw new RuleExecutionFailedExeption(exception.getMessage()); + } + + logger.debug("======== Azure Policy Evaluation Rule ended========="); + return new RuleResult(PacmanSdkConstants.STATUS_SUCCESS, PacmanRuleConstants.SUCCESS_MESSAGE); + } + + @Override + public String getHelpText() { + return "Azure Policy Evaluation Rule "; + } + +} diff --git a/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/constants/PacmanRuleConstants.java b/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/constants/PacmanRuleConstants.java index 68a02ce34..a329658bc 100644 --- a/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/constants/PacmanRuleConstants.java +++ b/jobs/pacman-awsrules/src/main/java/com/tmobile/cloud/constants/PacmanRuleConstants.java @@ -359,4 +359,6 @@ private PacmanRuleConstants() { public static final String AZURERESOURCEID= "recommendation._resourceIdLower"; public static final String RECOMMENDATION = "recommendation"; public static final String DETAILS = "details"; + public static final String AZURE_SUBSCRIPTION = "subscription"; + public static final String AZURE_SUBSCRIPTION_NAME = "subscriptionName"; } From 3c4248c25a61f0f9ad498464d6ef36a1a3875651 Mon Sep 17 00:00:00 2001 From: Kanchana Date: Wed, 4 Dec 2019 15:55:15 +0530 Subject: [PATCH 106/107] Policy evaluation rules ported --- .../files/rule_engine_cloudwatch_rules.json | 597 +++++++++++++++++- installer/resources/pacbot_app/files/DB.sql | 58 ++ 2 files changed, 653 insertions(+), 2 deletions(-) diff --git a/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json b/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json index 1db6480f4..4a1b160b2 100644 --- a/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json +++ b/installer/resources/lambda_rule_engine/files/rule_engine_cloudwatch_rules.json @@ -2968,6 +2968,599 @@ "modifiedDate": "2019-10-25", "severity": "high", "category": "security" - } - + }, + { + "ruleId": "PacMan_Access_Through_Internet_version-1_AIE_virtualmachine", + "ruleUUID": "azure_aie_virtualmachine", + "policyId": "PacMan_Access_Through_Internet_version-1", + "ruleName": "AIE", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "AIE", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"9daedab3-fb2d-461e-b861-71790eead4f6\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Access_Through_Internet_version-1_AIE_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"AIE\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Access_Through_Internet_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_aie_virtualmachine\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Access through Internet facing endpoint should be restricted.", + "createdDate": "2019-09-26", + "modifiedDate": "2019-11-23", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Adaptive_App_Control_version-1_AAC_virtualmachine", + "ruleUUID": "azure_aac_virtualmachine", + "policyId": "PacMan_Adaptive_App_Control_version-1", + "ruleName": "AAC", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "AdaptiveAppControl", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"47a6b606-51aa-4496-8bb7-64b11cf66adc\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"medium\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Adaptive_App_Control_version-1_AAC_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"AdaptiveAppControl\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Adaptive_App_Control_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_aac_virtualmachine\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Adaptive Application Controls should be enabled on virtual machines.", + "createdDate": "2019-08-27", + "modifiedDate": "2019-11-06", + "severity": "medium", + "category": "security" + }, + { + "ruleId": "PacMan_Auditing_Advanced_Data_Security_version-1_SADS_sqlserver", + "ruleUUID": "azure_sads_sqlserver", + "policyId": "PacMan_Auditing_Advanced_Data_Security_version-1", + "ruleName": "SADS", + "targetType": "sqlserver", + "assetGroup": "azure", + "alexaKeyword": "SADS", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"a6fb4358-5bf4-4ad7-ba82-2cd2f41ce5e9\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"medium\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Auditing_Advanced_Data_Security_version-1_SADS_sqlserver\",\"autofix\":false,\"alexaKeyword\":\"SADS\",\"ruleRestUrl\":\"\",\"targetType\":\"sqlserver\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Auditing_Advanced_Data_Security_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_sads_sqlserver\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Advanced Data Security SQL Server should have auditing enabled.", + "createdDate": "2019-09-26", + "modifiedDate": "2019-11-06", + "severity": "medium", + "category": "security" + }, + { + "ruleId": "PacMan_AzureAuditNetworkAccessToStorageAccounts_version-1_ConfigureNetworkRulesOnStorageAccount_storageaccount", + "ruleUUID": "azure_ConfigureNetworkRules_storageaccount", + "policyId": "PacMan_AzureAuditNetworkAccessToStorageAccounts_version-1", + "ruleName": "ConfigureNetworkRulesOnStorageAccount", + "targetType": "storageaccount", + "assetGroup": "azure", + "alexaKeyword": "ConfigureNetworkRulesOnStorageAccount", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"34c877ad-507e-4c82-993e-3452a6e0ad3c\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_AzureAuditNetworkAccessToStorageAccounts_version-1_ConfigureNetworkRulesOnStorageAccount_storageaccount\",\"autofix\":false,\"alexaKeyword\":\"ConfigureNetworkRulesOnStorageAccount\",\"ruleRestUrl\":\"\",\"targetType\":\"storageaccount\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_AzureAuditNetworkAccessToStorageAccounts_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_ConfigureNetworkRules_storageaccount\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Audit unrestricted network access to storage accounts.", + "createdDate": "2019-11-08", + "modifiedDate": "2019-11-23", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_AzureAuditStorageVnetIntegration_version-1_AuditStorageVnetIntegration_storageaccount", + "ruleUUID": "azure_AuditStorageVnetInt_storageaccount", + "policyId": "PacMan_AzureAuditStorageVnetIntegration_version-1", + "ruleName": "AuditStorageVnetIntegration", + "targetType": "storageaccount", + "assetGroup": "azure", + "alexaKeyword": "AuditStorageVnetIntegration", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"deb3d6f0-8113-4ed4-8492-d64f90919223\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_AzureAuditStorageVnetIntegration_version-1_AuditStorageVnetIntegration_storageaccount\",\"autofix\":false,\"alexaKeyword\":\"AuditStorageVnetIntegration\",\"ruleRestUrl\":\"\",\"targetType\":\"storageaccount\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_AzureAuditStorageVnetIntegration_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_AuditStorageVnetInt_storageaccount\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Audit Storage VNet Integration.", + "createdDate": "2019-11-08", + "modifiedDate": "2019-11-23", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_AzureAuditTransferToStorageAccounts_version-1_EnableSecureTransferToStorageAccounts_storageaccount", + "ruleUUID": "azure_EnableSecureTransfer_storageaccount", + "policyId": "PacMan_AzureAuditTransferToStorageAccounts_version-1", + "ruleName": "EnableSecureTransferToStorageAccounts", + "targetType": "storageaccount", + "assetGroup": "azure", + "alexaKeyword": "EnableSecureTransferToStorageAccounts", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"404c3081-a854-4457-ae30-26a93ef643f9\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_AzureAuditTransferToStorageAccounts_version-1_EnableSecureTransferToStorageAccounts_storageaccount\",\"autofix\":false,\"alexaKeyword\":\"EnableSecureTransferToStorageAccounts\",\"ruleRestUrl\":\"\",\"targetType\":\"storageaccount\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_AzureAuditTransferToStorageAccounts_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_EnableSecureTransfer_storageaccount\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Secure transfer to storage accounts should be enabled.", + "createdDate": "2019-11-08", + "modifiedDate": "2019-11-23", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_AzureBatchAccountMetricAlertRules_version-1_ConfigureMetricAlertRules_batchaccounts", + "ruleUUID": "azure_ConfigureMetric_batchaccounts", + "policyId": "PacMan_AzureBatchAccountMetricAlertRules_version-1", + "ruleName": "ConfigureMetricAlertRules", + "targetType": "batchaccounts", + "assetGroup": "azure", + "alexaKeyword": "ConfigureMetricAlertRules", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"26ee67a2-f81a-4ba8-b9ce-8550bd5ee1a7\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_AzureBatchAccountMetricAlertRules_version-1_ConfigureMetricAlertRules_batchaccounts\",\"autofix\":false,\"alexaKeyword\":\"ConfigureMetricAlertRules\",\"ruleRestUrl\":\"\",\"targetType\":\"batchaccounts\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_AzureBatchAccountMetricAlertRules_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_ConfigureMetric_batchaccounts\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Metric alert rules should be configured on Batch accounts.", + "createdDate": "2019-11-08", + "modifiedDate": "2019-11-23", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_AzureBatchAccountsDiagnosticLogs_version-1_EnableDiagnosticLogsInBatchAccount_batchaccounts", + "ruleUUID": "azure_EnableDiagnosticLogs_batchaccounts", + "policyId": "PacMan_AzureBatchAccountsDiagnosticLogs_version-1", + "ruleName": "EnableDiagnosticLogsInBatchAccount", + "targetType": "batchaccounts", + "assetGroup": "azure", + "alexaKeyword": "EnableDiagnosticLogsInBatchAccount", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"428256e6-1fac-4f48-a757-df34c2b3336d\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_AzureBatchAccountsDiagnosticLogs_version-1_EnableDiagnosticLogsInBatchAccount_batchaccounts\",\"autofix\":false,\"alexaKeyword\":\"EnableDiagnosticLogsInBatchAccount\",\"ruleRestUrl\":\"\",\"targetType\":\"batchaccounts\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_AzureBatchAccountsDiagnosticLogs_version-1\",\"assetGroup\":null,\"ruleUUID\":\"azure_EnableDiagnosticLogs_batchaccounts\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Diagnostic logs in Batch accounts should be enabled.", + "createdDate": "2019-11-08", + "modifiedDate": "2019-11-23", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_AzureNamespacesDiagnosticLogs_version-1_EnableDiagnosticLogsInServiceBus_namespaces", + "ruleUUID": "azure_EnableDiagnosticLogs_namespaces", + "policyId": "PacMan_AzureNamespacesDiagnosticLogs_version-1", + "ruleName": "EnableDiagnosticLogsInServiceBus", + "targetType": "namespaces", + "assetGroup": "azure", + "alexaKeyword": "EnableDiagnosticLogsInServiceBus", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"f8d36e2f-389b-4ee4-898d-21aeb69a0f45\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_AzureNamespacesDiagnosticLogs_version-1_EnableDiagnosticLogsInServiceBus_namespaces\",\"autofix\":false,\"alexaKeyword\":\"EnableDiagnosticLogsInServiceBus\",\"ruleRestUrl\":\"\",\"targetType\":\"namespaces\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_AzureNamespacesDiagnosticLogs_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_EnableDiagnosticLogs_namespaces\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Diagnostic logs in Service Bus should be enabled.", + "createdDate": "2019-11-08", + "modifiedDate": "2019-11-23", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_AzureSQLAuditLogging_version-1_ConfigureActionGroupsInSQLAuditSettings_sqlserver", + "ruleUUID": "azure_ConfigureActionGroups_sqlserver", + "policyId": "PacMan_AzureSQLAuditLogging_version-1", + "ruleName": "ConfigureActionGroupsInSQLAuditSettings", + "targetType": "sqlserver", + "assetGroup": "azure", + "alexaKeyword": "ConfigureActionGroupsInSQLAuditSettings", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"7ff426e2-515f-405a-91c8-4f2333442eb5\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"governance\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_AzureSQLAuditLogging_version-1_ConfigureActionGroupsInSQLAuditSettings_sqlserver\",\"autofix\":false,\"alexaKeyword\":\"ConfigureActionGroupsInSQLAuditSettings\",\"ruleRestUrl\":\"\",\"targetType\":\"sqlserver\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_AzureSQLAuditLogging_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_ConfigureActionGroups_sqlserver\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "SQL Auditing settings should have Action-Groups configured to capture critical activities.", + "createdDate": "2019-11-08", + "modifiedDate": "2019-11-23", + "severity": "high", + "category": "governance" + }, + { + "ruleId": "PacMan_AzureSQLServerAuthenticationSettings_version-1_EnableAzureADAuthentication_sqlserver", + "ruleUUID": "azure_EnableAzureADAuth_sqlserver", + "policyId": "PacMan_AzureSQLServerAuthenticationSettings_version-1", + "ruleName": "EnableAzureADAuthentication", + "targetType": "sqlserver", + "assetGroup": "azure", + "alexaKeyword": "EnableAzureADAuthentication", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"1f314764-cb73-4fc9-b863-8eca98ac36e9\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_AzureSQLServerAuthenticationSettings_version-1_EnableAzureADAuthentication_sqlserver\",\"autofix\":false,\"alexaKeyword\":\"EnableAzureADAuthentication\",\"ruleRestUrl\":\"\",\"targetType\":\"sqlserver\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_AzureSQLServerAuthenticationSettings_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_EnableAzureADAuth_sqlserver\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "An Active Directory administrator should be provisioned for SQL Servers.", + "createdDate": "2019-11-08", + "modifiedDate": "2019-11-09", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Azure_SQLServerDataSecuritySettings_version-1_ConfigureEmailInDataSecuritySettings_sqlserver", + "ruleUUID": "azure_ConfigureEmailInDataSecurity_sqlserver", + "policyId": "PacMan_Azure_SQLServerDataSecuritySettings_version-1", + "ruleName": "ConfigureEmailInDataSecuritySettings", + "targetType": "sqlserver", + "assetGroup": "azure", + "alexaKeyword": "ConfigureEmailInDSS", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"9677b740-f641-4f3c-b9c5-466005c85278\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_SQLServerDataSecuritySettings_version-1_ConfigureEmailInDataSecuritySettings_sqlserver\",\"autofix\":false,\"alexaKeyword\":\"ConfigureEmailInDSS\",\"ruleRestUrl\":\"\",\"targetType\":\"sqlserver\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_SQLServerDataSecuritySettings_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_ConfigureEmailInDataSecurity_sqlserver\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0 1/1 * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Configure Email In Data Security Settings.", + "createdDate": "2019-11-14", + "modifiedDate": "2019-11-23", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Azure_policies-Container_Security_Configurations_version-1_CSC_virtualmachine", + "ruleUUID": "azure_CSC_virtualmachine", + "policyId": "PacMan_Azure_policies-Container_Security_Configurations_version-1", + "ruleName": "CSC", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "Container security", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"e8cbc669-f12d-49eb-93e7-9273119e9933\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_policies-Container_Security_Configurations_version-1_CSC_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"Container security\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_policies-Container_Security_Configurations_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_CSC_virtualmachine\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Container Security Configuration.", + "createdDate": "2019-09-26", + "modifiedDate": "2019-11-06", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Azure_policies-Disk_Encryption_version-1_DE_virtualmachine", + "ruleUUID": "azure_DiskEncryption_virtualmachine", + "policyId": "PacMan_Azure_policies-Disk_Encryption_version-1", + "ruleName": "DE", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "AzurepoliciesDiskEncryption", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"0961003e-5a0a-4549-abde-af6a37f2724d\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"medium\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_policies-Disk_Encryption_version-1_DE_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"AzurepoliciesDiskEncryption\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_policies-Disk_Encryption_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_DiskEncryption_virtualmachine\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "VMs w/o enabled disk encryption will be monitored by Security Center as recommendations.", + "createdDate": "2019-08-27", + "modifiedDate": "2019-11-09", + "severity": "medium", + "category": "security" + }, + { + "ruleId": "PacMan_Azure_policies-JIT_Network_Access_version-1_JIT_virtualmachine", + "ruleUUID": "azure_JIT_virtualmachine", + "policyId": "PacMan_Azure_policies-JIT_Network_Access_version-1", + "ruleName": "JIT", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "AzurepoliciesJITNetworkAccess", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"b0f33259-77d7-4c9e-aac6-3aabcfae693c\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_policies-JIT_Network_Access_version-1_JIT_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"AzurepoliciesJITNetworkAccess\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_policies-JIT_Network_Access_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_JIT_virtualmachine\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Just-In-Time network access control should be applied on Virtual Machines.", + "createdDate": "2019-08-27", + "modifiedDate": "2019-11-06", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Data_Encryption_SQL_version-1_SDE_sqldatabase", + "ruleUUID": "azure_SDE_sqldatabase", + "policyId": "PacMan_Data_Encryption_SQL_version-1", + "ruleName": "SDE", + "targetType": "sqldatabase", + "assetGroup": "azure", + "alexaKeyword": "SDE", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"17k78e20-9358-41c9-923c-fb736d382a12\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Data_Encryption_SQL_version-1_SDE_sqldatabase\",\"autofix\":false,\"alexaKeyword\":\"SDE\",\"ruleRestUrl\":\"\",\"targetType\":\"sqldatabase\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Data_Encryption_SQL_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_SDE_sqldatabase\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Transparent Data Encryption on SQL databases should be enabled.", + "createdDate": "2019-09-26", + "modifiedDate": "2019-11-07", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_IP_Forwarding_version-1_IPF_virtualmachine", + "ruleUUID": "azure_IPF_virtualmachine", + "policyId": "PacMan_IP_Forwarding_version-1", + "ruleName": "IPF", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "IPF", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"bd352bd5-2853-4985-bf0d-73806b4a5744\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_IP_Forwarding_version-1_IPF_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"IPF\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_IP_Forwarding_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_IPF_virtualmachine\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "IP Forwarding on Virtual Machines should be disabled.", + "createdDate": "2019-09-26", + "modifiedDate": "2019-11-06", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Missing_Endpoint_Protection_version-1_MEP_virtualmachine", + "ruleUUID": "azure_MEP_virtualmachine", + "policyId": "PacMan_Missing_Endpoint_Protection_version-1", + "ruleName": "MEP", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "MEP", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"af6cd1bd-1635-48cb-bde7-5b15693900b9\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Missing_Endpoint_Protection_version-1_MEP_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"MEP\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Missing_Endpoint_Protection_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_MEP_virtualmachine\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Monitor missing Endpoint Protection in Security Center.", + "createdDate": "2019-09-26", + "modifiedDate": "2019-11-09", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Namespaces_DiagnosticLogs_version-1_NDL_namespaces", + "ruleUUID": "azure_NDL_namespaces", + "policyId": "PacMan_Namespaces_DiagnosticLogs_version-1", + "ruleName": "NDL", + "targetType": "namespaces", + "assetGroup": "azure", + "alexaKeyword": "NDL", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"83a214f7-d01a-484b-91a9-ed54470c9a6a\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"medium\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Namespaces_DiagnosticLogs_version-1_NDL_namespaces\",\"autofix\":false,\"alexaKeyword\":\"NDL\",\"ruleRestUrl\":\"\",\"targetType\":\"namespaces\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Namespaces_DiagnosticLogs_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_NDL_namespaces\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Diagnostic logs in namespaces should be enabled.", + "createdDate": "2019-09-30", + "modifiedDate": "2019-11-23", + "severity": "medium", + "category": "security" + }, + { + "ruleId": "PacMan_SQL_Auditing_Retention_version-1_SAR_sqlserver", + "ruleUUID": "azure_SAR_sqlserver", + "policyId": "PacMan_SQL_Auditing_Retention_version-1", + "ruleName": "SAR", + "targetType": "sqlserver", + "assetGroup": "azure", + "alexaKeyword": "SAR", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"89099bee-89e0-4b26-a5f4-165451757743\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_SQL_Auditing_Retention_version-1_SAR_sqlserver\",\"autofix\":false,\"alexaKeyword\":\"SAR\",\"ruleRestUrl\":\"\",\"targetType\":\"sqlserver\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_SQL_Auditing_Retention_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_SAR_sqlserver\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "SQL Auditing for configured number of retention days.", + "createdDate": "2019-09-30", + "modifiedDate": "2019-11-07", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_SQL_TDE_Encryption_version-1_TPS_sqlserver", + "ruleUUID": "azure_TPS_sqlserver", + "policyId": "PacMan_SQL_TDE_Encryption_version-1", + "ruleName": "TPS", + "targetType": "sqlserver", + "assetGroup": "azure", + "alexaKeyword": "TPS", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"0d134df8-db83-46fb-ad72-fe0c9428c8dd\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_SQL_TDE_Encryption_version-1_TPS_sqlserver\",\"autofix\":false,\"alexaKeyword\":\"TPS\",\"ruleRestUrl\":\"\",\"targetType\":\"sqlserver\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_SQL_TDE_Encryption_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_TPS_sqlserver\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "SQL TDE protector encryption check.", + "createdDate": "2019-09-30", + "modifiedDate": "2019-11-07", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_SQL_Vulnerability_Assessment_version-1_SVA_sqlserver", + "ruleUUID": "azure_SVA_sqlserver", + "policyId": "PacMan_SQL_Vulnerability_Assessment_version-1", + "ruleName": "SVA", + "targetType": "sqlserver", + "assetGroup": "azure", + "alexaKeyword": "SVA", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"ef2a8f2a-b3d9-49cd-a8a8-9a3aaaf647d9\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_SQL_Vulnerability_Assessment_version-1_SVA_sqlserver\",\"autofix\":false,\"alexaKeyword\":\"SVA\",\"ruleRestUrl\":\"\",\"targetType\":\"sqlserver\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_SQL_Vulnerability_Assessment_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_SVA_sqlserver\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Vulnerability assessment should be enabled on SQL servers.", + "createdDate": "2019-09-30", + "modifiedDate": "2019-11-07", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Sensitive_Data_SQL_version-1_SDS_sqldatabase", + "ruleUUID": "azure_SDS_sqldatabase", + "policyId": "PacMan_Sensitive_Data_SQL_version-1", + "ruleName": "SDS", + "targetType": "sqldatabase", + "assetGroup": "azure", + "alexaKeyword": "SDS", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"cc9835f2-9f6b-4cc8-ab4a-f8ef615eb349\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Sensitive_Data_SQL_version-1_SDS_sqldatabase\",\"autofix\":false,\"alexaKeyword\":\"SDS\",\"ruleRestUrl\":\"\",\"targetType\":\"sqldatabase\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Sensitive_Data_SQL_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_SDS_sqldatabase\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Sensitive data in SQL databases should be classified.", + "createdDate": "2019-09-26", + "modifiedDate": "2019-11-07", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Subnet_NSG_rule_version-1_SNR_subnets", + "ruleUUID": "azure_SNR_subnets", + "policyId": "PacMan_Subnet_NSG_rule_version-1", + "ruleName": "SNR", + "targetType": "subnets", + "assetGroup": "azure", + "alexaKeyword": "SNR", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"e71308d3-144b-4262-b144-efdc3cc90517\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Subnet_NSG_rule_version-1_SNR_subnets\",\"autofix\":false,\"alexaKeyword\":\"SNR\",\"ruleRestUrl\":\"\",\"targetType\":\"subnets\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Subnet_NSG_rule_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_SNR_subnets\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Subnets should be associated with a Network Security Group.", + "createdDate": "2019-09-30", + "modifiedDate": "2019-11-07", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_System_Updates_version-1_SSU_virtualmachine", + "ruleUUID": "azure_SSU_virtualmachine", + "policyId": "PacMan_System_Updates_version-1", + "ruleName": "SSU", + "targetType": "virtualmachine", + "assetGroup": "azure", + "alexaKeyword": "SSU", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"86b3d65f-7626-441e-b690-81a8b71cff60\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_System_Updates_version-1_SSU_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"SSU\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_System_Updates_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_SSU_virtualmachine\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "System updates should be installed on Virtual Machines.", + "createdDate": "2019-09-26", + "modifiedDate": "2019-11-07", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Vaults_DiagnosticLogs_version-1_VDR_vaults", + "ruleUUID": "azure_VDR_vaults", + "policyId": "PacMan_Vaults_DiagnosticLogs_version-1", + "ruleName": "VDR", + "targetType": "vaults", + "assetGroup": "azure", + "alexaKeyword": "VDR", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"cf820ca0-f99e-4f3e-84fb-66e913812d21\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Vaults_DiagnosticLogs_version-1_VDR_vaults\",\"autofix\":false,\"alexaKeyword\":\"VDR\",\"ruleRestUrl\":\"\",\"targetType\":\"vaults\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Vaults_DiagnosticLogs_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_VDR_vaults\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Diagnostic logs in Key Vault should be enabled.", + "createdDate": "2019-09-30", + "modifiedDate": "2019-11-06", + "severity": "high", + "category": "security" + }, + { + "ruleId": "PacMan_Workflows_DiagnlsticLogs_version-1_WDR_workflows", + "ruleUUID": "azure_WDR_workflows", + "policyId": "PacMan_Workflows_DiagnlsticLogs_version-1", + "ruleName": "WDR", + "targetType": "workflows", + "assetGroup": "azure", + "alexaKeyword": "WDR", + "ruleParams": "{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"34f95f76-5386-4de7-b824-0d8478470c9d\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Workflows_DiagnlsticLogs_version-1_WDR_workflows\",\"autofix\":false,\"alexaKeyword\":\"WDR\",\"ruleRestUrl\":\"\",\"targetType\":\"workflows\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Workflows_DiagnlsticLogs_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_WDR_workflows\",\"ruleType\":\"ManageRule\"}", + "ruleFrequency": "0 0/6 * * ? *", + "ruleExecutable": "", + "ruleRestUrl": "", + "ruleType": "ManageRule", + "ruleArn": null, + "status": "ENABLED", + "userId": "", + "displayName": "Diagnostic logs in Logic Apps should be enabled.", + "createdDate": "2019-09-30", + "modifiedDate": "2019-11-06", + "severity": "high", + "category": "security" + } ] diff --git a/installer/resources/pacbot_app/files/DB.sql b/installer/resources/pacbot_app/files/DB.sql index 6e1eaf25f..d6b7e06d1 100644 --- a/installer/resources/pacbot_app/files/DB.sql +++ b/installer/resources/pacbot_app/files/DB.sql @@ -1268,6 +1268,34 @@ INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_Azure_Remediate_vulnerabilities_version-1','Remediate vulnerabilities - by a Vulnerability Assessment solution','This is Azure security rule',"",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_Azure_Resolve_monitoring_agent_version-1','Resolve monitoring agent health issues on your machines','This is Azure Secuirty Rule',"",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); INSERT IGNORE INTO `cf_Policy` (`policyId`,`policyName`,`policyDesc`,`resolution`,`policyUrl`,`policyVersion`,`status`,`userId`,`createdDate`,`modifiedDate`) VALUES ('PacMan_Azure_harden-NSGs_internet_version-1','Harden Network Security Group rules of internet facing Virtual Machines','This is Azure Secuirty Rule',"",NULL,'version-1','fed',NULL,'2019-08-05','2019-08-05'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_Access_Through_Internet_version-1','Access_Through_Internet','Inbound rules should not allow access from Any or Internet ranges','Access through Internet-facing endpoint should be restricted.',NULL,'version-1',NULL,NULL,'2019-09-26','2019-11-07'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_Adaptive_App_Control_version-1','Adaptive_App_Control','Adaptive Application Controls should be enabled on virtual machines',NULL,NULL,'version-1',NULL,NULL,'2019-08-27','2019-08-27'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_Auditing_Advanced_Data_Security_version-1','Auditing_Advanced_Data_Security','Auditing should be enabled on advanced data security settings on SQL Server','Auditing should be enabled on advanced data security settings on SQL Server',NULL,'version-1',NULL,NULL,'2019-09-26','2019-09-26'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_AzureAuditNetworkAccessToStorageAccounts_version-1','AzureAuditNetworkAccessToStorageAccounts','Audit unrestricted network access in your storage account firewall settings. Instead, configure network rules so only applications from allowed networks can access the storage account. To allow connections from specific internet or on-premise clients, access can be granted to traffic from specific Azure virtual networks or to public internet IP address ranges',NULL,NULL,'version-1',NULL,NULL,'2019-11-08','2019-11-08'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_AzureAuditStorageVnetIntegration_version-1','AzureAuditStorageVnetIntegration','Audit Storage Vnet Integration',NULL,NULL,'version-1',NULL,NULL,'2019-11-08','2019-11-08'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_AzureAuditTransferToStorageAccounts_version-1','AzureAuditTransferToStorageAccounts','Audit requirement of Secure transfer in your storage account. Secure transfer is an option that forces your storage account to accept requests only from secure connections (HTTPS). Use of HTTPS ensures authentication between the server and the service and protects data in transit from network layer attacks such as man-in-the-middle, eavesdropping, and session-hijacking',NULL,NULL,'version-1',NULL,NULL,'2019-11-08','2019-11-08'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_AzureBatchAccountMetricAlertRules_version-1','AzureBatchAccountMetricAlertRules','Audit configuration of metric alert rules on Batch account to enable the required metric',NULL,NULL,'version-1',NULL,NULL,'2019-11-08','2019-11-08'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_AzureBatchAccountsDiagnosticLogs_version-1','AzureBatchAccountsDiagnosticLogs','Audit enabling of diagnostic logs. This enables you to recreate activity trails to use for investigation purposes; when a security incident occurs or when your network is compromised',NULL,NULL,'version-1',NULL,NULL,'2019-11-08','2019-11-08'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_AzureNamespacesDiagnosticLogs_version-1','AzureNamespacesDiagnosticLogs','Audit enabling of diagnostic logs. This enables you to recreate activity trails to use for investigation purposes; when a security incident occurs or when your network is compromised',NULL,NULL,'version-1',NULL,NULL,'2019-11-08','2019-11-08'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_AzureSQLAuditLogging_version-1','AzureSQLAuditLogging','The AuditActionsAndGroups property should contain at least SUCCESSFUL_DATABASE_AUTHENTICATION_GROUP, FAILED_DATABASE_AUTHENTICATION_GROUP, BATCH_COMPLETED_GROUP to ensure a thorough audit logging',NULL,NULL,'version-1',NULL,NULL,'2019-11-08','2019-11-08'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_AzureSQLServerAuthenticationSettings_version-1','AzureSQLServerAuthenticationSettings','Audit provisioning of an Azure Active Directory administrator for SQL Server to enable Azure AD authentication. Azure AD authentication enables simplified permission management and centralized identity management of database users and other Microsoft services',NULL,NULL,'version-1',NULL,NULL,'2019-11-08','2019-11-09'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_Azure_SQLServerDataSecuritySettings_version-1','Azure_SQLServerDataSecuritySettings','Ensure that an email address is provided for the \'Send alerts to\' field in the Advanced Data Security server settings. This email address receives alert notifications when anomalous activities are detected on SQL servers.',NULL,NULL,'version-1',NULL,NULL,'2019-11-14','2019-11-14'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_Azure_policies-Container_Security_Configurations_version-1','Azure_policies-Container_Security_Configurations','Vulnerabilities in container security configurations should be remediated',NULL,NULL,'version-1',NULL,NULL,'2019-08-27','2019-08-27'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_Azure_policies-Disk_Encryption_version-1','Azure_policies-Disk_Encryption','Disk encryption should be applied on virtual machines',NULL,NULL,'version-1',NULL,NULL,'2019-08-27','2019-08-27'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_Azure_policies-JIT_Network_Access_version-1','Azure_policies-JIT_Network_Access','Possible network Just In Time (JIT) access will be monitored by Azure Security Center as recommendations',NULL,NULL,'version-1',NULL,NULL,'2019-08-27','2019-08-27'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_Data_Encryption_SQL_version-1','Data_Encryption_SQL','Transparent Data Encryption on SQL databases should be enabled','Audit transparent data encryption status for SQL databases',NULL,'version-1',NULL,NULL,'2019-09-26','2019-09-26'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_IP_Forwarding_version-1','IP_Forwarding','Enabling IP forwarding on a virtual machine should be reviewed by the network security team','Enabling IP forwarding on a virtual machine should be reviewed by the network security team',NULL,'version-1',NULL,NULL,'2019-09-26','2019-09-26'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_Missing_Endpoint_Protection_version-1','Missing_Endpoint_Protection','Servers without an installed Endpoint Protection agent will be monitored by Azure Security Center as recommendations','Servers should be installed with Endpoint Protection agent ',NULL,'version-1',NULL,NULL,'2019-09-26','2019-09-26'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_Namespaces_DiagnosticLogs_version-1','Namespaces_DiagnosticLogs','Diagnostic logs in Namespaces should be enabled','This enables you to recreate activity trails to use for investigation purposes,when a security incident occurs or when your network is compromised',NULL,'version-1',NULL,NULL,'2019-09-30','2019-09-30'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_SQL_Auditing_Retention_version-1','SQL_Auditing_Retention','SQL servers should be configured with auditing retention days greater than 90 days','Audit SQL servers configured with an auditing retention period of less than 90 days',NULL,'version-1',NULL,NULL,'2019-09-30','2019-09-30'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_SQL_TDE_Encryption_version-1','SQL_TDE_Encryption','SQL server TDE protector should be encrypted with your own key','Transparent Data Encryption (TDE) with your own key support provides increased transparency and control over the TDE Protector, increased security with an HSM-backed external service, and promotion of separation of duties',NULL,'version-1',NULL,NULL,'2019-09-30','2019-09-30'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_SQL_Vulnerability_Assessment_version-1','SQL_Vulnerability_Assessment','Vulnerability assessment should be enabled on your SQL servers','Audit Azure SQL servers which do not have recurring vulnerability assessment scans enabled. Vulnerability assessment can discover, track, and help you remediate potential database vulnerabilities',NULL,'version-1',NULL,NULL,'2019-09-30','2019-09-30'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_Sensitive_Data_SQL_version-1','Sensitive_Data_SQL','Sensitive data in your SQL databases should be classified','Sensitive data in your SQL databases should be classified',NULL,'version-1',NULL,NULL,'2019-09-26','2019-09-26'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_Subnet_NSG_rule_version-1','Subnet_NSG_rule','Subnets should be associated with a Network Security Group','Protect your subnet from potential threats by restricting access to it with a Network Security Group (NSG)',NULL,'version-1',NULL,NULL,'2019-09-30','2019-09-30'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_System_Updates_version-1','System_Updates','Missing security system updates on your servers will be monitored by Azure Security Center as recommendations','System updates should be up to date',NULL,'version-1',NULL,NULL,'2019-09-26','2019-09-26'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_Vaults_DiagnosticLogs_version-1','Vaults_DiagnosticLogs','Diagnostic logs in Key Vault should be enabled','Diagnostic logs in Key Vault should be enabled',NULL,'version-1',NULL,NULL,'2019-09-30','2019-09-30'); +INSERT IGNORE INTO `cf_Policy` (`policyId`, `policyName`, `policyDesc`, `resolution`, `policyUrl`, `policyVersion`, `status`, `userId`, `createdDate`, `modifiedDate`) values('PacMan_Workflows_DiagnlsticLogs_version-1','Workflows_DiagnlsticLogs','Diagnostic logs in Logic Apps should be enabled','This enables you to recreate activity trails to use for investigation purposes,when a security incident occurs or when your network is compromised',NULL,'version-1',NULL,NULL,'2019-09-30','2019-09-30'); + /* Rule Initialisation */ @@ -1411,6 +1439,36 @@ INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Azure_Resolve_monitoring_agent_version-1_SecurityCenter_virtualmachine','azure_virtualmachine_resolve_monitoring_agent','PacMan_Azure_Resolve_monitoring_agent_version-1','resolve_monitoring_agent','virtualmachine','azure','resolve_monitoring_agent','{"params":[{"encrypt":false,"value":"check-for-azure-security-rule","key":"ruleKey"},{"key":"policyName","value":"Monitoring@agent@health@issues@should@be@resolved@on@your@machines","isValueNew":true,"encrypt":false},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"security","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_Azure_Resolve_monitoring_agent_version-1_SecurityCenter_virtualmachine","autofix":false,"alexaKeyword":"resolve monitoring agent findings","ruleRestUrl":"","targetType":"virtualmachine","pac_ds":"azure","policyId":"PacMan_Azure_Resolve_monitoring_agent_version-1","assetGroup":"azure","ruleUUID":"azure_virtualmachine_resolve_monitoring_agent","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_virtualmachine_resolve_monitoring_agent'),'ENABLED','ASGC','Resolve monitoring agent health issues on your machines',{d '2019-10-25'},{d '2019-10-25'},null,null); INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Azure_harden-NSGs_internet_version-1_SecurityCenter_virtualmachine','azure_virtualmachine_harden-nsgs_internet','PacMan_Azure_harden-NSGs_internet_version-1','harden-NSGs_internet','virtualmachine','azure','harden-NSGs_internet','{"params":[{"encrypt":false,"value":"check-for-azure-security-rule","key":"ruleKey"},{"key":"policyName","value":"Harden@Network@Security@Group@rules@of@internet@facing@virtual@machines","isValueNew":true,"encrypt":false},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"security","key":"ruleCategory"},{"encrypt":false,"value":"","key":"ruleOwner"}],"environmentVariables":[],"ruleId":"PacMan_Azure_harden-NSGs_internet_version-1_SecurityCenter_virtualmachine","autofix":false,"alexaKeyword":"harden nsgs for internet facing vms","ruleRestUrl":"","targetType":"virtualmachine","pac_ds":"azure","policyId":"PacMan_Azure_harden-NSGs_internet_version-1","assetGroup":"azure","ruleUUID":"azure_virtualmachine_harden-nsgs_internet","ruleType":"ManageRule"}','0 0/12 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_virtualmachine_harden-nsgs_internet'),'ENABLED','ASGC','Harden Network Security Group rules of internet facing Virtual Machines',{d '2019-10-25'},{d '2019-10-25'},null,null); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Access_Through_Internet_version-1_AIE_virtualmachine','azure_aie_virtualmachine','PacMan_Access_Through_Internet_version-1','AIE','virtualmachine','azure','AIE','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"9daedab3-fb2d-461e-b861-71790eead4f6\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Access_Through_Internet_version-1_AIE_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"AIE\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Access_Through_Internet_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_aie_virtualmachine\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_aie_virtualmachine'),'ENABLED','','Access through Internet facing endpoint should be restricted.','2019-09-26','2019-11-23','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Adaptive_App_Control_version-1_AAC_virtualmachine','azure_aac_virtualmachine','PacMan_Adaptive_App_Control_version-1','AAC','virtualmachine','azure','AdaptiveAppControl','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"47a6b606-51aa-4496-8bb7-64b11cf66adc\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"medium\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Adaptive_App_Control_version-1_AAC_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"AdaptiveAppControl\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Adaptive_App_Control_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_aac_virtualmachine\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_aac_virtualmachine'),'ENABLED','','Adaptive Application Controls should be enabled on virtual machines.','2019-08-27','2019-11-06','medium','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Auditing_Advanced_Data_Security_version-1_SADS_sqlserver','azure_sads_sqlserver','PacMan_Auditing_Advanced_Data_Security_version-1','SADS','sqlserver','azure','SADS','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"a6fb4358-5bf4-4ad7-ba82-2cd2f41ce5e9\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"medium\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Auditing_Advanced_Data_Security_version-1_SADS_sqlserver\",\"autofix\":false,\"alexaKeyword\":\"SADS\",\"ruleRestUrl\":\"\",\"targetType\":\"sqlserver\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Auditing_Advanced_Data_Security_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_sads_sqlserver\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_sads_sqlserver'),'ENABLED','','Advanced Data Security SQL Server should have auditing enabled.','2019-09-26','2019-11-06','medium','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_AzureAuditNetworkAccessToStorageAccounts_version-1_ConfigureNetworkRulesOnStorageAccount_storageaccount','azure_ConfigureNetworkRules_storageaccount','PacMan_AzureAuditNetworkAccessToStorageAccounts_version-1','ConfigureNetworkRulesOnStorageAccount','storageaccount','azure','ConfigureNetworkRulesOnStorageAccount','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"34c877ad-507e-4c82-993e-3452a6e0ad3c\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_AzureAuditNetworkAccessToStorageAccounts_version-1_ConfigureNetworkRulesOnStorageAccount_storageaccount\",\"autofix\":false,\"alexaKeyword\":\"ConfigureNetworkRulesOnStorageAccount\",\"ruleRestUrl\":\"\",\"targetType\":\"storageaccount\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_AzureAuditNetworkAccessToStorageAccounts_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_ConfigureNetworkRules_storageaccount\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_ConfigureNetworkRules_storageaccount'),'ENABLED','','Audit unrestricted network access to storage accounts.','2019-11-08','2019-11-23','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_AzureAuditStorageVnetIntegration_version-1_AuditStorageVnetIntegration_storageaccount','azure_AuditStorageVnetInt_storageaccount','PacMan_AzureAuditStorageVnetIntegration_version-1','AuditStorageVnetIntegration','storageaccount','azure','AuditStorageVnetIntegration','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"deb3d6f0-8113-4ed4-8492-d64f90919223\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_AzureAuditStorageVnetIntegration_version-1_AuditStorageVnetIntegration_storageaccount\",\"autofix\":false,\"alexaKeyword\":\"AuditStorageVnetIntegration\",\"ruleRestUrl\":\"\",\"targetType\":\"storageaccount\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_AzureAuditStorageVnetIntegration_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_AuditStorageVnetInt_storageaccount\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_AuditStorageVnetInt_storageaccount'),'ENABLED','','Audit Storage VNet Integration.','2019-11-08','2019-11-23','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_AzureAuditTransferToStorageAccounts_version-1_EnableSecureTransferToStorageAccounts_storageaccount','azure_EnableSecureTransfer_storageaccount','PacMan_AzureAuditTransferToStorageAccounts_version-1','EnableSecureTransferToStorageAccounts','storageaccount','azure','EnableSecureTransferToStorageAccounts','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"404c3081-a854-4457-ae30-26a93ef643f9\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_AzureAuditTransferToStorageAccounts_version-1_EnableSecureTransferToStorageAccounts_storageaccount\",\"autofix\":false,\"alexaKeyword\":\"EnableSecureTransferToStorageAccounts\",\"ruleRestUrl\":\"\",\"targetType\":\"storageaccount\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_AzureAuditTransferToStorageAccounts_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_EnableSecureTransfer_storageaccount\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_EnableSecureTransfer_storageaccount'),'ENABLED','','Secure transfer to storage accounts should be enabled.','2019-11-08','2019-11-23','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_AzureBatchAccountMetricAlertRules_version-1_ConfigureMetricAlertRules_batchaccounts','azure_ConfigureMetric_batchaccounts','PacMan_AzureBatchAccountMetricAlertRules_version-1','ConfigureMetricAlertRules','batchaccounts','azure','ConfigureMetricAlertRules','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"26ee67a2-f81a-4ba8-b9ce-8550bd5ee1a7\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_AzureBatchAccountMetricAlertRules_version-1_ConfigureMetricAlertRules_batchaccounts\",\"autofix\":false,\"alexaKeyword\":\"ConfigureMetricAlertRules\",\"ruleRestUrl\":\"\",\"targetType\":\"batchaccounts\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_AzureBatchAccountMetricAlertRules_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_ConfigureMetric_batchaccounts\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_ConfigureMetric_batchaccounts'),'ENABLED','','Metric alert rules should be configured on Batch accounts.','2019-11-08','2019-11-23','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_AzureBatchAccountsDiagnosticLogs_version-1_EnableDiagnosticLogsInBatchAccount_batchaccounts','azure_EnableDiagnosticLogs_batchaccounts','PacMan_AzureBatchAccountsDiagnosticLogs_version-1','EnableDiagnosticLogsInBatchAccount','batchaccounts','azure','EnableDiagnosticLogsInBatchAccount','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"428256e6-1fac-4f48-a757-df34c2b3336d\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_AzureBatchAccountsDiagnosticLogs_version-1_EnableDiagnosticLogsInBatchAccount_batchaccounts\",\"autofix\":false,\"alexaKeyword\":\"EnableDiagnosticLogsInBatchAccount\",\"ruleRestUrl\":\"\",\"targetType\":\"batchaccounts\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_AzureBatchAccountsDiagnosticLogs_version-1\",\"assetGroup\":null,\"ruleUUID\":\"azure_EnableDiagnosticLogs_batchaccounts\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_EnableDiagnosticLogs_batchaccounts'),'ENABLED','','Diagnostic logs in Batch accounts should be enabled.','2019-11-08','2019-11-23','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_AzureNamespacesDiagnosticLogs_version-1_EnableDiagnosticLogsInServiceBus_namespaces','azure_EnableDiagnosticLogs_namespaces','PacMan_AzureNamespacesDiagnosticLogs_version-1','EnableDiagnosticLogsInServiceBus','namespaces','azure','EnableDiagnosticLogsInServiceBus','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"f8d36e2f-389b-4ee4-898d-21aeb69a0f45\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_AzureNamespacesDiagnosticLogs_version-1_EnableDiagnosticLogsInServiceBus_namespaces\",\"autofix\":false,\"alexaKeyword\":\"EnableDiagnosticLogsInServiceBus\",\"ruleRestUrl\":\"\",\"targetType\":\"namespaces\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_AzureNamespacesDiagnosticLogs_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_EnableDiagnosticLogs_namespaces\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_EnableDiagnosticLogs_namespaces'),'ENABLED','','Diagnostic logs in Service Bus should be enabled.','2019-11-08','2019-11-23','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_AzureSQLAuditLogging_version-1_ConfigureActionGroupsInSQLAuditSettings_sqlserver','azure_ConfigureActionGroups_sqlserver','PacMan_AzureSQLAuditLogging_version-1','ConfigureActionGroupsInSQLAuditSettings','sqlserver','azure','ConfigureActionGroupsInSQLAuditSettings','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"7ff426e2-515f-405a-91c8-4f2333442eb5\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"governance\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_AzureSQLAuditLogging_version-1_ConfigureActionGroupsInSQLAuditSettings_sqlserver\",\"autofix\":false,\"alexaKeyword\":\"ConfigureActionGroupsInSQLAuditSettings\",\"ruleRestUrl\":\"\",\"targetType\":\"sqlserver\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_AzureSQLAuditLogging_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_ConfigureActionGroups_sqlserver\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_ConfigureActionGroups_sqlserver'),'ENABLED','','SQL Auditing settings should have Action-Groups configured to capture critical activities.','2019-11-08','2019-11-23','high','governance'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_AzureSQLServerAuthenticationSettings_version-1_EnableAzureADAuthentication_sqlserver','azure_EnableAzureADAuth_sqlserver','PacMan_AzureSQLServerAuthenticationSettings_version-1','EnableAzureADAuthentication','sqlserver','azure','EnableAzureADAuthentication','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"1f314764-cb73-4fc9-b863-8eca98ac36e9\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_AzureSQLServerAuthenticationSettings_version-1_EnableAzureADAuthentication_sqlserver\",\"autofix\":false,\"alexaKeyword\":\"EnableAzureADAuthentication\",\"ruleRestUrl\":\"\",\"targetType\":\"sqlserver\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_AzureSQLServerAuthenticationSettings_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_EnableAzureADAuth_sqlserver\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_EnableAzureADAuth_sqlserver'),'ENABLED','','An Active Directory administrator should be provisioned for SQL Servers.','2019-11-08','2019-11-09','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Azure_SQLServerDataSecuritySettings_version-1_ConfigureEmailInDataSecuritySettings_sqlserver','azure_ConfigureEmailInDataSecurity_sqlserver','PacMan_Azure_SQLServerDataSecuritySettings_version-1','ConfigureEmailInDataSecuritySettings','sqlserver','azure','ConfigureEmailInDSS','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"9677b740-f641-4f3c-b9c5-466005c85278\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_SQLServerDataSecuritySettings_version-1_ConfigureEmailInDataSecuritySettings_sqlserver\",\"autofix\":false,\"alexaKeyword\":\"ConfigureEmailInDSS\",\"ruleRestUrl\":\"\",\"targetType\":\"sqlserver\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_SQLServerDataSecuritySettings_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_ConfigureEmailInDataSecurity_sqlserver\",\"ruleType\":\"ManageRule\"}','0 0 1/1 * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_ConfigureEmailInDataSecurity_sqlserver'),'ENABLED','','Configure Email In Data Security Settings.','2019-11-14','2019-11-23','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Azure_policies-Container_Security_Configurations_version-1_CSC_virtualmachine','azure_CSC_virtualmachine','PacMan_Azure_policies-Container_Security_Configurations_version-1','CSC','virtualmachine','azure','Container security','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"e8cbc669-f12d-49eb-93e7-9273119e9933\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_policies-Container_Security_Configurations_version-1_CSC_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"Container security\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_policies-Container_Security_Configurations_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_CSC_virtualmachine\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_CSC_virtualmachine'),'ENABLED','','Container Security Configuration.','2019-09-26','2019-11-06','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Azure_policies-Disk_Encryption_version-1_DE_virtualmachine','azure_DiskEncryption_virtualmachine','PacMan_Azure_policies-Disk_Encryption_version-1','DE','virtualmachine','azure','AzurepoliciesDiskEncryption','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"0961003e-5a0a-4549-abde-af6a37f2724d\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"medium\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_policies-Disk_Encryption_version-1_DE_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"AzurepoliciesDiskEncryption\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_policies-Disk_Encryption_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_DiskEncryption_virtualmachine\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_DiskEncryption_virtualmachine'),'ENABLED','','VMs w/o enabled disk encryption will be monitored by Security Center as recommendations.','2019-08-27','2019-11-09','medium','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Azure_policies-JIT_Network_Access_version-1_JIT_virtualmachine','azure_JIT_virtualmachine','PacMan_Azure_policies-JIT_Network_Access_version-1','JIT','virtualmachine','azure','AzurepoliciesJITNetworkAccess','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"b0f33259-77d7-4c9e-aac6-3aabcfae693c\",\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Azure_policies-JIT_Network_Access_version-1_JIT_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"AzurepoliciesJITNetworkAccess\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Azure_policies-JIT_Network_Access_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_JIT_virtualmachine\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_JIT_virtualmachine'),'ENABLED','','Just-In-Time network access control should be applied on Virtual Machines.','2019-08-27','2019-11-06','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Data_Encryption_SQL_version-1_SDE_sqldatabase','azure_SDE_sqldatabase','PacMan_Data_Encryption_SQL_version-1','SDE','sqldatabase','azure','SDE','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"17k78e20-9358-41c9-923c-fb736d382a12\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Data_Encryption_SQL_version-1_SDE_sqldatabase\",\"autofix\":false,\"alexaKeyword\":\"SDE\",\"ruleRestUrl\":\"\",\"targetType\":\"sqldatabase\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Data_Encryption_SQL_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_SDE_sqldatabase\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_SDE_sqldatabase'),'ENABLED','','Transparent Data Encryption on SQL databases should be enabled.','2019-09-26','2019-11-07','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_IP_Forwarding_version-1_IPF_virtualmachine','azure_IPF_virtualmachine','PacMan_IP_Forwarding_version-1','IPF','virtualmachine','azure','IPF','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"bd352bd5-2853-4985-bf0d-73806b4a5744\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_IP_Forwarding_version-1_IPF_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"IPF\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_IP_Forwarding_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_IPF_virtualmachine\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_IPF_virtualmachine'),'ENABLED','','IP Forwarding on Virtual Machines should be disabled.','2019-09-26','2019-11-06','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Missing_Endpoint_Protection_version-1_MEP_virtualmachine','azure_MEP_virtualmachine','PacMan_Missing_Endpoint_Protection_version-1','MEP','virtualmachine','azure','MEP','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"af6cd1bd-1635-48cb-bde7-5b15693900b9\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Missing_Endpoint_Protection_version-1_MEP_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"MEP\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Missing_Endpoint_Protection_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_MEP_virtualmachine\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_MEP_virtualmachine'),'ENABLED','','Monitor missing Endpoint Protection in Security Center.','2019-09-26','2019-11-09','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Namespaces_DiagnosticLogs_version-1_NDL_namespaces','azure_NDL_namespaces','PacMan_Namespaces_DiagnosticLogs_version-1','NDL','namespaces','azure','NDL','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"83a214f7-d01a-484b-91a9-ed54470c9a6a\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"medium\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Namespaces_DiagnosticLogs_version-1_NDL_namespaces\",\"autofix\":false,\"alexaKeyword\":\"NDL\",\"ruleRestUrl\":\"\",\"targetType\":\"namespaces\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Namespaces_DiagnosticLogs_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_NDL_namespaces\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_NDL_namespaces'),'ENABLED','','Diagnostic logs in namespaces should be enabled.','2019-09-30','2019-11-23','medium','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_SQL_Auditing_Retention_version-1_SAR_sqlserver','azure_SAR_sqlserver','PacMan_SQL_Auditing_Retention_version-1','SAR','sqlserver','azure','SAR','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"89099bee-89e0-4b26-a5f4-165451757743\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_SQL_Auditing_Retention_version-1_SAR_sqlserver\",\"autofix\":false,\"alexaKeyword\":\"SAR\",\"ruleRestUrl\":\"\",\"targetType\":\"sqlserver\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_SQL_Auditing_Retention_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_SAR_sqlserver\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_SAR_sqlserver'),'ENABLED','','SQL Auditing for configured number of retention days.','2019-09-30','2019-11-07','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_SQL_TDE_Encryption_version-1_TPS_sqlserver','azure_TPS_sqlserver','PacMan_SQL_TDE_Encryption_version-1','TPS','sqlserver','azure','TPS','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"0d134df8-db83-46fb-ad72-fe0c9428c8dd\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_SQL_TDE_Encryption_version-1_TPS_sqlserver\",\"autofix\":false,\"alexaKeyword\":\"TPS\",\"ruleRestUrl\":\"\",\"targetType\":\"sqlserver\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_SQL_TDE_Encryption_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_TPS_sqlserver\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_TPS_sqlserver'),'ENABLED','','SQL TDE protector encryption check.','2019-09-30','2019-11-07','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_SQL_Vulnerability_Assessment_version-1_SVA_sqlserver','azure_SVA_sqlserver','PacMan_SQL_Vulnerability_Assessment_version-1','SVA','sqlserver','azure','SVA','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"ef2a8f2a-b3d9-49cd-a8a8-9a3aaaf647d9\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_SQL_Vulnerability_Assessment_version-1_SVA_sqlserver\",\"autofix\":false,\"alexaKeyword\":\"SVA\",\"ruleRestUrl\":\"\",\"targetType\":\"sqlserver\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_SQL_Vulnerability_Assessment_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_SVA_sqlserver\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_SVA_sqlserver'),'ENABLED','','Vulnerability assessment should be enabled on SQL servers.','2019-09-30','2019-11-07','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Sensitive_Data_SQL_version-1_SDS_sqldatabase','azure_SDS_sqldatabase','PacMan_Sensitive_Data_SQL_version-1','SDS','sqldatabase','azure','SDS','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"cc9835f2-9f6b-4cc8-ab4a-f8ef615eb349\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Sensitive_Data_SQL_version-1_SDS_sqldatabase\",\"autofix\":false,\"alexaKeyword\":\"SDS\",\"ruleRestUrl\":\"\",\"targetType\":\"sqldatabase\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Sensitive_Data_SQL_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_SDS_sqldatabase\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_SDS_sqldatabase'),'ENABLED','','Sensitive data in SQL databases should be classified.','2019-09-26','2019-11-07','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Subnet_NSG_rule_version-1_SNR_subnets','azure_SNR_subnets','PacMan_Subnet_NSG_rule_version-1','SNR','subnets','azure','SNR','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"e71308d3-144b-4262-b144-efdc3cc90517\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Subnet_NSG_rule_version-1_SNR_subnets\",\"autofix\":false,\"alexaKeyword\":\"SNR\",\"ruleRestUrl\":\"\",\"targetType\":\"subnets\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Subnet_NSG_rule_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_SNR_subnets\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_SNR_subnets'),'ENABLED','','Subnets should be associated with a Network Security Group.','2019-09-30','2019-11-07','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_System_Updates_version-1_SSU_virtualmachine','azure_SSU_virtualmachine','PacMan_System_Updates_version-1','SSU','virtualmachine','azure','SSU','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"86b3d65f-7626-441e-b690-81a8b71cff60\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_System_Updates_version-1_SSU_virtualmachine\",\"autofix\":false,\"alexaKeyword\":\"SSU\",\"ruleRestUrl\":\"\",\"targetType\":\"virtualmachine\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_System_Updates_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_SSU_virtualmachine\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_SSU_virtualmachine'),'ENABLED','','System updates should be installed on Virtual Machines.','2019-09-26','2019-11-07','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Vaults_DiagnosticLogs_version-1_VDR_vaults','azure_VDR_vaults','PacMan_Vaults_DiagnosticLogs_version-1','VDR','vaults','azure','VDR','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"cf820ca0-f99e-4f3e-84fb-66e913812d21\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Vaults_DiagnosticLogs_version-1_VDR_vaults\",\"autofix\":false,\"alexaKeyword\":\"VDR\",\"ruleRestUrl\":\"\",\"targetType\":\"vaults\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Vaults_DiagnosticLogs_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_VDR_vaults\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_VDR_vaults'),'ENABLED','','Diagnostic logs in Key Vault should be enabled.','2019-09-30','2019-11-06','high','security'); +INSERT IGNORE INTO cf_RuleInstance (ruleId,ruleUUID,policyId,ruleName,targetType,assetGroup,alexaKeyword,ruleParams,ruleFrequency,ruleExecutable,ruleRestUrl,ruleType,ruleArn,status,userId,displayName,createdDate,modifiedDate,severity,category) VALUES ('PacMan_Workflows_DiagnlsticLogs_version-1_WDR_workflows','azure_WDR_workflows','PacMan_Workflows_DiagnlsticLogs_version-1','WDR','workflows','azure','WDR','{\"params\":[{\"key\":\"ruleKey\",\"value\":\"check-for-azure-policy-evaluation-results\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"azurePolicyEvaluationResults\",\"value\":\"/azure_policyevaluationresults/_search\",\"isValueNew\":true,\"encrypt\":false},{\"key\":\"policyDefinitionName\",\"value\":\"34f95f76-5386-4de7-b824-0d8478470c9d\",\"isValueNew\":true,\"encrypt\":false},{\"encrypt\":false,\"value\":\"high\",\"key\":\"severity\"},{\"encrypt\":false,\"value\":\"security\",\"key\":\"ruleCategory\"},{\"encrypt\":false,\"value\":\"\",\"key\":\"ruleOwner\"}],\"environmentVariables\":[],\"ruleId\":\"PacMan_Workflows_DiagnlsticLogs_version-1_WDR_workflows\",\"autofix\":false,\"alexaKeyword\":\"WDR\",\"ruleRestUrl\":\"\",\"targetType\":\"workflows\",\"pac_ds\":\"azure\",\"policyId\":\"PacMan_Workflows_DiagnlsticLogs_version-1\",\"assetGroup\":\"azure\",\"ruleUUID\":\"azure_WDR_workflows\",\"ruleType\":\"ManageRule\"}','0 0/6 * * ? *','','','ManageRule',concat('arn:aws:events:',@region,':',@account,':rule/azure_WDR_workflows'),'ENABLED','','Diagnostic logs in Logic Apps should be enabled.','2019-09-30','2019-11-06','high','security'); + + + /* Omni Seach Configuration */ From eb79e9c2a7baf812b2acdbc5deea002dcc725d6b Mon Sep 17 00:00:00 2001 From: Harminder Singh Date: Thu, 5 Dec 2019 02:10:30 -0800 Subject: [PATCH 107/107] Update the logout url and handling of interceptor to not make retry when logout API is called. --- webapp/src/app/core/services/auth.service.ts | 12 ++++++++---- .../app/core/services/request-interceptor.service.ts | 3 +++ webapp/src/environments/environment.prod.ts | 2 +- webapp/src/environments/environment.stg.ts | 2 +- webapp/src/environments/environment.ts | 2 +- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/webapp/src/app/core/services/auth.service.ts b/webapp/src/app/core/services/auth.service.ts index 8e9cb34ba..d08919bc0 100644 --- a/webapp/src/app/core/services/auth.service.ts +++ b/webapp/src/app/core/services/auth.service.ts @@ -66,17 +66,21 @@ export class AuthService { } doLogout() { - this.dataStore.clearAll(); // Calling clear session from data store - localStorage.setItem('logout', 'true'); - localStorage.removeItem('logout'); - if (this.adAuthentication) { + this.clearSessionStorage(); this.adalService.logout(); } else { this.onPremAuthentication.logout(); + this.clearSessionStorage(); } } + clearSessionStorage() { + this.dataStore.clearAll(); // Calling clear session from data store + localStorage.setItem('logout', 'true'); + localStorage.removeItem('logout'); + } + authenticateUserOnPrem(url, method, payload, headers) { return this.httpService.getHttpResponse(url, method, payload, {}, headers) diff --git a/webapp/src/app/core/services/request-interceptor.service.ts b/webapp/src/app/core/services/request-interceptor.service.ts index 2943c4fc4..df202877a 100644 --- a/webapp/src/app/core/services/request-interceptor.service.ts +++ b/webapp/src/app/core/services/request-interceptor.service.ts @@ -28,6 +28,9 @@ export class RequestInterceptorService implements HttpInterceptor { if (req.url.includes('user/authorize') || req.url.includes('user/login') || req.url.includes('refreshtoken')) { this.loggerService.log('info', 'Not adding the access token for this api - ' + req.url); return next.handle(req); + } else if (req.url.includes('user/logout-session')) { + this.loggerService.log('info', 'Do not retry when logging user out - ' + req.url); + return next.handle(this.addToken(req, authService.getAuthToken())); } return next.handle(this.addToken(req, authService.getAuthToken())).pipe( catchError(error => { diff --git a/webapp/src/environments/environment.prod.ts b/webapp/src/environments/environment.prod.ts index 85f430598..9630ceb10 100644 --- a/webapp/src/environments/environment.prod.ts +++ b/webapp/src/environments/environment.prod.ts @@ -426,7 +426,7 @@ export const environment = { method: 'GET' }, logout: { - url: '{{baseUrl}}/user/logout-session', + url: '{{baseUrl}}/auth/user/logout-session', method: 'GET' }, refresh: { diff --git a/webapp/src/environments/environment.stg.ts b/webapp/src/environments/environment.stg.ts index 90a4a7086..d1fcd324d 100644 --- a/webapp/src/environments/environment.stg.ts +++ b/webapp/src/environments/environment.stg.ts @@ -426,7 +426,7 @@ export const environment = { method: 'GET' }, logout: { - url: '{{baseUrl}}/user/logout-session', + url: '{{baseUrl}}/auth/user/logout-session', method: 'GET' }, refresh: { diff --git a/webapp/src/environments/environment.ts b/webapp/src/environments/environment.ts index 3b4bf816a..e04baeb96 100644 --- a/webapp/src/environments/environment.ts +++ b/webapp/src/environments/environment.ts @@ -426,7 +426,7 @@ export const environment = { method: 'GET' }, logout: { - url: '{{baseUrl}}/user/logout-session', + url: '{{baseUrl}}/auth/user/logout-session', method: 'GET' }, refresh: {