From ce58f6caf1b208b963d2252a6447a7f89a000d6f Mon Sep 17 00:00:00 2001 From: Adrian Vladu Date: Tue, 3 Oct 2023 18:00:29 +0300 Subject: [PATCH] pep8: fix using autopep8 agressive --- coriolis/api/v1/minion_pools.py | 13 ++-- coriolis/api/v1/replica_actions.py | 3 +- coriolis/api/v1/replicas.py | 3 +- coriolis/api/v1/router.py | 10 +-- coriolis/api/v1/utils.py | 4 +- coriolis/api/v1/views/diagnostic_view.py | 2 +- coriolis/cmd/api.py | 7 +- coriolis/cmd/conductor.py | 7 +- coriolis/cmd/minion_manager.py | 7 +- coriolis/cmd/scheduler.py | 7 +- coriolis/cmd/worker.py | 7 +- coriolis/conductor/rpc/client.py | 20 +++--- coriolis/conductor/rpc/server.py | 71 ++++++++++--------- coriolis/constants.py | 4 +- coriolis/context.py | 24 +++---- coriolis/cron/cron.py | 9 +-- coriolis/db/api.py | 6 +- .../migrate_repo/versions/001_initial.py | 41 ++++++----- .../versions/016_adds_minion_vm_pools.py | 71 +++++++++++-------- coriolis/db/sqlalchemy/models.py | 6 +- coriolis/db/sqlalchemy/types.py | 2 +- coriolis/events.py | 7 +- coriolis/exception.py | 6 +- coriolis/keystone.py | 14 ++-- coriolis/licensing/client.py | 3 +- coriolis/migrations/manager.py | 3 +- coriolis/minion_manager/rpc/client.py | 20 +++--- coriolis/minion_manager/rpc/server.py | 58 ++++++++------- coriolis/minion_manager/rpc/tasks.py | 6 +- coriolis/osmorphing/base.py | 3 +- coriolis/osmorphing/osdetect/base.py | 3 +- coriolis/osmorphing/osdetect/centos.py | 3 +- coriolis/osmorphing/osmount/base.py | 14 ++-- coriolis/osmorphing/osmount/windows.py | 18 +++-- coriolis/osmorphing/redhat.py | 11 +-- coriolis/osmorphing/suse.py | 12 ++-- coriolis/osmorphing/ubuntu.py | 3 +- coriolis/osmorphing/windows.py | 17 +++-- coriolis/providers/backup_writers.py | 9 +-- coriolis/providers/base.py | 1 - coriolis/providers/provider_utils.py | 45 ++++++------ coriolis/providers/replicator.py | 9 ++- coriolis/qemu.py | 3 +- coriolis/rpc.py | 6 +- coriolis/scheduler/rpc/client.py | 2 +- coriolis/scheduler/rpc/server.py | 4 +- coriolis/scheduler/scheduler_utils.py | 1 + coriolis/service.py | 9 ++- coriolis/taskflow/base.py | 31 ++++---- coriolis/taskflow/runner.py | 5 +- coriolis/tasks/minion_pool_tasks.py | 5 +- coriolis/tests/conductor/rpc/test_server.py | 34 ++++----- coriolis/tests/test_schemas.py | 6 +- coriolis/tests/worker/rpc/test_server.py | 30 ++++---- coriolis/utils.py | 3 +- coriolis/worker/rpc/client.py | 12 ++-- coriolis/worker/rpc/server.py | 12 ++-- coriolis/wsman.py | 5 +- 58 files changed, 408 insertions(+), 349 deletions(-) diff --git a/coriolis/api/v1/minion_pools.py b/coriolis/api/v1/minion_pools.py index 0baab3488..70fc2f391 100644 --- a/coriolis/api/v1/minion_pools.py +++ b/coriolis/api/v1/minion_pools.py @@ -65,15 +65,16 @@ def _check_pool_numeric_values( if maximum_minions < minimum_minions: raise Exception( "'maximum_minions' value (%s) must be at least as large as" - " the 'minimum_minions' value (%s)." % ( - maximum_minions, minimum_minions)) + " the 'minimum_minions' value (%s)." % + (maximum_minions, minimum_minions)) if minion_max_idle_time is not None: if minion_max_idle_time <= 0: raise Exception( "'minion_max_idle_time' must be a strictly positive " "integer. Got: %s" % maximum_minions) - @api_utils.format_keyerror_message(resource='minion_pool', method='create') + @api_utils.format_keyerror_message(resource='minion_pool', + method='create') def _validate_create_body(self, ctxt, body): minion_pool = body["minion_pool"] name = minion_pool["name"] @@ -141,7 +142,8 @@ def create(self, req, body): minion_max_idle_time, minion_retention_strategy, notes=notes, skip_allocation=skip_allocation)) - @api_utils.format_keyerror_message(resource='minion_pool', method='update') + @api_utils.format_keyerror_message(resource='minion_pool', + method='update') def _validate_update_body(self, id, context, body): minion_pool = body["minion_pool"] if 'endpoint_id' in minion_pool: @@ -179,8 +181,7 @@ def _validate_update_body(self, id, context, body): elif minion_pool['platform'] == ( constants.PROVIDER_PLATFORM_DESTINATION): self._endpoints_api.validate_endpoint_destination_minion_pool_options( - context, minion_pool['endpoint_id'], - vals['environment_options']) + context, minion_pool['endpoint_id'], vals['environment_options']) else: raise Exception( "Unknown pool platform: %s" % minion_pool[ diff --git a/coriolis/api/v1/replica_actions.py b/coriolis/api/v1/replica_actions.py index 123a0c476..173ec9b7d 100644 --- a/coriolis/api/v1/replica_actions.py +++ b/coriolis/api/v1/replica_actions.py @@ -18,7 +18,8 @@ def __init__(self): @api_wsgi.action('delete-disks') def _delete_disks(self, req, id, body): context = req.environ['coriolis.context'] - context.can(replica_policies.get_replicas_policy_label("delete_disks")) + context.can( + replica_policies.get_replicas_policy_label("delete_disks")) try: return replica_tasks_execution_view.single( req, self._replica_api.delete_disks(context, id)) diff --git a/coriolis/api/v1/replicas.py b/coriolis/api/v1/replicas.py index 3acaa0c31..0497586ed 100644 --- a/coriolis/api/v1/replicas.py +++ b/coriolis/api/v1/replicas.py @@ -158,7 +158,8 @@ def _update_storage_mappings(original_storage_mappings, backend_mappings = original_storage_mappings.get( 'backend_mappings', []) - new_backend_mappings = new_storage_mappings.get('backend_mappings', []) + new_backend_mappings = new_storage_mappings.get( + 'backend_mappings', []) new_backend_mapping_sources = [mapping['source'] for mapping in new_backend_mappings] diff --git a/coriolis/api/v1/router.py b/coriolis/api/v1/router.py index 85d68e3ff..3a17fdd56 100644 --- a/coriolis/api/v1/router.py +++ b/coriolis/api/v1/router.py @@ -89,11 +89,11 @@ def _setup_routes(self, mapper, ext_mgr): self.resources['endpoint_destination_minion_pool_options'] = \ endpoint_destination_minion_pool_options.create_resource() - mapper.resource('minion_pool_options', - 'endpoints/{endpoint_id}/destination-minion-pool-options', - controller=( - self.resources[ - 'endpoint_destination_minion_pool_options'])) + mapper.resource( + 'minion_pool_options', + 'endpoints/{endpoint_id}/destination-minion-pool-options', + controller=(self.resources + ['endpoint_destination_minion_pool_options'])) endpoint_actions_resource = endpoint_actions.create_resource() self.resources['endpoint_actions'] = endpoint_actions_resource diff --git a/coriolis/api/v1/utils.py b/coriolis/api/v1/utils.py index dd142e288..d5334ff88 100644 --- a/coriolis/api/v1/utils.py +++ b/coriolis/api/v1/utils.py @@ -153,7 +153,7 @@ def validate_instances_list_for_transfer(instances): inst: count for (inst, count) in appearances.items() if count > 1} if duplicates: raise exception.InvalidInput( - "Transfer action instances (%s) list contained duplicates: %s " % ( - instances, duplicates)) + "Transfer action instances (%s) list contained duplicates: %s " % + (instances, duplicates)) return instances diff --git a/coriolis/api/v1/views/diagnostic_view.py b/coriolis/api/v1/views/diagnostic_view.py index 295ee9c2a..871b7f64c 100644 --- a/coriolis/api/v1/views/diagnostic_view.py +++ b/coriolis/api/v1/views/diagnostic_view.py @@ -3,7 +3,7 @@ def single(req, diag): - return {"diagnostic": diag} + return {"diagnostic": diag} def collection(req, diag): diff --git a/coriolis/cmd/api.py b/coriolis/cmd/api.py index d175f32b6..8806bcd70 100644 --- a/coriolis/cmd/api.py +++ b/coriolis/cmd/api.py @@ -10,10 +10,9 @@ from coriolis import utils api_opts = [ - cfg.IntOpt('worker_count', - min=1, default=processutils.get_worker_count(), - help='Number of processes in which the service will be running') -] + cfg.IntOpt( + 'worker_count', min=1, default=processutils.get_worker_count(), + help='Number of processes in which the service will be running')] CONF = cfg.CONF CONF.register_opts(api_opts, 'api') diff --git a/coriolis/cmd/conductor.py b/coriolis/cmd/conductor.py index d4d79b98a..30d218ec2 100644 --- a/coriolis/cmd/conductor.py +++ b/coriolis/cmd/conductor.py @@ -12,10 +12,9 @@ from coriolis import utils conductor_opts = [ - cfg.IntOpt('worker_count', - min=1, default=processutils.get_worker_count(), - help='Number of processes in which the service will be running') -] + cfg.IntOpt( + 'worker_count', min=1, default=processutils.get_worker_count(), + help='Number of processes in which the service will be running')] CONF = cfg.CONF CONF.register_opts(conductor_opts, 'conductor') diff --git a/coriolis/cmd/minion_manager.py b/coriolis/cmd/minion_manager.py index ea165d2ab..7424344e5 100644 --- a/coriolis/cmd/minion_manager.py +++ b/coriolis/cmd/minion_manager.py @@ -11,10 +11,9 @@ from coriolis.minion_manager.rpc import server as rpc_server minion_manager_opts = [ - cfg.IntOpt('worker_count', - min=1, default=1, - help='Number of processes in which the service will be running') -] + cfg.IntOpt( + 'worker_count', min=1, default=1, + help='Number of processes in which the service will be running')] CONF = cfg.CONF CONF.register_opts(minion_manager_opts, 'minion_manager') diff --git a/coriolis/cmd/scheduler.py b/coriolis/cmd/scheduler.py index 54c94d8e8..0d53594d1 100644 --- a/coriolis/cmd/scheduler.py +++ b/coriolis/cmd/scheduler.py @@ -11,10 +11,9 @@ from coriolis.scheduler.rpc import server as rpc_server scheduler_opts = [ - cfg.IntOpt('worker_count', - min=1, default=1, - help='Number of processes in which the service will be running') -] + cfg.IntOpt( + 'worker_count', min=1, default=1, + help='Number of processes in which the service will be running')] CONF = cfg.CONF CONF.register_opts(scheduler_opts, 'scheduler') diff --git a/coriolis/cmd/worker.py b/coriolis/cmd/worker.py index 2e7980d94..a9bdbf602 100644 --- a/coriolis/cmd/worker.py +++ b/coriolis/cmd/worker.py @@ -12,10 +12,9 @@ from coriolis.worker.rpc import server as rpc_server worker_opts = [ - cfg.IntOpt('worker_count', - min=1, default=processutils.get_worker_count(), - help='Number of processes in which the service will be running') -] + cfg.IntOpt( + 'worker_count', min=1, default=processutils.get_worker_count(), + help='Number of processes in which the service will be running')] CONF = cfg.CONF CONF.register_opts(worker_opts, 'worker') diff --git a/coriolis/conductor/rpc/client.py b/coriolis/conductor/rpc/client.py index f214c41c5..3bfd216e5 100644 --- a/coriolis/conductor/rpc/client.py +++ b/coriolis/conductor/rpc/client.py @@ -242,11 +242,10 @@ def migrate_instances(self, ctxt, origin_endpoint_id, source_environment=source_environment, user_scripts=user_scripts) - def deploy_replica_instances(self, ctxt, replica_id, - instance_osmorphing_minion_pool_mappings=None, - clone_disks=False, - force=False, skip_os_morphing=False, - user_scripts=None): + def deploy_replica_instances( + self, ctxt, replica_id, + instance_osmorphing_minion_pool_mappings=None, clone_disks=False, + force=False, skip_os_morphing=False, user_scripts=None): return self._call( ctxt, 'deploy_replica_instances', replica_id=replica_id, instance_osmorphing_minion_pool_mappings=( @@ -286,8 +285,8 @@ def set_task_error(self, ctxt, task_id, exception_details): exception_details=exception_details) def add_task_event(self, ctxt, task_id, level, message): - self._cast( - ctxt, 'add_task_event', task_id=task_id, level=level, message=message) + self._cast(ctxt, 'add_task_event', task_id=task_id, + level=level, message=message) def add_task_progress_update( self, ctxt, task_id, message, initial_step=0, total_steps=0, @@ -306,8 +305,8 @@ def update_task_progress_update( self._cast( ctxt, 'update_task_progress_update', task_id=task_id, progress_update_index=progress_update_index, - new_current_step=new_current_step, new_total_steps=new_total_steps, - new_message=new_message) + new_current_step=new_current_step, + new_total_steps=new_total_steps, new_message=new_message) def create_replica_schedule(self, ctxt, replica_id, schedule, enabled, exp_date, @@ -426,7 +425,8 @@ def confirm_replica_minions_allocation( def report_replica_minions_allocation_error( self, ctxt, replica_id, minion_allocation_error_details): self._call( - ctxt, 'report_replica_minions_allocation_error', replica_id=replica_id, + ctxt, 'report_replica_minions_allocation_error', + replica_id=replica_id, minion_allocation_error_details=minion_allocation_error_details) def confirm_migration_minions_allocation( diff --git a/coriolis/conductor/rpc/server.py b/coriolis/conductor/rpc/server.py index 4ca7d46a5..0e82c4e62 100644 --- a/coriolis/conductor/rpc/server.py +++ b/coriolis/conductor/rpc/server.py @@ -763,7 +763,8 @@ def _check_task_cls_param_requirements(task, instance_task_info_keys): for new_field in _check_task_cls_param_requirements( task, task_info_keys): if new_field not in modified_fields_by_queued_tasks: - modified_fields_by_queued_tasks[new_field] = [task] + modified_fields_by_queued_tasks[new_field] = [ + task] else: modified_fields_by_queued_tasks[new_field].append( task) @@ -793,9 +794,8 @@ def _check_task_cls_param_requirements(task, instance_task_info_keys): LOG.debug( "Successfully processed following tasks for instance '%s' " "for execution %s (type '%s') for any state conflict " - "checks: %s", - instance, execution.id, execution.type, [ - (t.id, t.task_type) for t in queued_tasks]) + "checks: %s", instance, execution.id, execution.type, + [(t.id, t.task_type) for t in queued_tasks]) LOG.debug( "Successfully checked all tasks for instance '%s' as part of " "execution '%s' (type '%s') for any state conflicts: %s", @@ -1020,7 +1020,8 @@ def execute_replica_tasks(self, ctxt, replica_id, shutdown_instances): else: self._begin_tasks(ctxt, replica, execution) - return self.get_replica_tasks_execution(ctxt, replica_id, execution.id) + return self.get_replica_tasks_execution( + ctxt, replica_id, execution.id) @replica_synchronized def get_replica_tasks_executions(self, ctxt, replica_id, @@ -1144,7 +1145,8 @@ def delete_replica_disks(self, ctxt, replica_id): LOG.info("Replica tasks execution created: %s", execution.id) self._begin_tasks(ctxt, replica, execution) - return self.get_replica_tasks_execution(ctxt, replica_id, execution.id) + return self.get_replica_tasks_execution( + ctxt, replica_id, execution.id) @staticmethod def _check_endpoints(ctxt, origin_endpoint, destination_endpoint): @@ -1169,7 +1171,8 @@ def create_instances_replica(self, ctxt, origin_endpoint_id, network_map, storage_mappings, notes=None, user_scripts=None): origin_endpoint = self.get_endpoint(ctxt, origin_endpoint_id) - destination_endpoint = self.get_endpoint(ctxt, destination_endpoint_id) + destination_endpoint = self.get_endpoint( + ctxt, destination_endpoint_id) self._check_endpoints(ctxt, origin_endpoint, destination_endpoint) replica = models.Replica() @@ -1271,11 +1274,10 @@ def _get_provider_types(self, ctxt, endpoint): return provider_types["types"] @replica_synchronized - def deploy_replica_instances(self, ctxt, replica_id, - clone_disks, force, - instance_osmorphing_minion_pool_mappings=None, - skip_os_morphing=False, - user_scripts=None): + def deploy_replica_instances( + self, ctxt, replica_id, clone_disks, force, + instance_osmorphing_minion_pool_mappings=None, + skip_os_morphing=False, user_scripts=None): replica = self._get_replica(ctxt, replica_id, include_task_info=True) self._check_reservation_for_transfer( replica, licensing_client.RESERVATION_TYPE_REPLICA) @@ -1710,16 +1712,16 @@ def report_migration_minions_allocation_error( ctxt, execution, constants.EXECUTION_STATUS_ERROR_ALLOCATING_MINIONS) - def migrate_instances(self, ctxt, origin_endpoint_id, - destination_endpoint_id, origin_minion_pool_id, - destination_minion_pool_id, - instance_osmorphing_minion_pool_mappings, - source_environment, destination_environment, - instances, network_map, storage_mappings, - replication_count, shutdown_instances=False, - notes=None, skip_os_morphing=False, user_scripts=None): + def migrate_instances( + self, ctxt, origin_endpoint_id, destination_endpoint_id, + origin_minion_pool_id, destination_minion_pool_id, + instance_osmorphing_minion_pool_mappings, source_environment, + destination_environment, instances, network_map, storage_mappings, + replication_count, shutdown_instances=False, notes=None, + skip_os_morphing=False, user_scripts=None): origin_endpoint = self.get_endpoint(ctxt, origin_endpoint_id) - destination_endpoint = self.get_endpoint(ctxt, destination_endpoint_id) + destination_endpoint = self.get_endpoint( + ctxt, destination_endpoint_id) self._check_endpoints(ctxt, origin_endpoint, destination_endpoint) destination_provider_types = self._get_provider_types( @@ -2611,9 +2613,9 @@ def _start_task(task): LOG.error( "No info present for instance '%s' in action '%s' for task" " '%s' (type '%s') of execution '%s' (type '%s'). " - "Defaulting to empty dict." % ( - task.instance, action.id, task.id, task.task_type, - execution.id, execution.type)) + "Defaulting to empty dict." % + (task.instance, action.id, task.id, task.task_type, + execution.id, execution.type)) task_info = {} else: task_info = action.info[task.instance] @@ -2751,8 +2753,8 @@ def _start_task(task): "tasks have been finalized and there are " "no non-error parents to directly depend on, " "but one or more on-error tasks have completed" - " successfully: %s", - task.id, parent_task_statuses) + " successfully: %s", task.id, + parent_task_statuses) task_statuses[task.id] = _start_task(task) # start on-error tasks only if at least one non-error # parent task has completed successfully: @@ -2989,7 +2991,8 @@ def _check_other_tasks_running(execution, current_task): task_info['destination_minion_machine_id'], task.id, task_type, updated_values) db_api.update_minion_machine( - ctxt, task_info['destination_minion_machine_id'], updated_values) + ctxt, task_info['destination_minion_machine_id'], + updated_values) elif task_type in ( constants.TASK_TYPE_ATTACH_VOLUMES_TO_OSMORPHING_MINION, @@ -3137,7 +3140,8 @@ def task_completed(self, ctxt, task_id, task_result): execution.type] % execution.action_id, external=True): action_id = execution.action_id - action = db_api.get_action(ctxt, action_id, include_task_info=True) + action = db_api.get_action( + ctxt, action_id, include_task_info=True) updated_task_info = None if task_result: @@ -3204,8 +3208,8 @@ def _cancel_execution_for_osmorphing_debugging(self, ctxt, execution): msg = ( "%s Please note that any cleanup operations this task " "should have included will need to performed manually " - "once the debugging process has been completed." % ( - msg)) + "once the debugging process has been completed." % + (msg)) db_api.set_task_status( ctxt, subtask.id, constants.TASK_STATUS_CANCELED_FOR_DEBUGGING, @@ -3503,8 +3507,8 @@ def delete_replica_schedule(self, ctxt, replica_id, schedule_id): if replica_status not in valid_statuses: raise exception.InvalidReplicaState( 'Replica Schedule cannot be deleted while the Replica is in ' - '%s state. Please wait for the Replica execution to finish' % ( - replica_status)) + '%s state. Please wait for the Replica execution to finish' % + (replica_status)) db_api.delete_replica_schedule( ctxt, replica_id, schedule_id, None, lambda ctxt, sched: self._cleanup_schedule_resources( @@ -3620,7 +3624,8 @@ def update_replica( self._begin_tasks(ctxt, replica, execution) - return self.get_replica_tasks_execution(ctxt, replica_id, execution.id) + return self.get_replica_tasks_execution( + ctxt, replica_id, execution.id) def get_diagnostics(self, ctxt): diagnostics = utils.get_diagnostics_info() diff --git a/coriolis/constants.py b/coriolis/constants.py index e4fad66b5..0f6c89815 100644 --- a/coriolis/constants.py +++ b/coriolis/constants.py @@ -268,8 +268,8 @@ DEFAULT_OS_TYPE = OS_TYPE_LINUX -VALID_OS_TYPES = [ - OS_TYPE_BSD, OS_TYPE_LINUX, OS_TYPE_OS_X, OS_TYPE_SOLARIS, OS_TYPE_WINDOWS] +VALID_OS_TYPES = [OS_TYPE_BSD, OS_TYPE_LINUX, + OS_TYPE_OS_X, OS_TYPE_SOLARIS, OS_TYPE_WINDOWS] TMP_DIRS_KEY = "__tmp_dirs" diff --git a/coriolis/context.py b/coriolis/context.py index fdbcdab11..a2a4dc49c 100644 --- a/coriolis/context.py +++ b/coriolis/context.py @@ -22,21 +22,15 @@ def __init__(self, user, project_id, is_admin=None, show_deleted=None, trust_id=None, delete_trust_id=False, **kwargs): - super(RequestContext, self).__init__(auth_token=auth_token, - user=user, - project_id=project_id, - domain_name=domain_name, - domain_id=domain_id, - user_domain_name=user_domain_name, - user_domain_id=user_domain_id, - project_domain_name=( - project_domain_name), - project_domain_id=( - project_domain_id), - is_admin=is_admin, - show_deleted=show_deleted, - request_id=request_id, - overwrite=overwrite) + super( + RequestContext, self).__init__( + auth_token=auth_token, user=user, project_id=project_id, + domain_name=domain_name, domain_id=domain_id, + user_domain_name=user_domain_name, user_domain_id=user_domain_id, + project_domain_name=(project_domain_name), + project_domain_id=(project_domain_id), + is_admin=is_admin, show_deleted=show_deleted, + request_id=request_id, overwrite=overwrite) self.roles = roles or [] self.project_name = project_name self.remote_address = remote_address diff --git a/coriolis/cron/cron.py b/coriolis/cron/cron.py index 5a23f9ff9..4c1ab4945 100644 --- a/coriolis/cron/cron.py +++ b/coriolis/cron/cron.py @@ -108,7 +108,8 @@ def should_run(self, dt): 'minute', 'second', 'dow') dt_fields = dict(zip(fields, dt.timetuple())) - pairs = [(dt_fields[i], self.schedule.get(i)) for i in SCHEDULE_FIELDS] + pairs = [(dt_fields[i], self.schedule.get(i)) + for i in SCHEDULE_FIELDS] compared = self._compare(pairs) return False not in compared @@ -211,9 +212,9 @@ def _result_loop(self): # TODO(gsamfira): send this to the controller and update # the logs table...or do something much more meaningful if error: - LOG.error("Job %(job_desc)s exited with error: %(job_err)r" % { - "job_desc": desc, - "job_err": error}) + LOG.error( + "Job %(job_desc)s exited with error: %(job_err)r" % + {"job_desc": desc, "job_err": error}) if result: LOG.info("Job %(desc)s returned: %(ret)r" % { "desc": desc, diff --git a/coriolis/db/api.py b/coriolis/db/api.py index 5a5d6d459..6a39ef6b3 100644 --- a/coriolis/db/api.py +++ b/coriolis/db/api.py @@ -65,7 +65,8 @@ def _model_query(context, *args): return session.query(*args) -def _update_sqlalchemy_object_fields(obj, updateable_fields, values_to_update): +def _update_sqlalchemy_object_fields( + obj, updateable_fields, values_to_update): """ Updates the given 'values_to_update' on the provided sqlalchemy object as long as they are included as 'updateable_fields'. :param obj: object: sqlalchemy object @@ -886,7 +887,8 @@ def add_task_progress_update( def update_task_progress_update( context, task_id, update_index, new_current_step, new_total_steps=None, new_message=None): - task_progress_update = _get_progress_update(context, task_id, update_index) + task_progress_update = _get_progress_update( + context, task_id, update_index) if not task_progress_update: raise exception.NotFound( "Could not find progress update for task with ID '%s' and " diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/001_initial.py b/coriolis/db/sqlalchemy/migrate_repo/versions/001_initial.py index 82bd89ad3..71b6c534d 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/001_initial.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/001_initial.py @@ -45,29 +45,36 @@ def upgrade(migrate_engine): ) task = sqlalchemy.Table( - 'task', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), + 'task', meta, sqlalchemy.Column( + 'id', sqlalchemy.String(36), + primary_key=True, default=lambda: str(uuid.uuid4())), sqlalchemy.Column('created_at', sqlalchemy.DateTime), sqlalchemy.Column('updated_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted', sqlalchemy.String(36)), - sqlalchemy.Column("execution_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey( - 'tasks_execution.id'), - nullable=False), - sqlalchemy.Column("instance", sqlalchemy.String(1024), nullable=False), - sqlalchemy.Column("host", sqlalchemy.String(1024), nullable=True), - sqlalchemy.Column("process_id", sqlalchemy.Integer, nullable=True), - sqlalchemy.Column("status", sqlalchemy.String(100), nullable=False), - sqlalchemy.Column("task_type", sqlalchemy.String(100), - nullable=False), - sqlalchemy.Column("exception_details", sqlalchemy.Text, nullable=True), + sqlalchemy.Column( + "execution_id", sqlalchemy.String(36), + sqlalchemy.ForeignKey('tasks_execution.id'), + nullable=False), + sqlalchemy.Column( + "instance", sqlalchemy.String(1024), + nullable=False), + sqlalchemy.Column( + "host", sqlalchemy.String(1024), + nullable=True), + sqlalchemy.Column( + "process_id", sqlalchemy.Integer, nullable=True), + sqlalchemy.Column( + "status", sqlalchemy.String(100), + nullable=False), + sqlalchemy.Column( + "task_type", sqlalchemy.String(100), + nullable=False), + sqlalchemy.Column( + "exception_details", sqlalchemy.Text, nullable=True), sqlalchemy.Column("depends_on", sqlalchemy.Text, nullable=True), sqlalchemy.Column("on_error", sqlalchemy.Boolean, nullable=True), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) + mysql_engine='InnoDB', mysql_charset='utf8') tasks_execution = sqlalchemy.Table( 'tasks_execution', meta, diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/016_adds_minion_vm_pools.py b/coriolis/db/sqlalchemy/migrate_repo/versions/016_adds_minion_vm_pools.py index 9cd6b8dc0..a2aec551e 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/016_adds_minion_vm_pools.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/016_adds_minion_vm_pools.py @@ -25,36 +25,42 @@ def upgrade(migrate_engine): # add table for pool lifecycles: tables.append( sqlalchemy.Table( - 'minion_pool', - meta, - sqlalchemy.Column( + 'minion_pool', meta, sqlalchemy.Column( "id", sqlalchemy.String(36), - default=lambda: str(uuid.uuid4()), primary_key=True), + default=lambda: str(uuid.uuid4()), + primary_key=True), sqlalchemy.Column("notes", sqlalchemy.Text, nullable=True), sqlalchemy.Column( - "user_id", sqlalchemy.String(255), nullable=False), + "user_id", sqlalchemy.String(255), + nullable=False), sqlalchemy.Column( - "project_id", sqlalchemy.String(255), nullable=False), + "project_id", sqlalchemy.String(255), + nullable=False), sqlalchemy.Column( - "maintenance_trust_id", sqlalchemy.String(255), nullable=True), + "maintenance_trust_id", sqlalchemy.String(255), + nullable=True), sqlalchemy.Column('created_at', sqlalchemy.DateTime), sqlalchemy.Column('updated_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted', sqlalchemy.String(36)), sqlalchemy.Column( - "name", sqlalchemy.String(255), nullable=False), + "name", sqlalchemy.String(255), + nullable=False), sqlalchemy.Column( "endpoint_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey('endpoint.id'), nullable=False), + sqlalchemy.ForeignKey('endpoint.id'), + nullable=False), sqlalchemy.Column( "environment_options", sqlalchemy.Text, nullable=False), sqlalchemy.Column( - "os_type", sqlalchemy.String(255), nullable=False), + "os_type", sqlalchemy.String(255), + nullable=False), sqlalchemy.Column( - "platform", sqlalchemy.String(255), nullable=True), + "platform", sqlalchemy.String(255), + nullable=True), sqlalchemy.Column( - "status", sqlalchemy.String(255), nullable=False, - default=lambda: "UNKNOWN"), + "status", sqlalchemy.String(255), + nullable=False, default=lambda: "UNKNOWN"), sqlalchemy.Column( "shared_resources", sqlalchemy.Text, nullable=True), sqlalchemy.Column( @@ -121,24 +127,27 @@ def upgrade(migrate_engine): mysql_engine='InnoDB', mysql_charset='utf8')) - tables.append(sqlalchemy.Table( - 'minion_pool_progress_update', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), - sqlalchemy.Column('index', sqlalchemy.Integer, default=0), - sqlalchemy.Column('deleted', sqlalchemy.String(36)), - sqlalchemy.Column("pool_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey('minion_pool.id'), - nullable=False), - sqlalchemy.Column( - "current_step", sqlalchemy.BigInteger, nullable=False), - sqlalchemy.Column("total_steps", sqlalchemy.BigInteger, nullable=True), - sqlalchemy.Column("message", sqlalchemy.Text, nullable=True), - mysql_engine='InnoDB', - mysql_charset='utf8')) + tables.append( + sqlalchemy.Table( + 'minion_pool_progress_update', meta, sqlalchemy.Column( + 'id', sqlalchemy.String(36), + primary_key=True, default=lambda: str(uuid.uuid4())), + sqlalchemy.Column('created_at', sqlalchemy.DateTime), + sqlalchemy.Column('updated_at', sqlalchemy.DateTime), + sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), + sqlalchemy.Column('index', sqlalchemy.Integer, default=0), + sqlalchemy.Column('deleted', sqlalchemy.String(36)), + sqlalchemy.Column( + "pool_id", sqlalchemy.String(36), + sqlalchemy.ForeignKey('minion_pool.id'), + nullable=False), + sqlalchemy.Column( + "current_step", sqlalchemy.BigInteger, nullable=False), + sqlalchemy.Column( + "total_steps", sqlalchemy.BigInteger, nullable=True), + sqlalchemy.Column( + "message", sqlalchemy.Text, nullable=True), + mysql_engine='InnoDB', mysql_charset='utf8')) # add the pool option properties for the transfer: origin_minion_pool_id = sqlalchemy.Column( diff --git a/coriolis/db/sqlalchemy/models.py b/coriolis/db/sqlalchemy/models.py index 6be5070c6..bd5da9881 100644 --- a/coriolis/db/sqlalchemy/models.py +++ b/coriolis/db/sqlalchemy/models.py @@ -45,7 +45,7 @@ def to_dict(self): class MinionPoolEvent(BASE, models.TimestampMixin, models.SoftDeleteMixin, - models.ModelBase): + models.ModelBase): __tablename__ = 'minion_pool_event' id = sqlalchemy.Column(sqlalchemy.String(36), @@ -538,8 +538,8 @@ def to_dict(self): class MinionPool( - BASE, models.TimestampMixin, models.ModelBase, - models.SoftDeleteMixin): + BASE, models.TimestampMixin, models.ModelBase, + models.SoftDeleteMixin): __tablename__ = 'minion_pool' id = sqlalchemy.Column( diff --git a/coriolis/db/sqlalchemy/types.py b/coriolis/db/sqlalchemy/types.py index a9c45704f..e1671b73b 100644 --- a/coriolis/db/sqlalchemy/types.py +++ b/coriolis/db/sqlalchemy/types.py @@ -56,7 +56,7 @@ class Bson(Blob): def process_bind_param(self, value, dialect): return zlib.compress( - jsonutils.dumps(value).encode('utf-8')) + jsonutils.dumps(value).encode('utf-8')) def process_result_value(self, value, dialect): if value is None: diff --git a/coriolis/events.py b/coriolis/events.py index 7026087d0..66021fcae 100644 --- a/coriolis/events.py +++ b/coriolis/events.py @@ -56,18 +56,19 @@ def add_percentage_step(self, message, total_steps, initial_step=0): if initial_step > 0 and total_steps > 0: perc = int(initial_step * 100 // total_steps) self._perc_steps[progress_update_id] = _PercStepData( - progress_update_id, perc, initial_step, total_steps) + progress_update_id, perc, initial_step, total_steps) return self._perc_steps[progress_update_id] def set_percentage_step(self, step, new_current_step): perc_step = self._perc_steps.get( - step.progress_update_id, None) + step.progress_update_id, None) if perc_step is None: return if perc_step.last_value > new_current_step: - LOG.warn("rollback for perc update %s not allowed" % step.progress_update_id) + LOG.warn("rollback for perc update %s not allowed" % + step.progress_update_id) return perc = 0 diff --git a/coriolis/exception.py b/coriolis/exception.py index eab5c0979..5d396f849 100644 --- a/coriolis/exception.py +++ b/coriolis/exception.py @@ -506,8 +506,7 @@ class OSMorphingSSHOperationTimeout(OSMorphingOperationTimeout): "Coriolis may have encountered connection issues to the minion machine" " or the command execution time exceeds the timeout set. Try extending" " the timeout by editing the 'default_osmorphing_operation_timeout' " - "in Coriolis' static configuration file." - ) + "in Coriolis' static configuration file.") class OSMorphingWinRMOperationTimeout(OSMorphingOperationTimeout): @@ -516,5 +515,4 @@ class OSMorphingWinRMOperationTimeout(OSMorphingOperationTimeout): "Coriolis may have encountered connection issues to the minion machine" " or the command execution time exceeds the timeout set. Try extending" " the timeout by editing the 'default_osmorphing_operation_timeout' " - "in Coriolis' static configuration file." - ) + "in Coriolis' static configuration file.") diff --git a/coriolis/keystone.py b/coriolis/keystone.py index 4a11ae012..c77a28a33 100644 --- a/coriolis/keystone.py +++ b/coriolis/keystone.py @@ -65,13 +65,13 @@ def create_trust(ctxt): trustor_proj_id = ctxt.project_id roles = ctxt.roles - LOG.debug("Granting Keystone trust. Trustor: %(trustor_user_id)s, trustee:" - " %(trustee_user_id)s, project: %(trustor_proj_id)s, roles:" - " %(roles)s", - {"trustor_user_id": trustor_user_id, - "trustee_user_id": trustee_user_id, - "trustor_proj_id": trustor_proj_id, - "roles": roles}) + LOG.debug( + "Granting Keystone trust. Trustor: %(trustor_user_id)s, trustee:" + " %(trustee_user_id)s, project: %(trustor_proj_id)s, roles:" + " %(roles)s", + {"trustor_user_id": trustor_user_id, + "trustee_user_id": trustee_user_id, + "trustor_proj_id": trustor_proj_id, "roles": roles}) # Trusts are not supported before Keystone v3 client = kc_v3.Client(session=session) diff --git a/coriolis/licensing/client.py b/coriolis/licensing/client.py index 1fceb008a..8f45165a8 100644 --- a/coriolis/licensing/client.py +++ b/coriolis/licensing/client.py @@ -140,7 +140,8 @@ def _put(self, resource, body, response_key=None, appliance_scoped=True): response_key=response_key, appliance_scoped=appliance_scoped) - def _delete(self, resource, body, response_key=None, appliance_scoped=True): + def _delete(self, resource, body, response_key=None, + appliance_scoped=True): return self._do_req( "DELETE", resource, body=body, response_key=response_key, diff --git a/coriolis/migrations/manager.py b/coriolis/migrations/manager.py index a1c837b79..98943e07f 100644 --- a/coriolis/migrations/manager.py +++ b/coriolis/migrations/manager.py @@ -37,7 +37,8 @@ def _copy_volume(volume, disk_image_reader, backup_writer, event_manager): offset, max_block_size) if not allocated or zero_block and skip_zeroes: if not allocated: - LOG.debug("Unallocated block detected: %s", block_size) + LOG.debug( + "Unallocated block detected: %s", block_size) else: LOG.debug("Skipping zero block: %s", block_size) offset += block_size diff --git a/coriolis/minion_manager/rpc/client.py b/coriolis/minion_manager/rpc/client.py index 7225afc7d..4803047d8 100644 --- a/coriolis/minion_manager/rpc/client.py +++ b/coriolis/minion_manager/rpc/client.py @@ -26,15 +26,16 @@ class MinionManagerClient(rpc.BaseRPCClient): def __init__(self, timeout=None): - target = messaging.Target(topic='coriolis_minion_manager', version=VERSION) + target = messaging.Target( + topic='coriolis_minion_manager', version=VERSION) if timeout is None: timeout = CONF.minion_manager.minion_mananger_rpc_timeout super(MinionManagerClient, self).__init__( target, timeout=timeout) def add_minion_pool_progress_update( - self, ctxt, minion_pool_id, message, initial_step=0, total_steps=0, - return_event=False): + self, ctxt, minion_pool_id, message, initial_step=0, + total_steps=0, return_event=False): operation = self._cast if return_event: operation = self._call @@ -44,14 +45,14 @@ def add_minion_pool_progress_update( initial_step=initial_step, total_steps=total_steps) def update_minion_pool_progress_update( - self, ctxt, minion_pool_id, progress_update_index, new_current_step, - new_total_steps=None, new_message=None): + self, ctxt, minion_pool_id, progress_update_index, + new_current_step, new_total_steps=None, new_message=None): self._cast( ctxt, 'update_minion_pool_progress_update', minion_pool_id=minion_pool_id, progress_update_index=progress_update_index, - new_current_step=new_current_step, new_total_steps=new_total_steps, - new_message=new_message) + new_current_step=new_current_step, + new_total_steps=new_total_steps, new_message=new_message) def add_minion_pool_event(self, ctxt, minion_pool_id, level, message): return self._cast( @@ -81,13 +82,14 @@ def allocate_minion_machines_for_migration( include_osmorphing_minions=include_osmorphing_minions) def deallocate_minion_machine(self, ctxt, minion_machine_id): - return self._cast( + return self._cast( ctxt, 'deallocate_minion_machine', minion_machine_id=minion_machine_id) def deallocate_minion_machines_for_action(self, ctxt, action_id): return self._cast( - ctxt, 'deallocate_minion_machines_for_action', action_id=action_id) + ctxt, 'deallocate_minion_machines_for_action', + action_id=action_id) def create_minion_pool( self, ctxt, name, endpoint_id, pool_platform, pool_os_type, diff --git a/coriolis/minion_manager/rpc/server.py b/coriolis/minion_manager/rpc/server.py index 114513c11..3fd600646 100644 --- a/coriolis/minion_manager/rpc/server.py +++ b/coriolis/minion_manager/rpc/server.py @@ -294,7 +294,8 @@ def add_minion_pool_event(self, ctxt, minion_pool_id, level, message): self._add_minion_pool_event(ctxt, minion_pool_id, level, message) def _add_minion_pool_progress_update( - self, ctxt, minion_pool_id, message, initial_step=0, total_steps=0): + self, ctxt, minion_pool_id, message, initial_step=0, + total_steps=0): LOG.info( "Adding pool progress update for %s: %s", minion_pool_id, message) db_api.add_minion_pool_progress_update( @@ -303,7 +304,8 @@ def _add_minion_pool_progress_update( @minion_manager_utils.minion_pool_synchronized_op def add_minion_pool_progress_update( - self, ctxt, minion_pool_id, message, initial_step=0, total_steps=0): + self, ctxt, minion_pool_id, message, initial_step=0, + total_steps=0): self._add_minion_pool_progress_update( ctxt, minion_pool_id, message, initial_step=initial_step, total_steps=total_steps) @@ -351,12 +353,14 @@ def validate_minion_pool_selections_for_action(self, ctxt, action): for pool in db_api.get_minion_pools( ctxt, include_machines=False, include_events=False, include_progress_updates=False, to_dict=False)} + def _get_pool(pool_id): pool = minion_pools.get(pool_id) if not pool: raise exception.NotFound( "Could not find minion pool with ID '%s'." % pool_id) return pool + def _check_pool_minion_count( minion_pool, instances, minion_pool_type=""): desired_minion_count = len(instances) @@ -364,7 +368,7 @@ def _check_pool_minion_count( raise exception.InvalidMinionPoolState( "Minion Pool '%s' is an invalid state ('%s') to be " "used as a %s pool for action '%s'. The pool must be " - "in '%s' status." % ( + "in '%s' status." % ( minion_pool.id, minion_pool.status, minion_pool_type.lower(), action['id'], constants.MINION_POOL_STATUS_ALLOCATED)) @@ -563,13 +567,14 @@ def _make_minion_machine_allocation_subflow_for_action( """ currently_available_machines = [ machine for machine in minion_pool.minion_machines - if machine.allocation_status == constants.MINION_MACHINE_STATUS_AVAILABLE] + if machine.allocation_status + == constants.MINION_MACHINE_STATUS_AVAILABLE] extra_available_machine_slots = ( minion_pool.maximum_minions - len(minion_pool.minion_machines)) num_instances = len(action_instances) num_currently_available_machines = len(currently_available_machines) if num_instances > (len(currently_available_machines) + ( - extra_available_machine_slots)): + extra_available_machine_slots)): raise exception.InvalidMinionPoolState( "Minion pool '%s' is unable to accommodate the requested " "number of machines (%s) for transfer action '%s', as it only " @@ -616,8 +621,8 @@ def _select_machine(minion_pool, exclude=None): raise exception.InvalidInput( "Instance with identifier '%s' passed twice for " "minion machine allocation from pool '%s' for action " - "'%s'. Full instances list was: %s" % ( - instance, minion_pool.id, action_id, action_instances)) + "'%s'. Full instances list was: %s" % + (instance, minion_pool.id, action_id, action_instances)) minion_machine = _select_machine( minion_pool, exclude=instance_minion_allocations.values()) if minion_machine: @@ -816,7 +821,7 @@ def _check_pool_allocation_status( raise exception.InvalidMinionPoolState( "Minion Pool '%s' is an invalid state ('%s') to be " "used as a %s pool for action '%s'. The pool must be " - "in '%s' status." % ( + "in '%s' status." % ( minion_pool.id, minion_pool.status, minion_pool_type.lower(), action['id'], constants.MINION_POOL_STATUS_ALLOCATED)) @@ -920,7 +925,8 @@ def _check_pool_allocation_status( "Reusing destination minion pool with ID '%s' for the " "following instances which had it selected as an " "OSMorphing pool for action '%s': %s", - osmorphing_pool_id, action['id'], action_instance_ids) + osmorphing_pool_id, action['id'], + action_instance_ids) for instance in action_instance_ids: instance_machine_allocations[ instance]['osmorphing_minion_id'] = ( @@ -1042,7 +1048,7 @@ def _cleanup_machines_with_statuses_for_action( for (pool_id, machines) in pool_machine_mappings.items(): with minion_manager_utils.get_minion_pool_lock( - pool_id, external=True): + pool_id, external=True): for machine in machines: LOG.debug( "Deleting machine with ID '%s' (pool '%s', status '%s') " @@ -1118,19 +1124,20 @@ def deallocate_minion_machines_for_action(self, ctxt, action_id): LOG.warn( "Found minion machine '%s' in pool '%s' which " "is in '%s' status. Removing from the DB " - "entirely." % ( - machine.id, pool_id, machine.allocation_status)) + "entirely." % + (machine.id, pool_id, machine.allocation_status)) db_api.delete_minion_machine( ctxt, machine.id) LOG.info( "Successfully deleted minion machine entry '%s' " - "from pool '%s' from the DB.", machine.id, pool_id) + "from pool '%s' from the DB.", machine.id, + pool_id) continue LOG.debug( "Going to mark minion machine '%s' (current status " "'%s') of pool '%s' as available following machine " - "deallocation request for action '%s'.", - machine.id, machine.allocation_status, pool_id, action_id) + "deallocation request for action '%s'.", machine.id, + machine.allocation_status, pool_id, action_id) machine_ids_to_deallocate.append(machine.id) LOG.info( @@ -1322,8 +1329,8 @@ def _get_minion_pool_refresh_flow( constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_DELETE): pool_refresh_flow.add( minion_manager_tasks.DeallocateMinionMachineTask( - minion_pool.id, machine.id, - minion_pool.platform)) + minion_pool.id, machine.id, + minion_pool.platform)) else: raise exception.InvalidMinionPoolState( "Unknown minion pool retention strategy '%s' for pool " @@ -1347,7 +1354,7 @@ def _get_minion_pool_refresh_flow( # update DB entried for all machines and emit relevant events: if skipped_machines: - base_msg = ( + base_msg = ( "The following minion machines were skipped during the " "refreshing of the minion pool as they were in other " "statuses than the serviceable ones: %s") @@ -1618,8 +1625,8 @@ def _check_pool_machines_in_use( def allocate_minion_pool(self, ctxt, minion_pool_id): LOG.info("Attempting to allocate Minion Pool '%s'.", minion_pool_id) minion_pool = self._get_minion_pool( - ctxt, minion_pool_id, include_events=False, include_machines=False, - include_progress_updates=False) + ctxt, minion_pool_id, include_events=False, + include_machines=False, include_progress_updates=False) endpoint_dict = self._rpc_conductor_client.get_endpoint( ctxt, minion_pool.endpoint_id) acceptable_allocation_statuses = [ @@ -1629,9 +1636,9 @@ def allocate_minion_pool(self, ctxt, minion_pool_id): raise exception.InvalidMinionPoolState( "Minion machines for pool '%s' cannot be allocated as the pool" " is in '%s' state instead of the expected %s. Please " - "force-deallocate the pool and try again." % ( - minion_pool_id, minion_pool.status, - acceptable_allocation_statuses)) + "force-deallocate the pool and try again." % + (minion_pool_id, minion_pool.status, + acceptable_allocation_statuses)) allocation_flow = self._get_minion_pool_allocation_flow(minion_pool) initial_store = self._get_pool_initial_taskflow_store_base( @@ -1691,7 +1698,8 @@ def _get_minion_pool_deallocation_flow( deallocation_flow.add(machines_flow) else: LOG.debug( - "No machines for pool '%s' require deallocating.", minion_pool.id) + "No machines for pool '%s' require deallocating.", + minion_pool.id) # transition pool to DEALLOCATING_SHARED_RESOURCES: deallocation_flow.add(minion_manager_tasks.UpdateMinionPoolStatusTask( @@ -1739,7 +1747,7 @@ def deallocate_minion_pool(self, ctxt, minion_pool_id, force=False): if not force: raise exception.InvalidMinionPoolState( "Minion pool '%s' cannot be deallocated as the pool" - " is in '%s' state instead of one of the expected %s"% ( + " is in '%s' state instead of one of the expected %s" % ( minion_pool_id, minion_pool.status, acceptable_deallocation_statuses)) else: diff --git a/coriolis/minion_manager/rpc/tasks.py b/coriolis/minion_manager/rpc/tasks.py index 8bf46947a..4439fa0b5 100644 --- a/coriolis/minion_manager/rpc/tasks.py +++ b/coriolis/minion_manager/rpc/tasks.py @@ -919,10 +919,8 @@ def revert(self, context, origin, destination, task_info, **kwargs): "[Task %s] Removing minion machine entry with ID '%s' for " "minion pool '%s' from the DB as part of reversion of its " "allocation task. Machine properties at deletion time " - "were: %s", - self._task_name, self._minion_machine_id, - self._minion_pool_id, - machine_db_entry.to_dict()) + "were: %s", self._task_name, self._minion_machine_id, + self._minion_pool_id, machine_db_entry.to_dict()) if not minion_provider_properties and ( machine_db_entry.provider_properties): minion_provider_properties = ( diff --git a/coriolis/osmorphing/base.py b/coriolis/osmorphing/base.py index 222d3be24..95c078126 100644 --- a/coriolis/osmorphing/base.py +++ b/coriolis/osmorphing/base.py @@ -21,7 +21,8 @@ # Required OS release fields which are expected from the OSDetect tools. # 'schemas.CORIOLIS_DETECTED_OS_MORPHING_INFO_SCHEMA' schema: REQUIRED_DETECTED_OS_FIELDS = [ - "os_type", "distribution_name", "release_version", "friendly_release_name"] + "os_type", "distribution_name", "release_version", + "friendly_release_name"] class BaseOSMorphingTools(object, with_metaclass(abc.ABCMeta)): diff --git a/coriolis/osmorphing/osdetect/base.py b/coriolis/osmorphing/osdetect/base.py index 1e519db79..82746d938 100644 --- a/coriolis/osmorphing/osdetect/base.py +++ b/coriolis/osmorphing/osdetect/base.py @@ -13,7 +13,8 @@ # Required OS release fields to be returned as declared in the # 'schemas.CORIOLIS_DETECTED_OS_MORPHING_INFO_SCHEMA' schema: REQUIRED_DETECTED_OS_FIELDS = [ - "os_type", "distribution_name", "release_version", "friendly_release_name"] + "os_type", "distribution_name", "release_version", + "friendly_release_name"] class BaseOSDetectTools(object, with_metaclass(abc.ABCMeta)): diff --git a/coriolis/osmorphing/osdetect/centos.py b/coriolis/osmorphing/osdetect/centos.py index 012c741c5..520556c0f 100644 --- a/coriolis/osmorphing/osdetect/centos.py +++ b/coriolis/osmorphing/osdetect/centos.py @@ -28,7 +28,8 @@ def detect_os(self): distro, version, _, _ = m.groups() if CENTOS_DISTRO_IDENTIFIER not in distro: LOG.debug( - "Distro does not appear to be a CentOS: %s", distro) + "Distro does not appear to be a CentOS: %s", + distro) return {} distribution_name = CENTOS_DISTRO_IDENTIFIER diff --git a/coriolis/osmorphing/osmount/base.py b/coriolis/osmorphing/osmount/base.py index fbf393a30..1bab4144a 100644 --- a/coriolis/osmorphing/osmount/base.py +++ b/coriolis/osmorphing/osmount/base.py @@ -81,8 +81,8 @@ def _connect(self): utils.wait_for_port_connectivity(ip, port) self._event_manager.progress_update( - "Connecting through SSH to OSMorphing host on: %(ip)s:%(port)s" % ( - {"ip": ip, "port": port})) + "Connecting through SSH to OSMorphing host on: %(ip)s:%(port)s" % + ({"ip": ip, "port": port})) ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname=ip, port=port, username=username, pkey=pkey, @@ -152,7 +152,8 @@ def _get_vgs(self): "VG with name '%s' already detected. Renaming VG with " "UUID '%s' to '%s' to avoid conflicts", vg_name, vg_uuid, new_name) - self._exec_cmd("sudo vgrename %s %s" % (vg_uuid, new_name)) + self._exec_cmd("sudo vgrename %s %s" % + (vg_uuid, new_name)) vgs[new_name] = pv_name else: LOG.warning("Ignoring improper `vgs` output entry: %s", line) @@ -279,8 +280,8 @@ def _check_mount_fstab_partitions( "%s. Only LVM volumes or devices referenced by UUID=* " "or /dev/disk/by-uuid/* notation are supported. " "Devicemapper paths for LVM volumes are also " - "supported. Skipping mounting directory." % ( - mountpoint, device)) + "supported. Skipping mounting directory." % + (mountpoint, device)) continue if mountpoint in skip_mounts: LOG.debug( @@ -614,7 +615,8 @@ def dismount_os(self, root_dir): self._exec_cmd('sudo umount %s' % d) dev_fs = "%s/%s" % (root_dir.rstrip('/'), "dev") - self._exec_cmd('mountpoint -q %s && sudo umount %s' % (dev_fs, dev_fs)) + self._exec_cmd('mountpoint -q %s && sudo umount %s' % + (dev_fs, dev_fs)) self._exec_cmd( 'mountpoint -q %s && sudo umount %s' % (root_dir, root_dir)) diff --git a/coriolis/osmorphing/osmount/windows.py b/coriolis/osmorphing/osmount/windows.py index a349d5d20..27a02f40b 100644 --- a/coriolis/osmorphing/osmount/windows.py +++ b/coriolis/osmorphing/osmount/windows.py @@ -77,9 +77,12 @@ def _service_disks_with_status( # Disk Group also imports the other ones), so we must take care to # re-status before performing any operation on any disk => O(n**2) disk_list = self._run_diskpart_script(disk_list_script) - servicable_disk_ids = [m.group(1) for m in [ - re.match(search_disk_entry_re, l) for l in disk_list.split("\r\n")] - if m is not None] + servicable_disk_ids = [ + m.group(1) + for m + in + [re.match(search_disk_entry_re, l) + for l in disk_list.split("\r\n")] if m is not None] LOG.debug( "Servicing disks with status '%s' (%s) from disk list: %s", status, servicable_disk_ids, disk_list) @@ -101,8 +104,8 @@ def _service_disks_with_status( LOG.warn( "Exception ocurred while servicing disk '%s' " "with status '%s'.Skipping running script '%s'" - ". Error message: %s" % ( - disk_id, status, script, ex)) + ". Error message: %s" % + (disk_id, status, script, ex)) else: raise break @@ -220,7 +223,8 @@ def _set_volumes_drive_letter(self): for vol_id in unhidden_volume_ids: try: LOG.info( - "Clearing NODEFAULTDRIVELETTER flag on volume %s" % vol_id) + "Clearing NODEFAULTDRIVELETTER flag on volume %s" % + vol_id) script = enable_default_drive_letter_script_fmt % vol_id self._run_diskpart_script(script) except Exception as ex: @@ -239,7 +243,7 @@ def mount_os(self): self._set_volumes_drive_letter() self._refresh_storage() fs_roots = utils.retry_on_error(sleep_seconds=5)(self._get_fs_roots)( - fail_if_empty=True) + fail_if_empty=True) system_drive = self._get_system_drive() for fs_root in [r for r in fs_roots if not r[:-1] == system_drive]: diff --git a/coriolis/osmorphing/redhat.py b/coriolis/osmorphing/redhat.py index 5f5a3480b..53cafb5ce 100644 --- a/coriolis/osmorphing/redhat.py +++ b/coriolis/osmorphing/redhat.py @@ -54,10 +54,10 @@ def check_os_supported(cls, detected_os_info): def __init__(self, conn, os_root_dir, os_root_dev, hypervisor, event_manager, detected_os_info, osmorphing_parameters, operation_timeout=None): - super(BaseRedHatMorphingTools, self).__init__( - conn, os_root_dir, os_root_dev, - hypervisor, event_manager, detected_os_info, osmorphing_parameters, - operation_timeout) + super( + BaseRedHatMorphingTools, self).__init__( + conn, os_root_dir, os_root_dev, hypervisor, event_manager, + detected_os_info, osmorphing_parameters, operation_timeout) def disable_predictable_nic_names(self): cmd = 'grubby --update-kernel=ALL --args="%s"' @@ -160,7 +160,8 @@ def _comment_keys_from_ifcfg_files( for ifcfgf in all_ifcfg_files: if not re.match(regex, ifcfgf): LOG.debug( - "Skipping ifcfg file with unknown filename '%s'." % ifcfgf) + "Skipping ifcfg file with unknown filename '%s'." % + ifcfgf) continue if interfaces and not any([i in ifcfgf for i in interfaces]): diff --git a/coriolis/osmorphing/suse.py b/coriolis/osmorphing/suse.py index d1490fb87..a553f3681 100644 --- a/coriolis/osmorphing/suse.py +++ b/coriolis/osmorphing/suse.py @@ -118,8 +118,8 @@ def _enable_sles_module(self, module): raise exception.CoriolisException( "Failed to activate SLES module: %s. Please check whether the " "SUSE system registration is still valid on the source VM " - "and retry. Review logs for more details. Error was: %s" % ( - module, str(err))) from err + "and retry. Review logs for more details. Error was: %s" % + (module, str(err))) from err def _add_cloud_tools_repo(self): repo_suffix = "" @@ -146,8 +146,9 @@ def _add_repo(self, uri, alias): repos = self._get_repos() if repos.get(alias): if repos[alias] == uri: - LOG.debug('Repo with alias %s already exists and has the same ' - 'URI. Enabling', alias) + LOG.debug( + 'Repo with alias %s already exists and has the same ' + 'URI. Enabling', alias) self._event_manager.progress_update( "Enabling repository: %s" % alias) self._exec_cmd_chroot( @@ -184,7 +185,8 @@ def install_packages(self, package_names): def uninstall_packages(self, package_names): try: self._exec_cmd_chroot( - 'zypper --non-interactive remove %s' % " ".join(package_names)) + 'zypper --non-interactive remove %s' % + " ".join(package_names)) except Exception: self._event_manager.progress_update( "Error occured while uninstalling packages. Ignoring") diff --git a/coriolis/osmorphing/ubuntu.py b/coriolis/osmorphing/ubuntu.py index 38c873d29..25ba11f0f 100644 --- a/coriolis/osmorphing/ubuntu.py +++ b/coriolis/osmorphing/ubuntu.py @@ -101,7 +101,8 @@ def _set_netplan_ethernet_configs( LOG.debug( "Renamed interface '%s' to '%s' in '%s'", iface_name, new_iface_name, config_path_chroot) - new_config = copy.deepcopy(ethernet_configurations[iface_name]) + new_config = copy.deepcopy( + ethernet_configurations[iface_name]) if set_dhcp: new_config["dhcp4"] = True new_config["dhcp6"] = True diff --git a/coriolis/osmorphing/windows.py b/coriolis/osmorphing/windows.py index 0adfca35c..c861bce4e 100644 --- a/coriolis/osmorphing/windows.py +++ b/coriolis/osmorphing/windows.py @@ -148,7 +148,7 @@ $IPS_INFO = ConvertFrom-Json $ips_info_json Invoke-Main $NICS_INFO $IPS_INFO -""" # noqa +""" # noqa class BaseWindowsMorphingTools(base.BaseOSMorphingTools): @@ -221,7 +221,8 @@ def _add_dism_driver(self, driver_path): return self._conn.exec_command( dism_path, ["/add-driver", "/image:%s" % self._os_root_dir, - "/driver:\"%s\"" % driver_path, "/recurse", "/forceunsigned"]) + "/driver:\"%s\"" % driver_path, "/recurse", + "/forceunsigned"]) except Exception as ex: dism_log_path = "%s\\Windows\\Logs\\DISM\\dism.log" % ( self._get_worker_os_drive_path()) @@ -234,7 +235,8 @@ def _add_dism_driver(self, driver_path): driver_path, dism_log_path, dism_log_contents) else: LOG.warn( - "Could not find DISM error logs for failure:'%s'", str(ex)) + "Could not find DISM error logs for failure:'%s'", + str(ex)) raise def _mount_disk_image(self, path): @@ -314,12 +316,9 @@ def _create_service(self, key_name, service_name, image_path, "16 -Type DWord -Force;" "New-ItemProperty -Path '%(path)s' -Name 'ErrorControl' -Value " "0 -Type DWord -Force" % - {"path": registry_path, - "image_path": image_path, - "display_name": display_name, - "description": description, - "depends_on": depends_on_ps, - "service_account": service_account, + {"path": registry_path, "image_path": image_path, + "display_name": display_name, "description": description, + "depends_on": depends_on_ps, "service_account": service_account, "start_mode": start_mode}, ignore_stdout=True) diff --git a/coriolis/providers/backup_writers.py b/coriolis/providers/backup_writers.py index a11a8ac7e..86780df44 100644 --- a/coriolis/providers/backup_writers.py +++ b/coriolis/providers/backup_writers.py @@ -368,8 +368,8 @@ def write(self, data): if self._exception: raise exception.CoriolisException( - "Failed to write data. See log " - "for details.") from self._exception + "Failed to write data. See log " + "for details.") from self._exception payload = { "offset": self._offset, @@ -482,7 +482,8 @@ def _get_impl(self, path, disk_id): if not matching_devs: base_msg = ( - "Could not locate disk with ID '%s' in volumes_info" % disk_id) + "Could not locate disk with ID '%s' in volumes_info" % + disk_id) LOG.error("%s: %s", base_msg, self._volumes_info) raise exception.CoriolisException(base_msg) elif len(matching_devs) > 1: @@ -923,7 +924,7 @@ def _init_writer(self, ssh, cert_paths): "srv_key": cert_paths["srv_key"], "srv_cert": cert_paths["srv_crt"], "listen_port": self._writer_port, - } + } self._change_binary_se_context(ssh) utils.create_service( ssh, cmdline, _CORIOLIS_HTTP_WRITER_CMD, start=True) diff --git a/coriolis/providers/base.py b/coriolis/providers/base.py index 06bedd7a9..d9190a3dc 100644 --- a/coriolis/providers/base.py +++ b/coriolis/providers/base.py @@ -201,7 +201,6 @@ def get_os_morphing_tools(self, os_type, osmorphing_info): """ raise exception.OSMorphingToolsNotFound(os_type=os_type) - def get_custom_os_detect_tools(self, os_type, osmorphing_info): """ Returns a list of custom OSDetect classes which inherit from coriolis.osmorphing.osdetect.base.BaseOSDetectTools. diff --git a/coriolis/providers/provider_utils.py b/coriolis/providers/provider_utils.py index cc5a1f464..eaeac144c 100644 --- a/coriolis/providers/provider_utils.py +++ b/coriolis/providers/provider_utils.py @@ -80,7 +80,8 @@ def get_storage_mapping_for_disk( disk_info) else: LOG.debug( - "No 'storage_backend_identifier' set for disk '%s'", disk_info) + "No 'storage_backend_identifier' set for disk '%s'", + disk_info) # 3) use provided default: if not mapped_backend: @@ -111,24 +112,24 @@ def get_storage_mapping_for_disk( def check_changed_storage_mappings(volumes_info, old_storage_mappings, new_storage_mappings): - if not volumes_info: - return - - old_backend_mappings = old_storage_mappings.get('backend_mappings', []) - old_disk_mappings = old_storage_mappings.get('disk_mappings', []) - new_backend_mappings = new_storage_mappings.get('backend_mappings', []) - new_disk_mappings = new_storage_mappings.get('disk_mappings', []) - - old_backend_mappings_set = [ - tuple(mapping.values()) for mapping in old_backend_mappings] - old_disk_mappings_set = [ - tuple(mapping.values()) for mapping in old_disk_mappings] - new_backend_mappings_set = [ - tuple(mapping.values()) for mapping in new_backend_mappings] - new_disk_mappings_set = [ - tuple(mapping.values()) for mapping in new_disk_mappings] - - if (old_backend_mappings_set != new_backend_mappings_set or - old_disk_mappings_set != new_disk_mappings_set): - raise exception.CoriolisException("Modifying storage mappings is " - "not supported.") + if not volumes_info: + return + + old_backend_mappings = old_storage_mappings.get('backend_mappings', []) + old_disk_mappings = old_storage_mappings.get('disk_mappings', []) + new_backend_mappings = new_storage_mappings.get('backend_mappings', []) + new_disk_mappings = new_storage_mappings.get('disk_mappings', []) + + old_backend_mappings_set = [ + tuple(mapping.values()) for mapping in old_backend_mappings] + old_disk_mappings_set = [ + tuple(mapping.values()) for mapping in old_disk_mappings] + new_backend_mappings_set = [ + tuple(mapping.values()) for mapping in new_backend_mappings] + new_disk_mappings_set = [ + tuple(mapping.values()) for mapping in new_disk_mappings] + + if (old_backend_mappings_set != new_backend_mappings_set or + old_disk_mappings_set != new_disk_mappings_set): + raise exception.CoriolisException("Modifying storage mappings is " + "not supported.") diff --git a/coriolis/providers/replicator.py b/coriolis/providers/replicator.py index 2347073de..a28ddcb08 100644 --- a/coriolis/providers/replicator.py +++ b/coriolis/providers/replicator.py @@ -375,10 +375,12 @@ def attach_new_disk( new_device_paths = None for i in range(retry_count): new_disks_status = self._cli.get_status() - new_device_paths = [dev['device-path'] for dev in new_disks_status] + new_device_paths = [dev['device-path'] + for dev in new_disks_status] LOG.debug( "Polled devices while waiting for disk '%s' to attach " - "(try %d/%d): %s", disk_id, i+1, retry_count, new_device_paths) + "(try %d/%d): %s", disk_id, i + 1, retry_count, + new_device_paths) # check for missing/multiple new device paths: missing_device_paths = ( @@ -390,7 +392,8 @@ def attach_new_disk( dev for dev in previous_disks_status if dev['device-path'] in missing_device_paths]) - new_device_paths = set(new_device_paths) - set(previous_device_paths) + new_device_paths = set( + new_device_paths) - set(previous_device_paths) if new_device_paths: break else: diff --git a/coriolis/qemu.py b/coriolis/qemu.py index 578be5c64..01fe56471 100644 --- a/coriolis/qemu.py +++ b/coriolis/qemu.py @@ -54,7 +54,8 @@ class Error(ctypes.Structure): _libqemu.qemu_init_exec_dir.restype = None qemu_init_exec_dir = _libqemu.qemu_init_exec_dir -_libqemu.qemu_init_main_loop.argtypes = [ctypes.POINTER(ctypes.POINTER(Error))] +_libqemu.qemu_init_main_loop.argtypes = [ + ctypes.POINTER(ctypes.POINTER(Error))] _libqemu.qemu_init_main_loop.res_type = ctypes.c_int qemu_init_main_loop = _libqemu.qemu_init_main_loop diff --git a/coriolis/rpc.py b/coriolis/rpc.py index 4a1dfe5a1..33bb3827c 100644 --- a/coriolis/rpc.py +++ b/coriolis/rpc.py @@ -102,9 +102,9 @@ def _transport(self): def _rpc_client(self): return messaging.RPCClient( - self._transport, self._target, - serializer=self._serializer, - timeout=self._timeout) + self._transport, self._target, + serializer=self._serializer, + timeout=self._timeout) def _call(self, ctxt, method, **kwargs): client = self._rpc_client() diff --git a/coriolis/scheduler/rpc/client.py b/coriolis/scheduler/rpc/client.py index 2bc2f9d60..61c009064 100644 --- a/coriolis/scheduler/rpc/client.py +++ b/coriolis/scheduler/rpc/client.py @@ -167,7 +167,7 @@ def get_worker_service_for_task( LOG.warn( "Failed to schedule task with ID '%s' (attempt %d/%d). " "Waiting %d seconds and then retrying. Error was: %s", - task['id'], i+1, retry_count, retry_period, + task['id'], i + 1, retry_count, retry_period, utils.get_exception_details()) time.sleep(retry_period) diff --git a/coriolis/scheduler/rpc/server.py b/coriolis/scheduler/rpc/server.py index d162ee91c..1456ff8c4 100644 --- a/coriolis/scheduler/rpc/server.py +++ b/coriolis/scheduler/rpc/server.py @@ -57,7 +57,6 @@ def _get_weighted_filtered_services( scores = [] - service_ids = [service.id for service in services] LOG.debug( "Running following filters on worker services '%s': %s", @@ -156,7 +155,8 @@ def get_workers_for_specs( "None of the selected Regions (%s) are enabled or " "otherwise usable." % region_set) filters.append( - trivial_filters.RegionsFilter(region_set, any_region=True)) + trivial_filters.RegionsFilter( + region_set, any_region=True)) if provider_requirements: filters.append( trivial_filters.ProviderTypesFilter(provider_requirements)) diff --git a/coriolis/scheduler/scheduler_utils.py b/coriolis/scheduler/scheduler_utils.py index 820b00ea1..c8baa79dd 100644 --- a/coriolis/scheduler/scheduler_utils.py +++ b/coriolis/scheduler/scheduler_utils.py @@ -58,6 +58,7 @@ def get_any_worker_service( return service return db_api.get_service(ctxt, service['id']) + def get_worker_rpc_for_host(host, *client_args, **client_kwargs): rpc_client_class = RPC_TOPIC_TO_CLIENT_CLASS_MAP[ constants.WORKER_MAIN_MESSAGING_TOPIC] diff --git a/coriolis/service.py b/coriolis/service.py index eb1ae6ec0..1042b241e 100644 --- a/coriolis/service.py +++ b/coriolis/service.py @@ -42,6 +42,7 @@ def get_worker_count_from_args(argv): --worker-process-count is not present), as well as the unprocessed args. """ parser = argparse.ArgumentParser() + def _check_positive_worker_count(worker_count): count = int(worker_count) if count <= 0: @@ -50,9 +51,10 @@ def _check_positive_worker_count(worker_count): "got: %s" % worker_count) return count parser.add_argument( - '--worker-process-count', metavar='N', type=_check_positive_worker_count, + '--worker-process-count', metavar='N', + type=_check_positive_worker_count, help="Number of worker processes for this service. Defaults to the " - "number of logical CPU cores on the system.") + "number of logical CPU cores on the system.") args, unknown_args = parser.parse_known_args(args=argv) return args.worker_process_count, unknown_args @@ -76,7 +78,8 @@ def check_locks_dir_empty(): if not os.path.exists(locks_dir): LOG.warn( - "Configured 'lock_path' directory '%s' does NOT exist!", locks_dir) + "Configured 'lock_path' directory '%s' does NOT exist!", + locks_dir) return if not os.path.isdir(locks_dir): diff --git a/coriolis/taskflow/base.py b/coriolis/taskflow/base.py index 5ba3eed7d..26a429bd7 100644 --- a/coriolis/taskflow/base.py +++ b/coriolis/taskflow/base.py @@ -15,15 +15,14 @@ TASK_RETURN_VALUE_FORMAT = "%s-result" % ( - constants.TASK_LOCK_NAME_FORMAT) + constants.TASK_LOCK_NAME_FORMAT) LOG = logging.getLogger() taskflow_opts = [ - cfg.IntOpt("worker_task_execution_timeout", - default=3600, - help="Number of seconds until Coriolis tasks which are executed" - "remotely on a Worker Service through taskflow timeout.") -] + cfg.IntOpt( + "worker_task_execution_timeout", default=3600, + help="Number of seconds until Coriolis tasks which are executed" + "remotely on a Worker Service through taskflow timeout.")] CONF = cfg.CONF CONF.register_opts(taskflow_opts, 'taskflow') @@ -154,8 +153,8 @@ def _set_requires_for_task_info_fields(self, kwargs): cleanup_task_deps = list( set( cleanup_task_runner.get_required_task_info_properties( - )).difference( - main_task_runner.get_returned_task_info_properties())) + )).difference( + main_task_runner.get_returned_task_info_properties())) new_requires.extend(cleanup_task_deps) kwargs['requires'] = new_requires @@ -173,8 +172,8 @@ def _set_provides_for_task_info_fields(self, kwargs): cleanup_task_res = list( set( cleanup_task_runner.get_returned_task_info_properties( - )).difference( - main_task_runner.get_returned_task_info_properties())) + )).difference( + main_task_runner.get_returned_task_info_properties())) new_provides.extend(cleanup_task_res) kwargs['provides'] = new_provides @@ -193,7 +192,7 @@ def _get_worker_service_rpc_for_task( LOG.debug( "[Task '%s'] Was offered the following worker service for executing " "Taskflow worker task '%s': %s", - self._task_name, task_id, worker_service['id']) + self._task_name, task_id, worker_service['id']) return rpc_worker_client.WorkerClient.from_service_definition( worker_service, timeout=rpc_timeout) @@ -241,8 +240,8 @@ def revert(self, context, origin, destination, task_info, **kwargs): try: res = self._execute_task( - context, self._task_id, self._cleanup_task_runner_type, origin, - destination, task_info) + context, self._task_id, self._cleanup_task_runner_type, + origin, destination, task_info) except Exception as ex: LOG.warn( "Task cleanup for '%s' (main task type '%s', cleanup task type" @@ -255,6 +254,6 @@ def revert(self, context, origin, destination, task_info, **kwargs): LOG.debug( "Reversion of taskflow task '%s' (ID '%s') was successfully " - "executed using task runner '%s' with the following result: %s" % ( - self._task_name, self._task_id, self._cleanup_task_runner_type, - res)) + "executed using task runner '%s' with the following result: %s" % + (self._task_name, self._task_id, self._cleanup_task_runner_type, + res)) diff --git a/coriolis/taskflow/runner.py b/coriolis/taskflow/runner.py index 92822aba9..e369d940e 100644 --- a/coriolis/taskflow/runner.py +++ b/coriolis/taskflow/runner.py @@ -6,7 +6,7 @@ import sys from logging import handlers -import eventlet #noqa +import eventlet # noqa from oslo_config import cfg from oslo_log import log as logging from six.moves import queue @@ -79,7 +79,8 @@ def _run_flow(self, flow, store=None): except Exception as ex: LOG.warn( "Fatal error occurred while attempting to run flow '%s'. " - "Full trace was: %s", flow.name, utils.get_exception_details()) + "Full trace was: %s", flow.name, + utils.get_exception_details()) raise LOG.info( "Successfully ran flow with name '%s'. Statistics were: %s", diff --git a/coriolis/tasks/minion_pool_tasks.py b/coriolis/tasks/minion_pool_tasks.py index 24e4db80e..d09d3cdc8 100644 --- a/coriolis/tasks/minion_pool_tasks.py +++ b/coriolis/tasks/minion_pool_tasks.py @@ -530,7 +530,8 @@ def _get_minion_task_info_field_mappings(cls): return TARGET_MINION_TASK_INFO_FIELD_MAPPINGS -class DetachVolumesFromDestinationMinionTask(AttachVolumesToDestinationMinionTask): +class DetachVolumesFromDestinationMinionTask( + AttachVolumesToDestinationMinionTask): @classmethod def _get_provider_disk_operation(cls, provider): @@ -764,7 +765,7 @@ def _get_provider_pool_validation_operation(cls, provider): @classmethod def _get_minion_task_info_field_mappings(cls): - return OSMOPRHING_MINION_TASK_INFO_FIELD_MAPPINGS + return OSMOPRHING_MINION_TASK_INFO_FIELD_MAPPINGS class _BaseReleaseMinionTask(base.TaskRunner): diff --git a/coriolis/tests/conductor/rpc/test_server.py b/coriolis/tests/conductor/rpc/test_server.py index ca66cfe67..702bdc7bd 100644 --- a/coriolis/tests/conductor/rpc/test_server.py +++ b/coriolis/tests/conductor/rpc/test_server.py @@ -368,7 +368,7 @@ def test_get_endpoint_destination_options( mock_get_endpoint.return_value.connection_info, mock.sentinel.environment, mock.sentinel.option_names, - ) + ) self.assertEqual( options, @@ -521,7 +521,7 @@ def test_validate_endpoint_target_environment( mock.sentinel.context, mock_get_endpoint.return_value.type, mock.sentinel.target_env, - ) + ) @mock.patch.object( server.ConductorServerEndpoint, "_get_worker_service_rpc_for_specs" @@ -556,7 +556,7 @@ def test_validate_endpoint_source_environment( mock.sentinel.context, mock_get_endpoint.return_value.type, mock.sentinel.source_env, - ) + ) @mock.patch.object( rpc_worker_client.WorkerClient, "from_service_definition" @@ -567,12 +567,12 @@ def test_get_available_providers( ): providers = self.server.get_available_providers(mock.sentinel.context) mock_service_definition.assert_called_once_with( - mock_scheduler_client.get_any_worker_service(mock.sentinel.context) - ) + mock_scheduler_client.get_any_worker_service( + mock.sentinel.context)) mock_service_definition.return_value\ .get_available_providers.assert_called_once_with( mock.sentinel.context - ) + ) self.assertEqual( providers, mock_service_definition @@ -592,14 +592,14 @@ def test_get_provider_schemas( mock.sentinel.provider_type, ) mock_service_definition.assert_called_once_with( - mock_scheduler_client.get_any_worker_service(mock.sentinel.context) - ) + mock_scheduler_client.get_any_worker_service( + mock.sentinel.context)) mock_service_definition.return_value\ .get_provider_schemas.assert_called_once_with( mock.sentinel.context, mock.sentinel.platform_name, mock.sentinel.provider_type, - ) + ) self.assertEqual( provider_schemas, mock_service_definition.return_value @@ -694,7 +694,7 @@ def test_get_worker_service_rpc_for_task( retry_count=5, retry_period=2, random_choice=True, - ) + ) mock_service_definition.assert_called_once_with( mock_scheduler_client.get_worker_service_for_task.return_value ) @@ -1030,7 +1030,8 @@ def create_task_side_effect( self.assertEqual( mock_tasks_execution.return_value.type, constants.EXECUTION_TYPE_REPLICA_EXECUTION) - self.assertEqual(result, mock_get_replica_tasks_execution.return_value) + self.assertEqual( + result, mock_get_replica_tasks_execution.return_value) @mock.patch.object( server.ConductorServerEndpoint, @@ -1215,7 +1216,8 @@ def create_task_side_effect( mock_tasks_execution.return_value.id ) - self.assertEqual(result, mock_get_replica_tasks_execution.return_value) + self.assertEqual( + result, mock_get_replica_tasks_execution.return_value) # raises exception if instances have no volumes info instances[0].get.return_value = None @@ -2466,8 +2468,8 @@ def call_advance_execution_state( task_info = { mock.sentinel.instance: { 'test': 'info', - }, - } + }, + } mock_get_action.return_value = mock.Mock( info=task_info ) @@ -3098,9 +3100,7 @@ def test_set_task_error_os_morphing( mock.sentinel.exception_details, ) mock_cancel_execution_for_osmorphing_debugging.assert_called_once_with( - mock.sentinel.context, - mock_get_tasks_execution.return_value, - ) + mock.sentinel.context, mock_get_tasks_execution.return_value, ) self.assertEqual(2, mock_set_task_status.call_count) mock_set_tasks_execution_status.assert_called_once_with( mock.sentinel.context, diff --git a/coriolis/tests/test_schemas.py b/coriolis/tests/test_schemas.py index 9d19d6578..637d35b7c 100644 --- a/coriolis/tests/test_schemas.py +++ b/coriolis/tests/test_schemas.py @@ -73,7 +73,8 @@ def test_validate_value(self, mock_validate): schemas.validate_value(test_value, test_schema) - mock_validate.assert_called_once_with(test_value, test_schema, format_checker=None) + mock_validate.assert_called_once_with( + test_value, test_schema, format_checker=None) @mock.patch.object(json, 'loads') @mock.patch.object(jsonschema, 'validate') @@ -87,4 +88,5 @@ def test_validate_string(self, mock_validate, mock_loads): schemas.validate_string(test_string, test_schema) mock_loads.assert_called_once_with(test_string) - mock_validate.assert_called_once_with(test_value, test_schema, format_checker=None) + mock_validate.assert_called_once_with( + test_value, test_schema, format_checker=None) diff --git a/coriolis/tests/worker/rpc/test_server.py b/coriolis/tests/worker/rpc/test_server.py index fbf2520aa..bfcc5fb3c 100644 --- a/coriolis/tests/worker/rpc/test_server.py +++ b/coriolis/tests/worker/rpc/test_server.py @@ -20,14 +20,14 @@ class WorkerServerEndpointTestCase(test_base.CoriolisBaseTestCase): """Test suite for the Coriolis Worker RPC server.""" - @mock.patch.object(server.WorkerServerEndpoint, "_register_worker_service") + @mock.patch.object(server.WorkerServerEndpoint, + "_register_worker_service") def setUp(self, _): # pylint: disable=arguments-differ super(WorkerServerEndpointTestCase, self).setUp() self.server = server.WorkerServerEndpoint() - @mock.patch.object( - server.WorkerServerEndpoint, "_start_process_with_custom_library_paths" - ) + @mock.patch.object(server.WorkerServerEndpoint, + "_start_process_with_custom_library_paths") @mock.patch.object(server, "_task_process") @mock.patch.object(eventlet, "spawn") @mock.patch.object(server.WorkerServerEndpoint, "_rpc_conductor_client") @@ -433,7 +433,7 @@ def call_get_endpoint_destination_options(): mock_get_secret.return_value, env=mock.sentinel.environment, option_names=mock.sentinel.option_names, - ) + ) mock_validate.assert_called_once_with( mock_get_provider.return_value .get_target_environment_options.return_value, @@ -484,7 +484,7 @@ def call_get_endpoint_source_minion_pool_options(): mock_get_secret.return_value, env=mock.sentinel.environment, option_names=mock.sentinel.option_names, - ) + ) mock_validate.assert_called_once_with( mock_get_provider.return_value .get_minion_pool_options.return_value, @@ -536,7 +536,7 @@ def call_get_endpoint_destination_minion_pool_options(): mock_get_secret.return_value, env=mock.sentinel.environment, option_names=mock.sentinel.option_names, - ) + ) mock_validate.assert_called_once_with( mock_get_provider.return_value .get_minion_pool_options.return_value, @@ -588,7 +588,7 @@ def call_get_endpoint_source_options(): mock_get_secret.return_value, env=mock.sentinel.environment, option_names=mock.sentinel.option_names, - ) + ) mock_validate.assert_called_once_with( mock_get_provider.return_value .get_source_environment_options.return_value, @@ -735,7 +735,8 @@ def test_validate_endpoint_source_environment( self.assertEqual(result, (True, None)) # handle SchemaValidationException - mock_validate.side_effect = exception.SchemaValidationException("test") + mock_validate.side_effect = exception.SchemaValidationException( + "test") result = self.server.validate_endpoint_source_environment( mock.sentinel.context, mock.sentinel.source_platform_name, @@ -769,7 +770,8 @@ def test_validate_endpoint_target_environment( self.assertEqual(result, (True, None)) # handle SchemaValidationException - mock_validate.side_effect = exception.SchemaValidationException("test") + mock_validate.side_effect = exception.SchemaValidationException( + "test") result = self.server.validate_endpoint_target_environment( mock.sentinel.context, mock.sentinel.target_platform_name, @@ -803,7 +805,8 @@ def test_validate_endpoint_source_minion_pool_options( self.assertEqual(result, (True, None)) # handle SchemaValidationException - mock_validate.side_effect = exception.SchemaValidationException("test") + mock_validate.side_effect = exception.SchemaValidationException( + "test") result = self.server.validate_endpoint_source_minion_pool_options( mock.sentinel.context, mock.sentinel.source_platform_name, @@ -837,7 +840,8 @@ def test_validate_endpoint_destination_minion_pool_options( self.assertEqual(result, (True, None)) # handle SchemaValidationException - mock_validate.side_effect = exception.SchemaValidationException("test") + mock_validate.side_effect = exception.SchemaValidationException( + "test") result = self.server.validate_endpoint_destination_minion_pool_options( mock.sentinel.context, mock.sentinel.destination_platform_name, @@ -875,7 +879,7 @@ def call_validate_endpoint_connection(): mock_get_provider.return_value\ .validate_connection.assert_called_once_with( mock.sentinel.context, mock_get_secret.return_value - ) + ) self.assertEqual(result, (True, None)) diff --git a/coriolis/utils.py b/coriolis/utils.py index a55191865..403117de2 100644 --- a/coriolis/utils.py +++ b/coriolis/utils.py @@ -625,7 +625,8 @@ def _bad_request_on_error(func): def wrapper(*args, **kwargs): (is_valid, message) = func(*args, **kwargs) if not is_valid: - raise exc.HTTPBadRequest(explanation=(error_message % message)) + raise exc.HTTPBadRequest( + explanation=(error_message % message)) return (is_valid, message) return wrapper return _bad_request_on_error diff --git a/coriolis/worker/rpc/client.py b/coriolis/worker/rpc/client.py index db2f8d8c0..7d46dd9e4 100644 --- a/coriolis/worker/rpc/client.py +++ b/coriolis/worker/rpc/client.py @@ -11,9 +11,9 @@ worker_opts = [ - cfg.IntOpt("worker_rpc_timeout", - help="Number of seconds until RPC calls to the worker timeout.") -] + cfg.IntOpt( + "worker_rpc_timeout", + help="Number of seconds until RPC calls to the worker timeout.")] CONF = cfg.CONF CONF.register_opts(worker_opts, 'worker') @@ -48,7 +48,8 @@ def from_service_definition( if topic_override: topic = topic_override return cls( - timeout=timeout, base_worker_topic=topic, host=service.get('host')) + timeout=timeout, base_worker_topic=topic, + host=service.get('host')) def begin_task(self, ctxt, task_id, task_type, origin, destination, instance, task_info): @@ -108,7 +109,8 @@ def get_endpoint_source_options( env=env, option_names=option_names) - def get_endpoint_networks(self, ctxt, platform_name, connection_info, env): + def get_endpoint_networks( + self, ctxt, platform_name, connection_info, env): return self._call( ctxt, 'get_endpoint_networks', platform_name=platform_name, diff --git a/coriolis/worker/rpc/server.py b/coriolis/worker/rpc/server.py index 626b1337b..0c0f34ac9 100644 --- a/coriolis/worker/rpc/server.py +++ b/coriolis/worker/rpc/server.py @@ -206,8 +206,9 @@ def _wait_for_process(self, p, mp_q): break return result - def _exec_task_process(self, ctxt, task_id, task_type, origin, destination, - instance, task_info, report_to_conductor=True): + def _exec_task_process( + self, ctxt, task_id, task_type, origin, destination, instance, + task_info, report_to_conductor=True): mp_ctx = multiprocessing.get_context('spawn') mp_q = mp_ctx.Queue() mp_log_q = mp_ctx.Queue() @@ -452,7 +453,8 @@ def get_endpoint_source_options( return options - def get_endpoint_networks(self, ctxt, platform_name, connection_info, env): + def get_endpoint_networks( + self, ctxt, platform_name, connection_info, env): env = env or {} provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_ENDPOINT_NETWORKS, None) @@ -575,8 +577,8 @@ def validate_endpoint_connection(self, ctxt, platform_name, "Schema validation for the provided connection parameters has " "failed. Please ensure that you have included all the " "necessary connection parameters and they are all properly " - "formatted for the '%s' Coriolis plugin in use." % ( - platform_name)) + "formatted for the '%s' Coriolis plugin in use." % + (platform_name)) except exception.ConnectionValidationException as ex: LOG.warn(utils.get_exception_details()) is_valid = False diff --git a/coriolis/wsman.py b/coriolis/wsman.py index a5dfdaaee..544e617dc 100644 --- a/coriolis/wsman.py +++ b/coriolis/wsman.py @@ -54,8 +54,9 @@ def from_connection_info(cls, connection_info, timeout=DEFAULT_TIMEOUT): """ Returns a wsman.WSManConnection object for the provided conn info. """ if not isinstance(connection_info, dict): raise ValueError( - "WSMan connection must be a dict. Got type '%s', value: %s" % ( - type(connection_info), connection_info)) + "WSMan connection must be a dict. Got type '%s', value: %s" % + (type(connection_info), + connection_info)) required_keys = ["ip", "username", "password"] missing = [key for key in required_keys if key not in connection_info]