From 35b7ee7af31fedb097d10dbd15d4c5283b4a2c49 Mon Sep 17 00:00:00 2001 From: Cristian Matiut Date: Mon, 12 Feb 2024 12:14:35 +0200 Subject: [PATCH] Add tests for `coriolis.conductor.rpc.server` module --- coriolis/conductor/rpc/server.py | 2 +- .../rpc/data/cancel_migration_config.yml | 34 + coriolis/tests/conductor/rpc/test_server.py | 2103 +++++++++++++++-- 3 files changed, 1941 insertions(+), 198 deletions(-) create mode 100644 coriolis/tests/conductor/rpc/data/cancel_migration_config.yml diff --git a/coriolis/conductor/rpc/server.py b/coriolis/conductor/rpc/server.py index 525905a5c..2d22ccbc5 100644 --- a/coriolis/conductor/rpc/server.py +++ b/coriolis/conductor/rpc/server.py @@ -1704,7 +1704,7 @@ def report_migration_minions_allocation_error( awaiting_minions_status = ( constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS) if migration.last_execution_status != awaiting_minions_status: - raise exception.InvalidReplicaState( + raise exception.InvalidMigrationState( "Migration is in '%s' status instead of the expected '%s' to " "have minion machines allocations fail for it." % ( migration.last_execution_status, awaiting_minions_status)) diff --git a/coriolis/tests/conductor/rpc/data/cancel_migration_config.yml b/coriolis/tests/conductor/rpc/data/cancel_migration_config.yml new file mode 100644 index 000000000..4ea053212 --- /dev/null +++ b/coriolis/tests/conductor/rpc/data/cancel_migration_config.yml @@ -0,0 +1,34 @@ +- config: + force: False + execution_statuses: [EXECUTION_STATUS_RUNNING] + raises_exception: False + +- config: + force: False + execution_statuses: [] + raises_exception: True + +- config: + force: False + execution_statuses: [EXECUTION_STATUS_COMPLETED] + raises_exception: True + +- config: + force: True + execution_statuses: [EXECUTION_STATUS_CANCELLING] + raises_exception: False + +- config: + force: False + execution_statuses: [EXECUTION_STATUS_CANCELLING] + raises_exception: True + +- config: + force: False + execution_statuses: [EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS] + raises_exception: False + +- config: + force: True + execution_statuses: [EXECUTION_STATUS_RUNNING, EXECUTION_STATUS_CANCELLING, EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS] + raises_exception: True diff --git a/coriolis/tests/conductor/rpc/test_server.py b/coriolis/tests/conductor/rpc/test_server.py index 7c093f956..4c9c7ca03 100644 --- a/coriolis/tests/conductor/rpc/test_server.py +++ b/coriolis/tests/conductor/rpc/test_server.py @@ -3,6 +3,7 @@ import copy import ddt +import logging import uuid from unittest import mock @@ -12,12 +13,14 @@ from coriolis.db import api as db_api from coriolis.db.sqlalchemy import models from coriolis import exception +from coriolis import keystone from coriolis.licensing import client as licensing_client from coriolis import schemas from coriolis.tests import test_base from coriolis.tests import testutils from coriolis import utils from coriolis.worker.rpc import client as rpc_worker_client + from oslo_concurrency import lockutils from oslo_config import cfg @@ -38,7 +41,11 @@ def setUp(self): rpc_worker_client.WorkerClient, "from_service_definition" ) @mock.patch.object(server.ConductorServerEndpoint, "_scheduler_client") - def test_get_all_diagnostics(self, mock_scheduler_client, _): + def test_get_all_diagnostics( + self, + mock_scheduler_client, + mock_from_service_definition + ): mock_scheduler_client.get_workers_for_specs.side_effect = ( CoriolisTestException()) self.assertRaises( @@ -51,6 +58,22 @@ def test_get_all_diagnostics(self, mock_scheduler_client, _): assert ( mock_scheduler_client.get_diagnostics.return_value in diagnostics ) + mock_scheduler_client.get_workers_for_specs.return_value = { + mock.sentinel.diagnostics + } + diagnostics = self.server.get_all_diagnostics(mock.sentinel.context) + assert ( + mock_scheduler_client.get_diagnostics.return_value in diagnostics + ) + mock_scheduler_client.get_workers_for_specs.return_value = [{ + 'host': mock.sentinel.host + }] + mock_from_service_definition.side_effect = ( + mock.sentinel.rpc_worker, CoriolisTestException()) + self.assertRaises( + CoriolisTestException, + lambda: self.server.get_all_diagnostics(mock.sentinel.context), + ) @mock.patch.object( rpc_worker_client.WorkerClient, "from_service_definition" @@ -86,6 +109,199 @@ def test_get_worker_service_rpc_for_specs( self.assertEqual(result, mock_from_service_definition.return_value) + @mock.patch.object(licensing_client.LicensingClient, 'from_env') + def test_check_delete_reservation_for_transfer( + self, + mock_from_env + ): + transfer_action = mock.Mock() + self.server = server.ConductorServerEndpoint() + self.server._check_delete_reservation_for_transfer(transfer_action) + mock_from_env.return_value.delete_reservation.assert_called_once_with( + transfer_action.reservation_id + ) + + @mock.patch.object(licensing_client.LicensingClient, 'from_env') + def test_check_delete_reservation_for_transfer_no_licensing_client( + self, + mock_from_env + ): + transfer_action = mock.Mock() + mock_from_env.return_value = None + self.server = server.ConductorServerEndpoint() + with self.assertLogs( + 'coriolis.conductor.rpc.server', level=logging.WARNING): + self.server._check_delete_reservation_for_transfer(transfer_action) + + @mock.patch.object(licensing_client.LicensingClient, 'from_env') + def test_check_delete_reservation_for_transfer_delete_fails( + self, + mock_from_env + ): + transfer_action = mock.Mock() + mock_from_env.return_value.delete_reservation.side_effect = \ + CoriolisTestException() + self.server = server.ConductorServerEndpoint() + with self.assertLogs( + 'coriolis.conductor.rpc.server', level=logging.WARNING): + self.server._check_delete_reservation_for_transfer(transfer_action) + mock_from_env.return_value.delete_reservation.assert_called_once_with( + transfer_action.reservation_id + ) + + @mock.patch.object(licensing_client.LicensingClient, 'from_env') + def test_check_create_reservation_for_transfer( + self, + mock_from_env + ): + transfer_action = mock.Mock() + transfer_action.instances = ['instance_1', 'instance_2'] + transfer_type = mock.sentinel.transfer_type + self.server = server.ConductorServerEndpoint() + self.server._check_create_reservation_for_transfer( + transfer_action, transfer_type) + mock_from_env.return_value.add_reservation.assert_called_once_with( + mock.sentinel.transfer_type, + 2 + ) + + @mock.patch.object(licensing_client.LicensingClient, 'from_env') + def test_check_create_reservation_for_transfer_no_licensing_client( + self, + mock_from_env + ): + transfer_action = mock.Mock() + transfer_type = mock.sentinel.transfer_type + mock_from_env.return_value = None + self.server = server.ConductorServerEndpoint() + with self.assertLogs( + 'coriolis.conductor.rpc.server', level=logging.WARNING): + self.server._check_create_reservation_for_transfer( + transfer_action, transfer_type) + + @mock.patch.object(licensing_client.LicensingClient, 'from_env') + def test_check_reservation_for_transfer( + self, + mock_from_env + ): + transfer_action = mock.Mock() + transfer_action.reservation_id = mock.sentinel.reservation_id + reservation_type = mock.sentinel.reservation_type + self.server = server.ConductorServerEndpoint() + self.server._check_reservation_for_transfer( + transfer_action, reservation_type) + (mock_from_env.return_value.check_refresh_reservation. + assert_called_once_with)( + mock.sentinel.reservation_id) + + @mock.patch.object(licensing_client.LicensingClient, 'from_env') + def test_check_reservation_for_transfer_no_licensing_client( + self, + mock_from_env + ): + transfer_action = mock.Mock() + reservation_type = mock.sentinel.reservation_type + self.server = server.ConductorServerEndpoint() + mock_from_env.return_value = None + self.server = server.ConductorServerEndpoint() + with self.assertLogs( + 'coriolis.conductor.rpc.server', level=logging.WARNING): + self.server._check_reservation_for_transfer( + transfer_action, reservation_type) + + @mock.patch.object(server.ConductorServerEndpoint, + '_check_create_reservation_for_transfer') + @mock.patch.object(licensing_client.LicensingClient, 'from_env') + def test_check_reservation_for_transfer_no_reservation_id( + self, + mock_from_env, + mock_check_create_reservation_for_transfer + ): + transfer_action = mock.Mock() + transfer_action.reservation_id = None + reservation_type = mock.sentinel.reservation_type + self.server = server.ConductorServerEndpoint() + self.server._check_reservation_for_transfer( + transfer_action, reservation_type) + (mock_from_env.return_value.check_refresh_reservation + .assert_not_called)() + mock_check_create_reservation_for_transfer.assert_called_once_with( + transfer_action, reservation_type + ) + + @mock.patch.object(server.ConductorServerEndpoint, + '_check_create_reservation_for_transfer') + @mock.patch.object(licensing_client.LicensingClient, 'from_env') + def test_check_reservation_for_transfer_exc_code_404( + self, + mock_from_env, + mock_check_create_reservation_for_transfer + ): + transfer_action = mock.Mock() + transfer_action.reservation_id = mock.sentinel.reservation_id + reservation_type = mock.sentinel.reservation_type + ex = CoriolisTestException() + ex.code = 404 + mock_from_env.return_value.check_refresh_reservation.side_effect = ex + self.server = server.ConductorServerEndpoint() + self.server._check_reservation_for_transfer( + transfer_action, reservation_type) + (mock_from_env.return_value.check_refresh_reservation + .assert_called_once_with)( + mock.sentinel.reservation_id) + mock_check_create_reservation_for_transfer.assert_called_once_with( + transfer_action, reservation_type + ) + + @mock.patch.object(server.ConductorServerEndpoint, + '_check_create_reservation_for_transfer') + @mock.patch.object(licensing_client.LicensingClient, 'from_env') + def test_check_reservation_for_transfer_exc_code_409( + self, + mock_from_env, + mock_check_create_reservation_for_transfer + ): + transfer_action = mock.Mock() + transfer_action.reservation_id = mock.sentinel.reservation_id + reservation_type = mock.sentinel.reservation_type + ex = CoriolisTestException() + ex.code = 409 + mock_from_env.return_value.check_refresh_reservation.side_effect = ex + self.server = server.ConductorServerEndpoint() + self.server._check_reservation_for_transfer( + transfer_action, reservation_type) + (mock_from_env.return_value.check_refresh_reservation + .assert_called_once_with)( + mock.sentinel.reservation_id) + mock_check_create_reservation_for_transfer.assert_called_once_with( + transfer_action, reservation_type + ) + + @mock.patch.object(server.ConductorServerEndpoint, + '_check_create_reservation_for_transfer') + @mock.patch.object(licensing_client.LicensingClient, 'from_env') + def test_check_reservation_for_transfer_exc_code_not_excepted( + self, + mock_from_env, + mock_check_create_reservation_for_transfer + ): + transfer_action = mock.Mock() + transfer_action.reservation_id = mock.sentinel.reservation_id + reservation_type = mock.sentinel.reservation_type + ex = CoriolisTestException() + mock_from_env.return_value.check_refresh_reservation.side_effect = ex + self.server = server.ConductorServerEndpoint() + self.assertRaises( + CoriolisTestException, + self.server._check_reservation_for_transfer, + transfer_action, + reservation_type + ) + (mock_from_env.return_value.check_refresh_reservation + .assert_called_once_with)( + mock.sentinel.reservation_id) + mock_check_create_reservation_for_transfer.assert_not_called() + @mock.patch.object(server.ConductorServerEndpoint, "get_endpoint") @mock.patch.object(db_api, "delete_endpoint") @mock.patch.object(db_api, "update_endpoint") @@ -208,6 +424,17 @@ def call_get_endpoint(): mock_get_endpoint.side_effect = exception.NotFound() self.assertRaises(exception.NotFound, call_get_endpoint) + @mock.patch.object(db_api, "get_endpoint") + def test_get_endpoint_not_found(self, mock_get_endpoint): + def call_get_endpoint(): + return testutils.get_wrapped_function(self.server.get_endpoint)( + self, mock.sentinel.context, + mock.sentinel.endpoint_id # type: ignore + ) + + mock_get_endpoint.return_value = None + self.assertRaises(exception.NotFound, call_get_endpoint) + @mock.patch.object(db_api, "delete_endpoint") @mock.patch.object(db_api, "get_endpoint_replicas_count") def test_delete_endpoint( @@ -676,6 +903,64 @@ def test_create_task( self.assertEqual(task.on_error, True) self.assertEqual(task.status, constants.TASK_STATUS_SCHEDULED) + @mock.patch.object(server.ConductorServerEndpoint, "get_endpoint") + def test_get_task_origin(self, mock_get_endpoint): + mock_endpoint = mock.Mock() + mock_endpoint.connection_info = mock.sentinel.connection_info + mock_endpoint.type = mock.sentinel.type + mock_get_endpoint.return_value = mock_endpoint + action = mock.Mock() + action.origin_endpoint_id = mock.sentinel.origin_endpoint_id + action.source_environment = mock.sentinel.source_environment + + expected_result = { + "connection_info": mock.sentinel.connection_info, + "type": mock.sentinel.type, + "source_environment": mock.sentinel.source_environment + } + + result = self.server._get_task_origin( + mock.sentinel.context, + action + ) + + self.assertEqual( + expected_result, + result + ) + + mock_get_endpoint.assert_called_once_with( + mock.sentinel.context, mock.sentinel.origin_endpoint_id) + + @mock.patch.object(server.ConductorServerEndpoint, "get_endpoint") + def test_get_task_destination(self, mock_get_endpoint): + mock_endpoint = mock.Mock() + mock_endpoint.connection_info = mock.sentinel.connection_info + mock_endpoint.type = mock.sentinel.type + mock_get_endpoint.return_value = mock_endpoint + action = mock.Mock() + action.destination_endpoint_id = mock.sentinel.destination_endpoint_id + action.destination_environment = mock.sentinel.destination_environment + + expected_result = { + "connection_info": mock.sentinel.connection_info, + "type": mock.sentinel.type, + "target_environment": mock.sentinel.destination_environment + } + + result = self.server._get_task_destination( + mock.sentinel.context, + action + ) + + self.assertEqual( + expected_result, + result + ) + + mock_get_endpoint.assert_called_once_with( + mock.sentinel.context, mock.sentinel.destination_endpoint_id) + @mock.patch.object( rpc_worker_client.WorkerClient, "from_service_definition" ) @@ -729,6 +1014,239 @@ def test_get_worker_service_rpc_for_task( exception_details="test", ) + @mock.patch.object(server.ConductorServerEndpoint, + "_set_tasks_execution_status") + @mock.patch.object(server.ConductorServerEndpoint, + "_get_worker_service_rpc_for_task") + @mock.patch.object(db_api, "set_task_status") + @mock.patch.object(db_api, "get_endpoint") + @mock.patch.object(server.ConductorServerEndpoint, "_get_task_destination") + @mock.patch.object(server.ConductorServerEndpoint, "_get_task_origin") + @mock.patch.object(keystone, 'create_trust') + def test_begin_tasks( + self, + mock_create_trust, + mock_get_task_origin, + mock_get_task_destination, + mock_get_endpoint, + mock_set_task_status, + mock_get_worker_service_rpc_for_task, + mock_set_tasks_execution_status, + ): + mock_context = mock.Mock() + task_info = mock.Mock() + mock_action = mock.Mock() + task_info.instance = {} + mock_action.info = task_info + mock_context.trust_id = None + mock_get_endpoint.side_effect = [ + mock.sentinel.origin_endpoint_id, + mock.sentinel.destination_endpoint_id] + + task = mock.Mock( + id=mock.sentinel.task_id, + index=mock.sentinel.task_index, + status=constants.TASK_STATUS_SCHEDULED, + depends_on=None, + on_error=None, + ) + execution = mock.Mock( + id=mock.sentinel.execution_id, + tasks=[task] + ) + + self.server._begin_tasks( + mock_context, + mock_action, + execution, + task_info, + mock.sentinel.scheduling_retry_count, + mock.sentinel.scheduling_retry_period, + ) + + mock_create_trust.assert_called_once_with(mock_context) + mock_get_task_origin.assert_called_once_with(mock_context, mock_action) + mock_get_task_destination.assert_called_once_with( + mock_context, mock_action) + mock_get_endpoint.assert_has_calls([ + mock.call(mock_context, mock_action.origin_endpoint_id), + mock.call(mock_context, mock_action.destination_endpoint_id) + ]) + mock_set_task_status.assert_called_once_with( + mock_context, mock.sentinel.task_id, constants.TASK_STATUS_PENDING) + mock_get_worker_service_rpc_for_task.assert_called_once_with( + mock_context, + task, + mock.sentinel.origin_endpoint_id, + mock.sentinel.destination_endpoint_id, + retry_count=mock.sentinel.scheduling_retry_count, + retry_period=mock.sentinel.scheduling_retry_period + ) + (mock_get_worker_service_rpc_for_task.return_value.begin_task. + assert_called_once_with)( + mock_context, + task_id=mock.sentinel.task_id, + task_type=task.task_type, + origin=mock_get_task_origin.return_value, + destination=mock_get_task_destination.return_value, + instance=task.instance, + task_info=task_info.get(task.instance) + ) + mock_set_tasks_execution_status.assert_called_once_with( + mock_context, execution, constants.TASK_STATUS_RUNNING + ) + + @mock.patch.object(server.ConductorServerEndpoint, + "_cancel_tasks_execution") + @mock.patch.object(server.ConductorServerEndpoint, + "_get_worker_service_rpc_for_task") + @mock.patch.object(db_api, "set_task_status") + @mock.patch.object(db_api, "get_endpoint") + @mock.patch.object(server.ConductorServerEndpoint, "_get_task_destination") + @mock.patch.object(server.ConductorServerEndpoint, "_get_task_origin") + @mock.patch.object(keystone, 'create_trust') + def test_begin_tasks_begin_task_raises( + self, + mock_create_trust, + mock_get_task_origin, + mock_get_task_destination, + mock_get_endpoint, + mock_set_task_status, + mock_get_worker_service_rpc_for_task, + mock_cancel_tasks_execution + ): + mock_context = mock.Mock() + task_info = mock.Mock() + mock_action = mock.Mock() + task_info.instance = {} + mock_action.info = task_info + mock_context.trust_id = None + mock_get_endpoint.side_effect = [ + mock.sentinel.origin_endpoint_id, + mock.sentinel.destination_endpoint_id] + + task = mock.Mock( + id=mock.sentinel.task_id, + index=mock.sentinel.task_index, + status=constants.TASK_STATUS_SCHEDULED, + depends_on=None, + on_error=None, + ) + execution = mock.Mock( + id=mock.sentinel.execution_id, + tasks=[task] + ) + (mock_get_worker_service_rpc_for_task.return_value.begin_task + .side_effect) = CoriolisTestException() + + self.assertRaises( + CoriolisTestException, + self.server._begin_tasks, + mock_context, + mock_action, + execution, + task_info, + mock.sentinel.scheduling_retry_count, + mock.sentinel.scheduling_retry_period, + ) + + mock_create_trust.assert_called_once_with(mock_context) + mock_get_task_origin.assert_called_once_with(mock_context, mock_action) + mock_get_task_destination.assert_called_once_with( + mock_context, mock_action) + mock_get_endpoint.assert_has_calls([ + mock.call(mock_context, mock_action.origin_endpoint_id), + mock.call(mock_context, mock_action.destination_endpoint_id) + ]) + mock_set_task_status.assert_called_once_with( + mock_context, mock.sentinel.task_id, constants.TASK_STATUS_PENDING) + mock_get_worker_service_rpc_for_task.assert_called_once_with( + mock_context, + task, + mock.sentinel.origin_endpoint_id, + mock.sentinel.destination_endpoint_id, + retry_count=mock.sentinel.scheduling_retry_count, + retry_period=mock.sentinel.scheduling_retry_period + ) + (mock_get_worker_service_rpc_for_task.return_value.begin_task. + assert_called_once_with)( + mock_context, + task_id=mock.sentinel.task_id, + task_type=task.task_type, + origin=mock_get_task_origin.return_value, + destination=mock_get_task_destination.return_value, + instance=task.instance, + task_info=task_info.get(task.instance) + ) + mock_cancel_tasks_execution.assert_called_once_with( + mock_context, execution, requery=True) + + @mock.patch.object(server.ConductorServerEndpoint, + "_cancel_tasks_execution") + @mock.patch.object(server.ConductorServerEndpoint, + "_get_worker_service_rpc_for_task") + @mock.patch.object(db_api, "set_task_status") + @mock.patch.object(db_api, "get_endpoint") + @mock.patch.object(server.ConductorServerEndpoint, "_get_task_destination") + @mock.patch.object(server.ConductorServerEndpoint, "_get_task_origin") + @mock.patch.object(keystone, 'create_trust') + def test_begin_tasks_no_newly_started_tasks( + self, + mock_create_trust, + mock_get_task_origin, + mock_get_task_destination, + mock_get_endpoint, + mock_set_task_status, + mock_get_worker_service_rpc_for_task, + mock_cancel_tasks_execution + ): + mock_context = mock.Mock() + task_info = mock.Mock() + mock_action = mock.Mock() + task_info.instance = {} + mock_action.info = task_info + mock_context.trust_id = None + mock_get_endpoint.side_effect = [ + mock.sentinel.origin_endpoint_id, + mock.sentinel.destination_endpoint_id] + + task = mock.Mock( + id=mock.sentinel.task_id, + index=mock.sentinel.task_index, + status=constants.TASK_STATUS_PENDING, + depends_on=None, + on_error=None, + ) + execution = mock.Mock( + id=mock.sentinel.execution_id, + tasks=[task] + ) + + self.assertRaises( + exception.InvalidActionTasksExecutionState, + self.server._begin_tasks, + mock_context, + mock_action, + execution, + task_info, + mock.sentinel.scheduling_retry_count, + mock.sentinel.scheduling_retry_period, + ) + + mock_create_trust.assert_called_once_with(mock_context) + mock_get_task_origin.assert_called_once_with(mock_context, mock_action) + mock_get_task_destination.assert_called_once_with( + mock_context, mock_action) + mock_get_endpoint.assert_has_calls([ + mock.call(mock_context, mock_action.origin_endpoint_id), + mock.call(mock_context, mock_action.destination_endpoint_id) + ]) + mock_set_task_status.assert_not_called() + mock_get_worker_service_rpc_for_task.assert_not_called() + (mock_get_worker_service_rpc_for_task.return_value.begin_task. + assert_not_called)() + mock_cancel_tasks_execution.assert_not_called() + @mock.patch.object(server.ConductorServerEndpoint, "_create_task") @mock.patch.object( server.ConductorServerEndpoint, "_check_replica_running_executions" @@ -904,7 +1422,7 @@ def call_execute_replica_tasks(): mock_replica = mock.Mock( instances=instances, network_map=mock.sentinel.network_map, - info={}, + info={mock.sentinel.instance1: {'volume_info': None}}, origin_minion_pool_id=mock.sentinel.origin_minion_pool_id if has_origin_minion_pool else None, destination_minion_pool_id=mock.sentinel.destination_minion_pool_id @@ -1077,98 +1595,268 @@ def call_delete_replica_tasks_execution(): exception.InvalidMigrationState, call_delete_replica_tasks_execution) - @mock.patch.object( - server.ConductorServerEndpoint, - 'get_replica_tasks_execution' - ) - @mock.patch.object( - server.ConductorServerEndpoint, - '_begin_tasks' - ) - @mock.patch.object(db_api, "add_replica_tasks_execution") - @mock.patch.object(db_api, "update_transfer_action_info_for_instance") - @mock.patch.object( - server.ConductorServerEndpoint, - '_check_execution_tasks_sanity' - ) - @mock.patch.object(copy, "deepcopy") - @mock.patch.object( - server.ConductorServerEndpoint, - '_create_task' - ) - @mock.patch.object(uuid, "uuid4") - @mock.patch.object(models, "TasksExecution") - @mock.patch.object( - server.ConductorServerEndpoint, - '_check_replica_running_executions' - ) - @mock.patch.object( - server.ConductorServerEndpoint, - '_get_replica' - ) - def test_delete_replica_disks( + @mock.patch.object(server.ConductorServerEndpoint, + '_get_replica_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, + '_cancel_tasks_execution') + def test_cancel_replica_tasks_execution( self, - mock_get_replica, - mock_check_replica_running_executions, - mock_tasks_execution, - mock_uuid4, - mock_create_task, - mock_deepcopy, - mock_check_execution_tasks_sanity, - mock_update_transfer_action_info_for_instance, - mock_add_replica_tasks_execution, - mock_begin_tasks, - mock_get_replica_tasks_execution, + mock_cancel_replica_tasks_execution, + mock_get_replica_tasks_execution ): - def call_delete_replica_disks(): - return testutils.get_wrapped_function( - self.server.delete_replica_disks)( - self.server, - mock.sentinel.context, - mock.sentinel.replica_id, # type: ignore - ) - instances = [mock.Mock(), mock.Mock()] - mock_replica = mock.Mock( - instances=instances, - id=mock.sentinel.replica_id, - network_map=mock.sentinel.network_map, - info={ - instance: instance - for instance in instances - } - ) - - def create_task_side_effect( - instance, - task_type, - execution, - depends_on=None, - ): - return mock.Mock( - id=task_type, - type=task_type, - instance=instance, - execution=execution, - depends_on=depends_on, - ) - - mock_create_task.side_effect = create_task_side_effect - - mock_get_replica.return_value = mock_replica - result = call_delete_replica_disks() - - mock_get_replica.assert_called_once_with( + mock_get_replica_tasks_execution.return_value.status = constants\ + .EXECUTION_STATUS_RUNNING + testutils.get_wrapped_function( + self.server.cancel_replica_tasks_execution)( + self.server, mock.sentinel.context, mock.sentinel.replica_id, - include_task_info=True + mock.sentinel.execution_id, + False ) - mock_check_replica_running_executions.assert_called_once_with( + mock_get_replica_tasks_execution.assert_called_once_with( mock.sentinel.context, - mock_replica - ) + mock.sentinel.replica_id, + mock.sentinel.execution_id) + mock_cancel_replica_tasks_execution.assert_called_once_with( + mock.sentinel.context, + mock_get_replica_tasks_execution.return_value, + force=False) - self.assertEqual( - mock_tasks_execution.return_value.status, + mock_get_replica_tasks_execution.reset_mock() + mock_cancel_replica_tasks_execution.reset_mock() + mock_get_replica_tasks_execution.return_value.status = constants\ + .EXECUTION_STATUS_CANCELLING + testutils.get_wrapped_function( + self.server.cancel_replica_tasks_execution)( + self.server, + mock.sentinel.context, + mock.sentinel.replica_id, + mock.sentinel.execution_id, + True + ) + mock_get_replica_tasks_execution.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.replica_id, + mock.sentinel.execution_id) + mock_cancel_replica_tasks_execution.assert_called_once_with( + mock.sentinel.context, + mock_get_replica_tasks_execution.return_value, + force=True) + + @mock.patch.object(server.ConductorServerEndpoint, + '_get_replica_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, + '_cancel_tasks_execution') + def test_cancel_replica_tasks_execution_status_not_active( + self, + mock_cancel_replica_tasks_execution, + mock_get_replica_tasks_execution + ): + self.assertRaises( + exception.InvalidReplicaState, + testutils.get_wrapped_function( + self.server.cancel_replica_tasks_execution), + self.server, + mock.sentinel.context, + mock.sentinel.replica_id, + mock.sentinel.execution_id, + False + ) + mock_get_replica_tasks_execution.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.replica_id, + mock.sentinel.execution_id) + mock_cancel_replica_tasks_execution.assert_not_called() + + @mock.patch.object(server.ConductorServerEndpoint, + '_get_replica_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, + '_cancel_tasks_execution') + def test_cancel_replica_tasks_execution_status_cancelling_no_force( + self, + mock_cancel_replica_tasks_execution, + mock_get_replica_tasks_execution + ): + mock_get_replica_tasks_execution.return_value.status = constants\ + .EXECUTION_STATUS_CANCELLING + self.assertRaises( + exception.InvalidReplicaState, + testutils.get_wrapped_function( + self.server.cancel_replica_tasks_execution), + self.server, + mock.sentinel.context, + mock.sentinel.replica_id, + mock.sentinel.execution_id, + False + ) + mock_get_replica_tasks_execution.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.replica_id, + mock.sentinel.execution_id) + mock_cancel_replica_tasks_execution.assert_not_called() + + @mock.patch.object(db_api, 'get_replica_tasks_execution') + def test_get_replica_tasks_execution( + self, + mock_get_replica_tasks_execution + ): + self.server._get_replica_tasks_execution( + mock.sentinel.context, + mock.sentinel.replica_id, + mock.sentinel.execution_id, + False, + False, + ) + mock_get_replica_tasks_execution.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.replica_id, + mock.sentinel.execution_id, + include_task_info=False, + to_dict=False) + + @mock.patch.object(db_api, 'get_replica_tasks_execution') + def test_get_replica_tasks_execution_no_execution( + self, + mock_get_replica_tasks_execution + ): + mock_get_replica_tasks_execution.return_value = None + self.assertRaises( + exception.NotFound, + self.server._get_replica_tasks_execution, + mock.sentinel.context, + mock.sentinel.replica_id, + mock.sentinel.execution_id, + False, + False, + ) + + mock_get_replica_tasks_execution.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.replica_id, + mock.sentinel.execution_id, + include_task_info=False, + to_dict=False) + + @mock.patch.object(db_api, 'delete_replica') + @mock.patch.object(server.ConductorServerEndpoint, + '_check_delete_reservation_for_transfer') + @mock.patch.object(server.ConductorServerEndpoint, + '_check_replica_running_executions') + @mock.patch.object(server.ConductorServerEndpoint, '_get_replica') + def test_delete_replica( + self, + mock_get_replica, + mock_check_replica_running_executions, + mock_check_delete_reservation_for_transfer, + mock_delete_replica, + ): + testutils.get_wrapped_function(self.server.delete_replica)( + self.server, + mock.sentinel.context, + mock.sentinel.replica_id + ) + mock_get_replica.assert_called_once_with( + mock.sentinel.context, mock.sentinel.replica_id) + mock_check_replica_running_executions.assert_called_once_with( + mock.sentinel.context, mock_get_replica.return_value) + mock_check_delete_reservation_for_transfer.assert_called_once_with( + mock_get_replica.return_value) + mock_delete_replica.assert_called_once_with( + mock.sentinel.context, mock.sentinel.replica_id) + + @mock.patch.object( + server.ConductorServerEndpoint, + 'get_replica_tasks_execution' + ) + @mock.patch.object( + server.ConductorServerEndpoint, + '_begin_tasks' + ) + @mock.patch.object(db_api, "add_replica_tasks_execution") + @mock.patch.object(db_api, "update_transfer_action_info_for_instance") + @mock.patch.object( + server.ConductorServerEndpoint, + '_check_execution_tasks_sanity' + ) + @mock.patch.object(copy, "deepcopy") + @mock.patch.object( + server.ConductorServerEndpoint, + '_create_task' + ) + @mock.patch.object(uuid, "uuid4") + @mock.patch.object(models, "TasksExecution") + @mock.patch.object( + server.ConductorServerEndpoint, + '_check_replica_running_executions' + ) + @mock.patch.object( + server.ConductorServerEndpoint, + '_get_replica' + ) + def test_delete_replica_disks( + self, + mock_get_replica, + mock_check_replica_running_executions, + mock_tasks_execution, + mock_uuid4, + mock_create_task, + mock_deepcopy, + mock_check_execution_tasks_sanity, + mock_update_transfer_action_info_for_instance, + mock_add_replica_tasks_execution, + mock_begin_tasks, + mock_get_replica_tasks_execution, + ): + def call_delete_replica_disks(): + return testutils.get_wrapped_function( + self.server.delete_replica_disks)( + self.server, + mock.sentinel.context, + mock.sentinel.replica_id, # type: ignore + ) + instances = [mock.Mock(), mock.Mock()] + mock_replica = mock.Mock( + instances=instances, + id=mock.sentinel.replica_id, + network_map=mock.sentinel.network_map, + info={ + instance: instance + for instance in instances + } + ) + + def create_task_side_effect( + instance, + task_type, + execution, + depends_on=None, + ): + return mock.Mock( + id=task_type, + type=task_type, + instance=instance, + execution=execution, + depends_on=depends_on, + ) + + mock_create_task.side_effect = create_task_side_effect + + mock_get_replica.return_value = mock_replica + result = call_delete_replica_disks() + + mock_get_replica.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.replica_id, + include_task_info=True + ) + mock_check_replica_running_executions.assert_called_once_with( + mock.sentinel.context, + mock_replica + ) + + self.assertEqual( + mock_tasks_execution.return_value.status, constants.EXECUTION_STATUS_UNEXECUTED ) self.assertEqual( @@ -1247,6 +1935,275 @@ def create_task_side_effect( call_delete_replica_disks ) + def test_check_endpoints(self): + origin_endpoint = mock.Mock() + destination_endpoint = mock.Mock() + self.server._check_endpoints( + mock.sentinel.context, origin_endpoint, destination_endpoint) + + def test_check_endpoints_same_destination_id(self): + origin_endpoint = mock.Mock() + destination_endpoint = mock.Mock() + origin_endpoint.id = mock.sentinel.origin_endpoint_id + destination_endpoint.id = mock.sentinel.origin_endpoint_id + self.assertRaises( + exception.SameDestination, + self.server._check_endpoints, + mock.sentinel.context, + origin_endpoint, + destination_endpoint + ) + + def test_check_endpoints_same_destination_connection_info(self): + origin_endpoint = mock.Mock() + destination_endpoint = mock.Mock() + origin_endpoint.connection_info = \ + mock.sentinel.origin_endpoint_connection_info + destination_endpoint.connection_info = \ + mock.sentinel.origin_endpoint_connection_info + self.assertRaises( + exception.SameDestination, + self.server._check_endpoints, + mock.sentinel.context, + origin_endpoint, + destination_endpoint + ) + + @mock.patch.object(server.ConductorServerEndpoint, 'get_replica') + @mock.patch.object(db_api, 'add_replica') + @mock.patch.object(server.ConductorServerEndpoint, + '_check_create_reservation_for_transfer') + @mock.patch.object(server.ConductorServerEndpoint, + '_check_minion_pools_for_action') + @mock.patch.object(models, 'Replica') + @mock.patch.object(server.ConductorServerEndpoint, '_check_endpoints') + @mock.patch.object(server.ConductorServerEndpoint, 'get_endpoint') + def test_create_instances_replica( + self, + mock_get_endpoint, + mock_check_endpoints, + mock_replica, + mock_check_minion_pools_for_action, + mock_check_create_reservation_for_transfer, + mock_add_replica, + mock_get_replica + ): + mock_get_endpoint.side_effect = mock.sentinel.origin_endpoint_id, \ + mock.sentinel.destination_endpoint_id + mock_replica.return_value = mock.Mock() + result = self.server.create_instances_replica( + mock.sentinel.context, + mock.sentinel.origin_endpoint_id, + mock.sentinel.destination_endpoint_id, + mock.sentinel.origin_minion_pool_id, + mock.sentinel.destination_minion_pool_id, + mock.sentinel.instance_osmorphing_minion_pool_mappings, + mock.sentinel.source_environment, + mock.sentinel.destination_environment, + [mock.sentinel.instance_1, mock.sentinel.instance_2], + mock.sentinel.network_map, + mock.sentinel.storage_mappings, + None, + None + ) + self.assertEqual( + mock_get_replica.return_value, + result + ) + mock_get_endpoint.assert_has_calls([ + mock.call(mock.sentinel.context, mock.sentinel.origin_endpoint_id), + mock.call(mock.sentinel.context, + mock.sentinel.destination_endpoint_id)]) + mock_check_endpoints.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.origin_endpoint_id, + mock.sentinel.destination_endpoint_id + ) + self.assertEqual( + ( + mock_replica.return_value.origin_endpoint_id, + mock_replica.return_value.destination_endpoint_id, + mock_replica.return_value.destination_endpoint_id, + mock_replica.return_value.origin_minion_pool_id, + mock_replica.return_value.destination_minion_pool_id, + (mock_replica.return_value. + instance_osmorphing_minion_pool_mappings), + mock_replica.return_value.source_environment, + mock_replica.return_value.destination_environment, + mock_replica.return_value.info, + mock_replica.return_value.notes, + mock_replica.return_value.user_scripts), + ( + mock.sentinel.origin_endpoint_id, + mock.sentinel.destination_endpoint_id, + mock.sentinel.destination_endpoint_id, + mock.sentinel.origin_minion_pool_id, + mock.sentinel.destination_minion_pool_id, + (mock.sentinel. + instance_osmorphing_minion_pool_mappings), + mock.sentinel.source_environment, + mock.sentinel.destination_environment, + {mock.sentinel.instance_1: {'volumes_info': []}, + mock.sentinel.instance_2: {'volumes_info': []}}, + None, + {}) + ) + mock_check_minion_pools_for_action.assert_called_once_with( + mock.sentinel.context, mock_replica.return_value) + mock_check_create_reservation_for_transfer.assert_called_once_with( + mock_replica.return_value, + licensing_client.RESERVATION_TYPE_REPLICA + ) + mock_add_replica.assert_called_once_with( + mock.sentinel.context, mock_replica.return_value) + mock_get_replica.assert_called_once_with( + mock.sentinel.context, mock_replica.return_value.id) + + @mock.patch.object(db_api, 'get_replica') + def test_get_replica(self, mock_get_replica): + result = self.server._get_replica( + mock.sentinel.context, + mock.sentinel.replica_id, + include_task_info=False, + to_dict=False + ) + self.assertEqual( + mock_get_replica.return_value, + result + ) + mock_get_replica.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.replica_id, + include_task_info=False, + to_dict=False + ) + + @mock.patch.object(db_api, 'get_replica') + def test_get_replica_not_found(self, mock_get_replica): + mock_get_replica.return_value = None + self.assertRaises( + exception.NotFound, + self.server._get_replica, + mock.sentinel.context, + mock.sentinel.replica_id, + include_task_info=False, + to_dict=False + ) + mock_get_replica.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.replica_id, + include_task_info=False, + to_dict=False + ) + + @mock.patch.object(db_api, 'get_migrations') + def test_get_migrations(self, mock_get_migrations): + result = self.server.get_migrations( + mock.sentinel.context, + mock.sentinel.migration_id, + include_task_info=False + ) + self.assertEqual( + mock_get_migrations.return_value, + result + ) + mock_get_migrations.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.migration_id, + include_task_info=False, + to_dict=True + ) + + @mock.patch.object(server.ConductorServerEndpoint, '_get_migration') + def test_get_migration(self, mock_get_migration): + result = testutils.get_wrapped_function(self.server.get_migration)( + self.server, + mock.sentinel.context, + mock.sentinel.migration_id, + include_task_info=False + ) + self.assertEqual( + mock_get_migration.return_value, + result + ) + mock_get_migration.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.migration_id, + include_task_info=False, + to_dict=True + ) + + @mock.patch.object(db_api, 'get_replica_migrations') + def test_check_running_replica_migrations( + self, + mock_get_replica_migrations + ): + migration_1 = mock.Mock() + migration_2 = mock.Mock() + migration_1.executions = [mock.Mock()] + migration_1.executions[0].status = \ + constants.EXECUTION_STATUS_UNEXECUTED + migration_2.executions = [mock.Mock()] + migration_2.executions[0].status = \ + constants.EXECUTION_STATUS_UNEXECUTED + migrations = [migration_1, migration_2] + mock_get_replica_migrations.return_value = migrations + self.server._check_running_replica_migrations( + mock.sentinel.context, + mock.sentinel.replica_id, + ) + mock_get_replica_migrations.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.replica_id, + ) + + @mock.patch.object(db_api, 'get_replica_migrations') + def test_check_running_replica_migrations_invalid_replica_state( + self, + mock_get_replica_migrations + ): + migration_1 = mock.Mock() + migration_2 = mock.Mock() + migration_1.executions = [mock.Mock()] + migration_1.executions[0].status = constants.EXECUTION_STATUS_RUNNING + migration_2.executions = [mock.Mock()] + migration_2.executions[0].status = \ + constants.EXECUTION_STATUS_UNEXECUTED + migrations = [migration_1, migration_2] + mock_get_replica_migrations.return_value = migrations + self.assertRaises( + exception.InvalidReplicaState, + self.server._check_running_replica_migrations, + mock.sentinel.context, + mock.sentinel.replica_id, + ) + mock_get_replica_migrations.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.replica_id, + ) + + def test_check_running_executions(self): + execution_1 = mock.Mock() + execution_2 = mock.Mock() + action = mock.Mock() + execution_1.status = constants.EXECUTION_STATUS_UNEXECUTED + execution_2.status = constants.EXECUTION_STATUS_UNEXECUTED + action.executions = [execution_1, execution_2] + self.server._check_running_executions(action) + + def test_check_running_executions_invalid_state(self): + execution_1 = mock.Mock() + execution_2 = mock.Mock() + action = mock.Mock() + execution_1.status = constants.EXECUTION_STATUS_UNEXECUTED + execution_2.status = constants.EXECUTION_STATUS_RUNNING + action.executions = [execution_1, execution_2] + self.assertRaises( + exception.InvalidActionTasksExecutionState, + self.server._check_running_executions, + action + ) + def test_check_valid_replica_tasks_execution(self): execution1 = mock.Mock( number=1, @@ -1421,154 +2378,760 @@ def call_deploy_replica_instance(): user_scripts=mock.sentinel.user_scripts, ) - # One of the instances has no volumes info + # One of the instances has no volumes info + self.assertRaises( + exception.InvalidReplicaState, + call_deploy_replica_instance, + ) + + mock_get_endpoint.assert_called_once_with( + mock.sentinel.context, + mock_get_replica.return_value.destination_endpoint_id + ) + + mock_get_replica.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.replica_id, + include_task_info=True, + ) + mock_check_reservation_for_transfer.assert_called_once_with( + mock_get_replica.return_value, + licensing_client.RESERVATION_TYPE_REPLICA + ) + mock_check_replica_running_executions.assert_called_once_with( + mock.sentinel.context, + mock_get_replica.return_value + ) + mock_check_valid_replica_tasks_execution.assert_called_once_with( + mock_get_replica.return_value, + False + ) + mock_get_provider_types.assert_called_once_with( + mock.sentinel.context, + mock_get_endpoint.return_value + ) + + # add the missing volumes info + mock_get_replica.return_value.info[mock.sentinel.instance2] = { + 'volumes_info': mock.sentinel.volumes_info2 + } + + def create_task_side_effect( + instance, + task_type, + execution, + depends_on=None, + on_error=False, + on_error_only=False + ): + return mock.Mock( + id=task_type, + type=task_type, + instance=instance, + execution=execution, + depends_on=depends_on, + on_error=on_error, + on_error_only=on_error_only, + ) + + mock_create_task.side_effect = create_task_side_effect + + # no longer raises exception + migration = call_deploy_replica_instance() + + mock_check_minion_pools_for_action.assert_called_once_with( + mock.sentinel.context, + mock_migration.return_value + ) + + self.assertEqual( + mock_tasks_execution.return_value.status, + constants.EXECUTION_STATUS_UNEXECUTED + ) + self.assertEqual( + mock_tasks_execution.return_value.type, + constants.EXECUTION_TYPE_REPLICA_DEPLOY + ) + + for instance in mock_get_replica.return_value.instances: + mock_get_instance_scripts.assert_any_call( + mock.sentinel.user_scripts, + instance, + ) + mock_create_task.assert_any_call( + instance, + constants.TASK_TYPE_VALIDATE_REPLICA_DEPLOYMENT_INPUTS, + mock_tasks_execution.return_value, + ) + + # tasks defined in the yaml config + for task in expected_tasks: + kwargs = {} + if 'on_error' in task: + kwargs = {'on_error': task['on_error']} + if 'on_error_only' in task: + kwargs = {'on_error_only': task['on_error_only']} + mock_create_task.assert_has_calls([ + mock.call( + instance, + task['type'], + mock_tasks_execution.return_value, + depends_on=task['depends_on'], + **kwargs, + ) + ]) + + mock_check_execution_tasks_sanity.assert_called_once_with( + mock_tasks_execution.return_value, + mock_migration.return_value.info, + ) + + mock_add_migration.assert_called_once_with( + mock.sentinel.context, + mock_migration.return_value, + ) + + if not skip_os_morphing and has_os_morphing_minion: + mock_lock.assert_any_call( + constants.MIGRATION_LOCK_NAME_FORMAT + % mock_migration.return_value.id, + external=True, + ) + mock_minion_manager_client\ + .allocate_minion_machines_for_migration\ + .assert_called_once_with( + mock.sentinel.context, + mock_migration.return_value, + include_transfer_minions=False, + include_osmorphing_minions=True + ) + mock_set_tasks_execution_status.assert_called_once_with( + mock.sentinel.context, + mock_tasks_execution.return_value, + constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS + ) + else: + mock_begin_tasks.assert_called_once_with( + mock.sentinel.context, + mock_migration.return_value, + mock_tasks_execution.return_value, + ) + + mock_get_migration.assert_called_once_with( + mock.sentinel.context, + mock_migration.return_value.id, + ) + + self.assertEqual( + migration, + mock_get_migration.return_value + ) + + def test_get_instance_scripts( + self + ): + user_scripts = { + "global": 'mock_user_scripts', + "instances": { + mock.sentinel.instance_1: mock.sentinel.scripts_1, + mock.sentinel.instance_2: mock.sentinel.scripts_2, + } + } + expected_result = { + 'global': 'mock_user_scripts', + 'instances': {mock.sentinel.instance_1: mock.sentinel.scripts_1} + } + result = self.server._get_instance_scripts( + user_scripts, mock.sentinel.instance_1 + ) + self.assertEqual( + expected_result, + result + ) + + def test_get_instance_scripts_no_instance_script( + self + ): + user_scripts = { + "global": 'mock_user_scripts', + "instances": { + mock.sentinel.instance_1: None, + mock.sentinel.instance_2: mock.sentinel.scripts_2, + } + } + expected_result = { + 'global': 'mock_user_scripts', + 'instances': {} + } + result = self.server._get_instance_scripts( + user_scripts, mock.sentinel.instance_1 + ) + self.assertEqual( + expected_result, + result + ) + + def test_get_instance_scripts_no_user_scripts( + self + ): + user_scripts = None + expected_result = { + 'global': {}, + 'instances': {} + } + result = self.server._get_instance_scripts( + user_scripts, mock.sentinel.instance_1 + ) + self.assertEqual( + expected_result, + result + ) + + @mock.patch.object(db_api, "update_transfer_action_info_for_instance") + def test_update_task_info_for_minion_allocations( + self, + mock_update_transfer_action_info_for_instance + ): + action = mock.Mock() + action.id = mock.sentinel.action_id + action.instances = [mock.sentinel.instance1, mock.sentinel.instance2] + action.info = { + mock.sentinel.instance1: {}, + mock.sentinel.instance2: { + 'origin_minion_machine_id': + mock.sentinel.origin_minion_id, + 'origin_minion_provider_properties': + mock.sentinel.origin_minion_provider_properties, + 'origin_minion_connection_info': + mock.sentinel.origin_minion_connection_info, + 'destination_minion_machine_id': + mock.sentinel.destination_minion_id, + 'destination_minion_provider_properties': + mock.sentinel.destination_minion_provider_properties, + 'destination_minion_connection_info': + mock.sentinel.destination_minion_connection_info, + 'destination_minion_backup_writer_connection_info': + mock.sentinel.destination_minion_backup_writer_connection_info, + 'osmorphing_minion_machine_id': + mock.sentinel.osmorphing_minion_id, + 'osmorphing_minion_provider_properties': + mock.sentinel.osmorphing_minion_provider_properties, + 'osmorphing_minion_connection_info': + mock.sentinel.osmorphing_minion_connection_info, + }, + mock.sentinel.instance3: {} + } + minion_machine_allocations = { + mock.sentinel.instance1: { + 'origin_minion': { + 'id': mock.sentinel.updated_origin_minion_id, + "provider_properties": + mock.sentinel.updated_origin_minion_provider_properties, + "connection_info": + mock.sentinel.updated_origin_minion_connection_info, + }, + 'destination_minion': { + 'id': mock.sentinel.updated_destination_minion_id, + "provider_properties": + (mock.sentinel. + updated_destination_minion_provider_properties), + "connection_info": + mock.sentinel.updated_destination_minion_connection_info, + "backup_writer_connection_info": + (mock.sentinel. + updated_destination_minion_backup_writer_connection_info) + }, + 'osmorphing_minion': { + 'id': mock.sentinel.updated_osmorphing_minion_id, + "provider_properties": + (mock.sentinel. + updated_osmorphing_minion_provider_properties), + "connection_info": + mock.sentinel.updated_osmorphing_minion_connection_info, + } + }, + } + + expected_action_info = { + mock.sentinel.instance1: { + 'origin_minion_machine_id': + mock.sentinel.updated_origin_minion_id, + 'origin_minion_provider_properties': + mock.sentinel.updated_origin_minion_provider_properties, + 'origin_minion_connection_info': + mock.sentinel.updated_origin_minion_connection_info, + 'destination_minion_machine_id': + mock.sentinel.updated_destination_minion_id, + 'destination_minion_provider_properties': + mock.sentinel.updated_destination_minion_provider_properties, + 'destination_minion_connection_info': + mock.sentinel.updated_destination_minion_connection_info, + 'destination_minion_backup_writer_connection_info': + (mock.sentinel. + updated_destination_minion_backup_writer_connection_info), + 'osmorphing_minion_machine_id': + mock.sentinel.updated_osmorphing_minion_id, + 'osmorphing_minion_provider_properties': + mock.sentinel.updated_osmorphing_minion_provider_properties, + 'osmorphing_minion_connection_info': + mock.sentinel.updated_osmorphing_minion_connection_info, + }, + } + self.server._update_task_info_for_minion_allocations( + mock.sentinel.context, + action, + minion_machine_allocations + ) + mock_update_transfer_action_info_for_instance.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.action_id, + mock.sentinel.instance1, + expected_action_info[mock.sentinel.instance1] + ) + minion_machine_allocations = { + mock.sentinel.instance2: {}, + mock.sentinel.instance4: {}, + } + expected_action_info = { + mock.sentinel.instance2: { + 'origin_minion_machine_id': + mock.sentinel.origin_minion_id, + 'origin_minion_provider_properties': + mock.sentinel.origin_minion_provider_properties, + 'origin_minion_connection_info': + mock.sentinel.origin_minion_connection_info, + 'destination_minion_machine_id': + mock.sentinel.destination_minion_id, + 'destination_minion_provider_properties': + mock.sentinel.destination_minion_provider_properties, + 'destination_minion_connection_info': + mock.sentinel.destination_minion_connection_info, + 'destination_minion_backup_writer_connection_info': + mock.sentinel.destination_minion_backup_writer_connection_info, + 'osmorphing_minion_machine_id': + mock.sentinel.osmorphing_minion_id, + 'osmorphing_minion_provider_properties': + mock.sentinel.osmorphing_minion_provider_properties, + 'osmorphing_minion_connection_info': + mock.sentinel.osmorphing_minion_connection_info, + }, + } + mock_update_transfer_action_info_for_instance.reset_mock() + self.server._update_task_info_for_minion_allocations( + mock.sentinel.context, + action, + minion_machine_allocations + ) + mock_update_transfer_action_info_for_instance.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.action_id, + mock.sentinel.instance2, + expected_action_info[mock.sentinel.instance2] + ) + + @mock.patch.object(server.ConductorServerEndpoint, '_get_replica') + def test_get_last_execution_for_replica( + self, + mock_get_replica + ): + replica = mock.Mock() + replica.id = mock.sentinel.id + execution1 = mock.Mock(id=mock.sentinel.execution_id1, number=1) + execution2 = mock.Mock(id=mock.sentinel.execution_id2, number=3) + execution3 = mock.Mock(id=mock.sentinel.execution_id3, number=2) + replica.executions = [execution1, execution2, execution3] + mock_get_replica.return_value = replica + result = self.server._get_last_execution_for_replica( + mock.sentinel.context, + replica, + requery=False + ) + self.assertEqual( + execution2, + result + ) + mock_get_replica.assert_not_called() + replica.executions = None + self.assertRaises( + exception.InvalidReplicaState, + self.server._get_last_execution_for_replica, + mock.sentinel.context, + replica, + requery=True + ) + mock_get_replica.assert_called_once_with( + mock.sentinel.context, mock.sentinel.id) + + @mock.patch.object(server.ConductorServerEndpoint, '_get_migration') + def test_get_execution_for_migration( + self, + mock_get_migration + ): + migration = mock.Mock() + migration.id = mock.sentinel.id + execution1 = mock.Mock(id=mock.sentinel.execution_id1) + execution2 = mock.Mock(id=mock.sentinel.execution_id2) + migration.executions = [execution1] + mock_get_migration.return_value = migration + result = self.server._get_execution_for_migration( + mock.sentinel.context, + migration, + requery=False + ) + self.assertEqual( + execution1, + result + ) + mock_get_migration.assert_not_called() + migration.executions = [execution1, execution2] + self.assertRaises( + exception.InvalidMigrationState, + self.server._get_execution_for_migration, + mock.sentinel.context, + migration, + requery=True + ) + mock_get_migration.assert_called_once_with( + mock.sentinel.context, mock.sentinel.id) + migration.executions = [] + self.assertRaises( + exception.InvalidMigrationState, + self.server._get_execution_for_migration, + mock.sentinel.context, + migration, + requery=False + ) + + @mock.patch.object(server.ConductorServerEndpoint, '_begin_tasks') + @mock.patch.object(db_api, 'get_replica_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, + '_update_task_info_for_minion_allocations') + @mock.patch.object(server.ConductorServerEndpoint, + '_get_last_execution_for_replica') + @mock.patch.object(server.ConductorServerEndpoint, '_get_replica') + def test_confirm_replica_minions_allocation( + self, + mock_get_replica, + mock_get_last_execution_for_replica, + mock_update_task_info_for_minion_allocations, + mock_get_replica_tasks_execution, + mock_begin_tasks + ): + mock_get_replica.return_value.last_execution_status = \ + constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS + + testutils.get_wrapped_function( + self.server.confirm_replica_minions_allocation)( + self.server, + mock.sentinel.context, + mock.sentinel.replica_id, + mock.sentinel.minion_machine_allocations + ) + + mock_get_replica.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.replica_id, + include_task_info=True + ) + mock_get_last_execution_for_replica.assert_called_once_with( + mock.sentinel.context, + mock_get_replica.return_value, + requery=False + ) + mock_update_task_info_for_minion_allocations.assert_called_once_with( + mock.sentinel.context, + mock_get_replica.return_value, + mock.sentinel.minion_machine_allocations + ) + mock_get_replica_tasks_execution.assert_called_once_with( + mock.sentinel.context, + mock_get_replica.return_value.id, + mock_get_last_execution_for_replica.return_value.id + ) + mock_begin_tasks.assert_called_once_with( + mock.sentinel.context, + mock_get_replica.return_value, + mock_get_replica_tasks_execution.return_value + ) + + @mock.patch.object(server.ConductorServerEndpoint, '_begin_tasks') + @mock.patch.object(db_api, 'get_replica_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, + '_update_task_info_for_minion_allocations') + @mock.patch.object(server.ConductorServerEndpoint, + '_get_last_execution_for_replica') + @mock.patch.object(server.ConductorServerEndpoint, '_get_replica') + def test_confirm_replica_minions_allocation_unexpected_status( + self, + mock_get_replica, + mock_get_last_execution_for_replica, + mock_update_task_info_for_minion_allocations, + mock_get_replica_tasks_execution, + mock_begin_tasks + ): + mock_get_replica.return_value.last_execution_status = \ + constants.EXECUTION_STATUS_CANCELED + self.assertRaises( exception.InvalidReplicaState, - call_deploy_replica_instance, - ) - - mock_get_endpoint.assert_called_once_with( + testutils.get_wrapped_function( + self.server.confirm_replica_minions_allocation), + self.server, mock.sentinel.context, - mock_get_replica.return_value.destination_endpoint_id + mock.sentinel.replica_id, + mock.sentinel.minion_machine_allocations ) mock_get_replica.assert_called_once_with( mock.sentinel.context, mock.sentinel.replica_id, - include_task_info=True, + include_task_info=True ) - mock_check_reservation_for_transfer.assert_called_once_with( - mock_get_replica.return_value, - licensing_client.RESERVATION_TYPE_REPLICA + mock_get_last_execution_for_replica.assert_not_called() + mock_update_task_info_for_minion_allocations.assert_not_called() + mock_get_replica_tasks_execution.assert_not_called() + mock_begin_tasks.assert_not_called() + + @mock.patch.object(server.ConductorServerEndpoint, + '_set_tasks_execution_status') + @mock.patch.object(server.ConductorServerEndpoint, + '_cancel_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, + '_get_last_execution_for_replica') + @mock.patch.object(server.ConductorServerEndpoint, '_get_replica') + def test_report_replica_minions_allocation_error( + self, + mock_get_replica, + mock_get_last_execution_for_replica, + mock_cancel_tasks_execution, + mock_set_tasks_execution_status + ): + mock_get_replica.return_value.last_execution_status = \ + constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS + + testutils.get_wrapped_function( + self.server.report_replica_minions_allocation_error)( + self.server, + mock.sentinel.context, + mock.sentinel.replica_id, + mock.sentinel.minion_allocation_error_details ) - mock_check_replica_running_executions.assert_called_once_with( + + mock_get_replica.assert_called_once_with( mock.sentinel.context, - mock_get_replica.return_value + mock.sentinel.replica_id ) - mock_check_valid_replica_tasks_execution.assert_called_once_with( + mock_get_last_execution_for_replica.assert_called_once_with( + mock.sentinel.context, mock_get_replica.return_value, - False + requery=False ) - mock_get_provider_types.assert_called_once_with( + mock_cancel_tasks_execution.assert_called_once_with( mock.sentinel.context, - mock_get_endpoint.return_value + mock_get_last_execution_for_replica.return_value, + requery=True ) + mock_set_tasks_execution_status.assert_called_once_with( + mock.sentinel.context, + mock_get_last_execution_for_replica.return_value, + constants.EXECUTION_STATUS_ERROR_ALLOCATING_MINIONS + ) + + @mock.patch.object(server.ConductorServerEndpoint, + '_set_tasks_execution_status') + @mock.patch.object(server.ConductorServerEndpoint, + '_cancel_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, + '_get_last_execution_for_replica') + @mock.patch.object(server.ConductorServerEndpoint, '_get_replica') + def test_report_replica_minions_allocation_error_unexpected_status( + self, + mock_get_replica, + mock_get_last_execution_for_replica, + mock_cancel_tasks_execution, + mock_set_tasks_execution_status + ): + mock_get_replica.return_value.last_execution_status = \ + constants.EXECUTION_STATUS_CANCELED - # add the missing volumes info - mock_get_replica.return_value.info[mock.sentinel.instance2] = { - 'volumes_info': mock.sentinel.volumes_info2 - } + self.assertRaises( + exception.InvalidReplicaState, + testutils.get_wrapped_function( + self.server.report_replica_minions_allocation_error), + self.server, + mock.sentinel.context, + mock.sentinel.replica_id, + mock.sentinel.minion_allocation_error_details + ) - def create_task_side_effect( - instance, - task_type, - execution, - depends_on=None, - on_error=False, - on_error_only=False - ): - return mock.Mock( - id=task_type, - type=task_type, - instance=instance, - execution=execution, - depends_on=depends_on, - on_error=on_error, - on_error_only=on_error_only, - ) + mock_get_replica.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.replica_id + ) + mock_get_last_execution_for_replica.assert_not_called() + mock_cancel_tasks_execution.assert_not_called() + mock_set_tasks_execution_status.assert_not_called() - mock_create_task.side_effect = create_task_side_effect + @mock.patch.object(server.ConductorServerEndpoint, '_begin_tasks') + @mock.patch.object(server.ConductorServerEndpoint, + '_update_task_info_for_minion_allocations') + @mock.patch.object(server.ConductorServerEndpoint, + '_get_execution_for_migration') + @mock.patch.object(server.ConductorServerEndpoint, '_get_migration') + def test_confirm_migration_minions_allocation( + self, + mock_get_migration, + mock_get_execution_for_migration, + mock_update_task_info_for_minion_allocations, + mock_begin_tasks + ): + mock_get_migration.return_value.last_execution_status = \ + constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS - # no longer raises exception - migration = call_deploy_replica_instance() + testutils.get_wrapped_function( + self.server.confirm_migration_minions_allocation)( + self.server, + mock.sentinel.context, + mock.sentinel.replica_id, + mock.sentinel.minion_machine_allocations + ) - mock_check_minion_pools_for_action.assert_called_once_with( + mock_get_migration.assert_called_once_with( mock.sentinel.context, - mock_migration.return_value + mock.sentinel.replica_id, + include_task_info=True ) - - self.assertEqual( - mock_tasks_execution.return_value.status, - constants.EXECUTION_STATUS_UNEXECUTED + mock_get_execution_for_migration.assert_called_once_with( + mock.sentinel.context, + mock_get_migration.return_value, + requery=False ) - self.assertEqual( - mock_tasks_execution.return_value.type, - constants.EXECUTION_TYPE_REPLICA_DEPLOY + mock_update_task_info_for_minion_allocations.assert_called_once_with( + mock.sentinel.context, + mock_get_migration.return_value, + mock.sentinel.minion_machine_allocations ) + mock_begin_tasks.assert_called_once_with( + mock.sentinel.context, + mock_get_migration.return_value, + mock_get_execution_for_migration.return_value + ) + + @mock.patch.object(server.ConductorServerEndpoint, '_begin_tasks') + @mock.patch.object(server.ConductorServerEndpoint, + '_update_task_info_for_minion_allocations') + @mock.patch.object(server.ConductorServerEndpoint, + '_get_execution_for_migration') + @mock.patch.object(server.ConductorServerEndpoint, '_get_migration') + def test_confirm_migration_minions_allocation_unexpected_status( + self, + mock_get_migration, + mock_get_execution_for_migration, + mock_update_task_info_for_minion_allocations, + mock_begin_tasks + ): + mock_get_migration.return_value.last_execution_status = \ + constants.EXECUTION_STATUS_CANCELED - for instance in mock_get_replica.return_value.instances: - mock_get_instance_scripts.assert_any_call( - mock.sentinel.user_scripts, - instance, - ) - mock_create_task.assert_any_call( - instance, - constants.TASK_TYPE_VALIDATE_REPLICA_DEPLOYMENT_INPUTS, - mock_tasks_execution.return_value, - ) - - # tasks defined in the yaml config - for task in expected_tasks: - kwargs = {} - if 'on_error' in task: - kwargs = {'on_error': task['on_error']} - if 'on_error_only' in task: - kwargs = {'on_error_only': task['on_error_only']} - mock_create_task.assert_has_calls([ - mock.call( - instance, - task['type'], - mock_tasks_execution.return_value, - depends_on=task['depends_on'], - **kwargs, - ) - ]) - - mock_check_execution_tasks_sanity.assert_called_once_with( - mock_tasks_execution.return_value, - mock_migration.return_value.info, + self.assertRaises( + exception.InvalidMigrationState, + testutils.get_wrapped_function( + self.server.confirm_migration_minions_allocation), + self.server, + mock.sentinel.context, + mock.sentinel.replica_id, + mock.sentinel.minion_machine_allocations ) - mock_add_migration.assert_called_once_with( + mock_get_migration.assert_called_once_with( mock.sentinel.context, - mock_migration.return_value, + mock.sentinel.replica_id, + include_task_info=True ) + mock_get_execution_for_migration.assert_not_called() + mock_update_task_info_for_minion_allocations.assert_not_called() + + mock_begin_tasks.assert_not_called() + + @mock.patch.object(server.ConductorServerEndpoint, + '_set_tasks_execution_status') + @mock.patch.object(server.ConductorServerEndpoint, + '_cancel_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, + '_get_execution_for_migration') + @mock.patch.object(server.ConductorServerEndpoint, '_get_migration') + def test_report_migration_minions_allocation_error( + self, + mock_get_migration, + mock_get_execution_for_migration, + mock_cancel_tasks_execution, + mock_set_tasks_execution_status + ): + mock_get_migration.return_value.last_execution_status = \ + constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS - if not skip_os_morphing and has_os_morphing_minion: - mock_lock.assert_any_call( - constants.MIGRATION_LOCK_NAME_FORMAT - % mock_migration.return_value.id, - external=True, - ) - mock_minion_manager_client\ - .allocate_minion_machines_for_migration\ - .assert_called_once_with( - mock.sentinel.context, - mock_migration.return_value, - include_transfer_minions=False, - include_osmorphing_minions=True - ) - mock_set_tasks_execution_status.assert_called_once_with( - mock.sentinel.context, - mock_tasks_execution.return_value, - constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS - ) - else: - mock_begin_tasks.assert_called_once_with( + testutils.get_wrapped_function( + self.server.report_migration_minions_allocation_error)( + self.server, mock.sentinel.context, - mock_migration.return_value, - mock_tasks_execution.return_value, - ) + mock.sentinel.replica_id, + mock.sentinel.minion_allocation_error_details + ) mock_get_migration.assert_called_once_with( mock.sentinel.context, - mock_migration.return_value.id, + mock.sentinel.replica_id + ) + mock_get_execution_for_migration.assert_called_once_with( + mock.sentinel.context, + mock_get_migration.return_value, + requery=False + ) + mock_cancel_tasks_execution.assert_called_once_with( + mock.sentinel.context, + mock_get_execution_for_migration.return_value, + requery=True ) + mock_set_tasks_execution_status.assert_called_once_with( + mock.sentinel.context, + mock_get_execution_for_migration.return_value, + constants.EXECUTION_STATUS_ERROR_ALLOCATING_MINIONS + ) + + @mock.patch.object(server.ConductorServerEndpoint, + '_set_tasks_execution_status') + @mock.patch.object(server.ConductorServerEndpoint, + '_cancel_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, + '_get_execution_for_migration') + @mock.patch.object(server.ConductorServerEndpoint, '_get_migration') + def test_report_migration_minions_allocation_error_unexpected_status( + self, + mock_get_migration, + mock_get_execution_for_migration, + mock_cancel_tasks_execution, + mock_set_tasks_execution_status + ): + mock_get_migration.return_value.last_execution_status = \ + constants.EXECUTION_STATUS_CANCELED - self.assertEqual( - migration, - mock_get_migration.return_value + self.assertRaises( + exception.InvalidMigrationState, + testutils.get_wrapped_function( + self.server.report_migration_minions_allocation_error), + self.server, + mock.sentinel.context, + mock.sentinel.replica_id, + mock.sentinel.minion_allocation_error_details + ) + + mock_get_migration.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.replica_id ) + mock_get_execution_for_migration.assert_not_called() + mock_cancel_tasks_execution.assert_not_called() + mock_set_tasks_execution_status.assert_not_called() @mock.patch.object( server.ConductorServerEndpoint, @@ -1959,6 +3522,152 @@ def call_cancel_tasks_execution( exception_details=mock.ANY, ) + @mock.patch.object(db_api, 'get_migration') + def test__get_migration( + self, + mock_get_migration + ): + result = self.server._get_migration( + mock.sentinel.context, + mock.sentinel.migration_id, + include_task_info=False, + to_dict=False + ) + self.assertEqual( + mock_get_migration.return_value, + result + ) + + mock_get_migration.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.migration_id, + include_task_info=False, + to_dict=False + ) + mock_get_migration.reset_mock() + mock_get_migration.return_value = None + + self.assertRaises( + exception.NotFound, + self.server._get_migration, + mock.sentinel.context, + mock.sentinel.migration_id, + include_task_info=False, + to_dict=False + ) + + mock_get_migration.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.migration_id, + include_task_info=False, + to_dict=False + ) + + @mock.patch.object(db_api, 'delete_migration') + @mock.patch.object(server.ConductorServerEndpoint, '_get_migration') + def test_delete_migration( + self, + mock_get_migration, + mock_delete_migration + ): + migration = mock.Mock() + execution = mock.Mock() + execution.status = constants.CANCELED_TASK_STATUSES + migration.executions = [execution] + mock_get_migration.return_value = migration + + testutils.get_wrapped_function(self.server.delete_migration)( + self.server, + mock.sentinel.context, + mock.sentinel.migration_id + ) + + mock_get_migration.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.migration_id + ) + mock_delete_migration.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.migration_id + ) + mock_get_migration.reset_mock() + mock_delete_migration.reset_mock() + execution.status = constants.EXECUTION_STATUS_RUNNING + + self.assertRaises( + exception.InvalidMigrationState, + testutils.get_wrapped_function(self.server.delete_migration), + self.server, + mock.sentinel.context, + mock.sentinel.migration_id + ) + + mock_get_migration.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.migration_id + ) + mock_delete_migration.assert_not_called() + + @mock.patch.object(server.ConductorServerEndpoint, + '_check_delete_reservation_for_transfer') + @mock.patch.object(server.ConductorServerEndpoint, + '_cancel_tasks_execution') + @mock.patch.object(lockutils, 'lock') + @mock.patch.object(server.ConductorServerEndpoint, '_get_migration') + @ddt.file_data("data/cancel_migration_config.yml") + @ddt.unpack + def test_cancel_migration( + self, + mock_get_migration, + mock_lock, + mock_cancel_tasks_execution, + mock_check_delete_reservation_for_transfer, + config, + raises_exception + ): + migration = mock.Mock() + migration.executions = [] + statuses = config.get('execution_statuses', []) + for status in statuses: + execution = mock.Mock() + execution.status = getattr(constants, status) + migration.executions.append(execution) + mock_get_migration.return_value = migration + force = config.get('force', False) + + if raises_exception: + self.assertRaises( + exception.InvalidMigrationState, + testutils.get_wrapped_function(self.server.cancel_migration), + self.server, + mock.sentinel.context, + mock.sentinel.migration_id, + force=force + ) + else: + testutils.get_wrapped_function(self.server.cancel_migration)( + self.server, + mock.sentinel.context, + mock.sentinel.migration_id, + force=force + ) + mock_lock.assert_called_once_with( + constants.EXECUTION_LOCK_NAME_FORMAT % execution.id, + external=True + ) + mock_cancel_tasks_execution.assert_called_once_with( + mock.sentinel.context, + execution, + force=force + ) + mock_check_delete_reservation_for_transfer.assert_called_once_with( + migration) + + mock_get_migration.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.migration_id + ) + @mock.patch.object(db_api, 'get_tasks_execution') @mock.patch.object( server.ConductorServerEndpoint,