diff --git a/bin/sm b/bin/sm index d841dff..5f3bc6a 100755 --- a/bin/sm +++ b/bin/sm @@ -10,7 +10,7 @@ from servicemanager.smstatus import dostatus from servicemanager.smcontext import SmApplication, SmContext, ServiceManagerException from servicemanager.smrepo import pull_rebase_repo from servicemanager.actions import actions -import servicemanager.smrepo +from servicemanager import smrepo def _process_command(): @@ -37,6 +37,7 @@ def _process_command(): parser.add_argument('--printconfig', type=str, nargs='*', help='Print the config for a given list of services, if empty, show all') parser.add_argument('-c', '--config', type=str, help='Sets the configuration directory location, defaults to $WORKSPACE/service-manager-config') parser.add_argument('--getdascode', action='store_true', help='Checkout all of the code if you haven\'t already') + parser.add_argument('--showcmdfor', type=str, nargs='*', help='Shows how sm will try start the service') args = parser.parse_args() @@ -122,6 +123,17 @@ def _process_command(): path = context.application.workspace + service_data["location"] smrepo.clone_repo_if_required_raw(service_name, service_data["sources"]["repo"], path, context) + if args.showcmdfor: + print "The parameters you provided would start the following services using the following commands:" + for service_name in service_resolver.resolve_services_from_array(args.showcmdfor): + if context.has_service(service_name): + cmd = actions.get_start_cmd(context, service_name, args.fatjar, args.release, args.proxy, actions.overridden_port(args.showcmdfor, args.port)) + print service_name + print " ".join(cmd) + else: + print "The requested service %s does not exist" % service_name + + try: _process_command() except ServiceManagerException as e: diff --git a/servicemanager/actions/actions.py b/servicemanager/actions/actions.py index 71d3659..ec61a83 100755 --- a/servicemanager/actions/actions.py +++ b/servicemanager/actions/actions.py @@ -1,6 +1,5 @@ #!/usr/bin/env python import os -import sys import time import calendar import glob @@ -30,6 +29,16 @@ def start_one(context, service_name, fatjar, release, proxy, port=None): return False +def get_start_cmd(context, service_name, fatjar, release, proxy, port=None): + if release: + run_from = "RELEASE" + elif fatjar: + run_from = "SNAPSHOT" + else: + run_from = "SOURCE" + + starter = context.get_service_starter(service_name, run_from, proxy, None, None, port) + return starter.get_start_command(run_from) def stop_profile(context, profile): for service_name in context.application.services_for_profile(profile): @@ -127,7 +136,6 @@ def _get_git_rev(context, service_name): return "" - def display_info(context, service_name): arguments = _get_running_process_args(context, service_name) git_revision = _get_git_rev(context, service_name) diff --git a/servicemanager/service/sbt-play-wrapper.py b/servicemanager/service/sbt-play-wrapper.py index fba7dbe..679d6ca 100644 --- a/servicemanager/service/sbt-play-wrapper.py +++ b/servicemanager/service/sbt-play-wrapper.py @@ -5,7 +5,6 @@ from servicemanager import subprocess - try: args = json.loads(sys.argv[1]) print u"Executing: %s" % " ".join(args) diff --git a/servicemanager/service/smdropwizardservice.py b/servicemanager/service/smdropwizardservice.py index 1729bcc..25bc0f9 100644 --- a/servicemanager/service/smdropwizardservice.py +++ b/servicemanager/service/smdropwizardservice.py @@ -33,6 +33,30 @@ def __init__(self, context, service_name, run_from, port, admin_port, classifier def _get_jar_filename(self): return self.context.get_jar_filename(self.service_name, self.run_from) + def _get_binary_start_cmd(self): + binary_data = self.service_data["binary"] + filename = self._get_jar_filename() + + if "configurationFile" not in binary_data: + print "ERROR: required config 'configurationFile' does not exist" + return None + + configuration_file = binary_data["configurationFile"] + output_configuration_file = os.path.join(self.run_from, configuration_file) + microservice_target_path = self.context.get_microservice_target_path(self.service_name) + _java_bin = os.path.join(self.java_home, os.path.join("bin", "java")) + extra_params = self._build_extra_params() + cmd = [_java_bin] + self.java_options + extra_params + os.path.join(microservice_target_path, filename) + return cmd + ["-jar", microservice_target_path + filename, "server", microservice_target_path + output_configuration_file] + + + def get_start_command(self, run_from): + if run_from == "SOURCE": + return self.sources["cmd"] + else: + return self._get_binary_start_cmd() + def start_from_binary(self): microservice_target_path = self.context.get_microservice_target_path(self.service_name) @@ -53,23 +77,12 @@ def start_from_binary(self): print "ERROR: required config 'configurationFile' does not exist" return None - output_configuration_file = os.path.join(self.run_from, configuration_file) - binary_to_run = os.path.join(microservice_target_path, filename) if os.path.exists(binary_to_run): force_chdir(self.run_from) os.system("jar xvf " + microservice_target_path + filename + " " + configuration_file + " > /dev/null") force_chdir(microservice_target_path) - - _java_bin = os.path.join(self.java_home, os.path.join("bin", "java")) - - extra_params = self._build_extra_params() - - cmd = [_java_bin] + self.java_options + extra_params - - os.path.join(microservice_target_path, filename) - - cmd = cmd + ["-jar", microservice_target_path + filename, "server", microservice_target_path + output_configuration_file] + cmd = self.get_start_command("BINARY") process = Popen(cmd, env=os.environ.copy(), stdout=SmDropwizardServiceStarter.DEV_NULL, stderr=SmDropwizardServiceStarter.DEV_NULL, close_fds=True) if process.returncode == 1: print b.fail + "ERROR: could not start '" + self.service_name + "' " + b.endc @@ -117,7 +130,7 @@ def start_from_sources(self): # SBT is so specific should this be in configuration? new_env["SBT_EXTRA_PARAMS"] = " ".join(sbt_extra_params) os.chdir(base_dir) - process = Popen(cmd, env=new_env, stdout=SmDropwizardServiceStarter.DEV_NULL, stderr=SmDropwizardServiceStarter.DEV_NULL, close_fds=True) + process = Popen(self.get_start_command("SOURCE"), env=new_env, stdout=SmDropwizardServiceStarter.DEV_NULL, stderr=SmDropwizardServiceStarter.DEV_NULL, close_fds=True) except Exception, e: self.log("Could not start service in '%s' due to exception: %s" % (base_dir, str(e))) diff --git a/servicemanager/service/smplayservice.py b/servicemanager/service/smplayservice.py index 574ded7..d46b21d 100644 --- a/servicemanager/service/smplayservice.py +++ b/servicemanager/service/smplayservice.py @@ -65,6 +65,14 @@ def _build_extra_params(self): return extra_params + def get_start_command(self, run_from): + if run_from == "SOURCE": + source_cmd = copy.copy(self.service_data["sources"]["cmd"]) + source_cmd[-1] = source_cmd[-1] + " " + " ".join(self.sbt_extra_params()) + return source_cmd + else: + return self.service_data["binary"]["cmd"] + self._build_extra_params() + def start_from_binary(self): microservice_target_path = self.context.get_microservice_target_path(self.service_name) force_chdir(microservice_target_path) @@ -77,7 +85,7 @@ def start_from_binary(self): parent, _ = os.path.split(unzip_dir) force_pushdir(parent) - cmd_with_params = self.service_data["binary"]["cmd"] + self._build_extra_params() + cmd_with_params = self.get_start_command("BINARY") if os.path.exists(cmd_with_params[0]): os.chmod(cmd_with_params[0], stat.S_IRWXU) else: @@ -111,26 +119,29 @@ def _unzip_play_application(self): return target_dir - def start_from_sources(self): + def sbt_extra_params(self): sbt_extra_params = self._build_extra_params() if "extra_params" in self.service_data["sources"]: sbt_extra_params += self.service_data["sources"]["extra_params"] + return sbt_extra_params + + def start_from_sources(self): + sbt_extra_params = self.sbt_extra_params() + service_data = self.context.service_data(self.service_name) microservice_path = self.context.application.workspace + service_data["location"] curr_dir = force_pushdir(microservice_path) remove_if_exists("RUNNING_PID") env_copy = os.environ.copy() - env_copy["SBT_EXTRA_PARAMS"] = " ".join(sbt_extra_params) + env_copy["SBT_EXTRA_PARAMS"] = " ".join(sbt_extra_params) # TODO: not needed i think anymore... makedirs_if_not_exists("logs") with open("logs/stdout.txt", "wb") as out, open("logs/stderr.txt", "wb") as err: - cmd = copy.copy(self.service_data["sources"]["cmd"]) - cmd[-1] = cmd[-1] + " " + " ".join(sbt_extra_params) - serialised_cmd = json.dumps(cmd) + serialised_cmd = json.dumps(self.self.get_start_command("SOURCE")) Popen(["python", self.sbt_py_filename, serialised_cmd], env=env_copy, stdout=out, stderr=err) seconds_remaining = SmPlayServiceStarter.PLAY_PROCESS_STARTUP_TIMEOUT_SECONDS diff --git a/servicemanager/service/smservice.py b/servicemanager/service/smservice.py index 75d47d5..61e2a8b 100644 --- a/servicemanager/service/smservice.py +++ b/servicemanager/service/smservice.py @@ -66,6 +66,10 @@ def start(self): def process_arguments(self): pass + @abstractmethod + def get_start_command(self, context = None): + return ["get_start_command() not implemented for this type of service - fork and make a pull request :)"] + class SmMicroServiceStarter(SmServiceStarter): diff --git a/servicemanager/smcontext.py b/servicemanager/smcontext.py index bf00431..7f44bc7 100755 --- a/servicemanager/smcontext.py +++ b/servicemanager/smcontext.py @@ -312,15 +312,10 @@ def get_run_from_service_override_value_or_use_default(self, service, original_r return service.service_data["always_run_from"] return original_runfrom - def start_service(self, service_name, run_from, proxy, classifier=None, service_mapping_ports=None, port=None, admin_port=None, version=None): - + def get_service_starter(self, service_name, run_from, proxy, classifier=None, service_mapping_ports=None, port=None, admin_port=None, version=None): service = self.get_service(service_name) run_from = self.get_run_from_service_override_value_or_use_default(service, run_from) - feature_string = pretty_print_list(" with feature$s $list enabled", self.features) - - self.log("Starting '%s' from %s%s..." % (service_name, run_from, feature_string)) - if not service_mapping_ports: service_mapping_ports = {} @@ -341,7 +336,13 @@ def start_service(self, service_name, run_from, proxy, classifier=None, service_ else: raise self.exception("Unknown service type '%s' for service '%s' - please check services.json" % (service_type, service_name)) - service_process_id = starter.start() + return starter + + def start_service(self, service_name, run_from, proxy, classifier=None, service_mapping_ports=None, port=None, admin_port=None, version=None): + service_starter = self.get_service_starter(service_name, run_from, proxy, classifier, service_mapping_ports, port, admin_port, version) + feature_string = pretty_print_list(" with feature$s $list enabled", self.features) + self.log("Starting '%s' from %s%s..." % (service_name, run_from, feature_string)) + service_process_id = service_starter.start() if service_process_id: feature_string = pretty_print_list(" and feature$s $list enabled", self.features) diff --git a/servicemanager/smprocess.py b/servicemanager/smprocess.py index e591be5..ed60712 100644 --- a/servicemanager/smprocess.py +++ b/servicemanager/smprocess.py @@ -43,6 +43,9 @@ def _is_system_or_smserver_or_test_process(pid): if _is_pycharm_test_process(pid): return True + if _is_pycharm_related_process(pid): + return True + return False @@ -95,6 +98,15 @@ def _is_pycharm_process(pid): return True return False +def _is_pycharm_related_process(pid): + command = "ps -eo pid,args | grep %d | grep 'pycharm' | awk '{print $1}'" % pid + ps_command = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) + ps_output = ps_command.stdout.read() + ps_pid_str = ps_output.strip() + if ps_pid_str and int(ps_pid_str) == pid: + return True + return False + def kill_by_test_id(context, force): pids = _get_process_ids_for_test(context) diff --git a/test/tests.py b/test/tests.py index cd209c7..5248908 100644 --- a/test/tests.py +++ b/test/tests.py @@ -12,6 +12,7 @@ import shutil import unittest import smcontext +from servicemanager.service.smplayservice import SmPlayServiceStarter from servicemanager.serviceresolver import ServiceResolver @@ -143,10 +144,12 @@ def test_failing_play_from_jar(self): service_resolver = ServiceResolver(sm_application) context.kill_everything() + time.sleep(5) response1 = actions.start_one(context, "FAKE_NEXUS", True, False, None, port=None) self.assertTrue(response1) self.assertIsNotNone(context.get_service("FAKE_NEXUS").status()) + time.sleep(5) try: servicetostart = ["BROKEN_PLAY_PROJECT"] @@ -203,6 +206,93 @@ def test_python_server_offline(self): time.sleep(5) self.assertEqual(context.get_service("PYTHON_SIMPLE_SERVER_ASSETS_FRONTEND").status(), []) +class TestStartCommands(unittest.TestCase): + + def setUp(self): + set_up_and_clean_workspace() + + def test_play_binary_config(self): + config_dir_override = os.path.join(os.path.dirname(__file__), "conf") + context = smcontext.SmContext(smcontext.SmApplication(config_dir_override), None, False, False) + starter = context.get_service_starter("PLAY_NEXUS_END_TO_END_TEST", True, False, None, port=None) + #starter = SmPlayServiceStarter(context, "PLAY_NEXUS_END_TO_END_TEST", True, False, None, None, None, None) + expected = [ './basicplayapp/bin/basicplayapp', + '-DProd.microservice.whitelist.useWhitelist=false', + '-DProd.mongodb.uri=mongodb://localhost:27017/auth', + '-J-Xmx256m', + '-J-Xms256m', + '-J-XX:MaxPermSize=128m', + '-Dhttp.port=8500', + '-Dservice.manager.serviceName=PLAY_NEXUS_END_TO_END_TEST', + '-Dservice.manager.runFrom=True'] + self.assertEqual(starter.get_start_command("BINARY"), expected) + + def test_play_source_config(self): + config_dir_override = os.path.join(os.path.dirname(__file__), "conf") + context = smcontext.SmContext(smcontext.SmApplication(config_dir_override), None, False, False) + starter = context.get_service_starter("PLAY_NEXUS_END_TO_END_TEST", True, False, None, port=None) + expected = [ 'play', 'start -Dhttp.port=8500 -Dservice.manager.serviceName=PLAY_NEXUS_END_TO_END_TEST -Dservice.manager.runFrom=True -DFoo=false'] + self.assertEqual(starter.get_start_command("SOURCE"), expected) + + def test_dropwizard_binary_config(self): + config_dir_override = os.path.join(os.path.dirname(__file__), "conf") + context = smcontext.SmContext(smcontext.SmApplication(config_dir_override), None, False, False) + starter = context.get_service_starter("DROPWIZARD_NEXUS_END_TO_END_TEST", "foo", proxy=None) + expected = [ + '/Library/Java/JavaVirtualMachines/jdk1.7.0_45.jdk/Contents/Home/bin/java', + '-Dfile.encoding=UTF8', + '-Xmx64M', + '-XX:+CMSClassUnloadingEnabled', + '-XX:MaxPermSize=64m', + '-Ddw.http.port=8080', + '-Dservice.manager.serviceName=DROPWIZARD_NEXUS_END_TO_END_TEST', + '-Dservice.manager.serviceName=DROPWIZARD_NEXUS_END_TO_END_TEST', + '-Dservice.manager.runFrom=foo', + '-jar', + 'dwtest-foo-shaded.jar', + 'server', + 'dev_config.yml'] + cmd = starter.get_start_command("BINARY") + cmd[-1] = cmd[-1].split("/")[-1] + cmd[len(cmd) -3] = cmd[len(cmd) -3].split("/")[-1] + self.assertEqual(cmd, expected) + + def test_dropwizard_source_config(self): + config_dir_override = os.path.join(os.path.dirname(__file__), "conf") + context = smcontext.SmContext(smcontext.SmApplication(config_dir_override), None, False, False) + starter = context.get_service_starter("DROPWIZARD_NEXUS_END_TO_END_TEST", "foo", proxy=None) + expected = ['./startappfromcode.sh'] + self.assertEqual(starter.get_start_command("SOURCE"), expected) + + def test_python_binary_config(self): + config_dir_override = os.path.join(os.path.dirname(__file__), "conf") + context = smcontext.SmContext(smcontext.SmApplication(config_dir_override), None, False, False) + starter = context.get_service_starter("PYTHON_SIMPLE_SERVER_ASSETS_FRONTEND", "foo", proxy=None) + expected = [ 'get_start_command() not implemented for this type of service - fork and make a pull request :)' ] + cmd = starter.get_start_command("BINARY") + self.assertEqual(cmd, expected) + + def test_python_source_config(self): + config_dir_override = os.path.join(os.path.dirname(__file__), "conf") + context = smcontext.SmContext(smcontext.SmApplication(config_dir_override), None, False, False) + starter = context.get_service_starter("PYTHON_SIMPLE_SERVER_ASSETS_FRONTEND", "foo", proxy=None) + expected = ['get_start_command() not implemented for this type of service - fork and make a pull request :)'] + self.assertEqual(starter.get_start_command("SOURCE"), expected) + + def test_external_binary_config(self): + config_dir_override = os.path.join(os.path.dirname(__file__), "conf") + context = smcontext.SmContext(smcontext.SmApplication(config_dir_override), None, False, False) + starter = context.get_service_starter("FAKE_NEXUS", "foo", proxy=None) + expected = [ 'get_start_command() not implemented for this type of service - fork and make a pull request :)' ] + cmd = starter.get_start_command("BINARY") + self.assertEqual(cmd, expected) + + def test_external_source_config(self): + config_dir_override = os.path.join(os.path.dirname(__file__), "conf") + context = smcontext.SmContext(smcontext.SmApplication(config_dir_override), None, False, False) + starter = context.get_service_starter("FAKE_NEXUS", "foo", proxy=None) + expected = ['get_start_command() not implemented for this type of service - fork and make a pull request :)'] + self.assertEqual(starter.get_start_command("SOURCE"), expected) class TestServerFunctionality(unittest.TestCase): def setUp(self):