Merge pull request #3 from hmrc/SHOW_START_CMDS

Show start cmds

bin/sm

@@ -10,7 +10,7 @@ from servicemanager.smstatus import dostatus
 from servicemanager.smcontext import SmApplication, SmContext, ServiceManagerException
 from servicemanager.smrepo import pull_rebase_repo
 from servicemanager.actions import actions
-import servicemanager.smrepo
+from servicemanager import smrepo
 
 def _process_command():
 
@@ -37,6 +37,7 @@ def _process_command():
     parser.add_argument('--printconfig', type=str, nargs='*', help='Print the config for a given list of services, if empty, show all')
     parser.add_argument('-c', '--config', type=str, help='Sets the configuration directory location, defaults to $WORKSPACE/service-manager-config')
     parser.add_argument('--getdascode', action='store_true', help='Checkout all of the code if you haven\'t already')
+    parser.add_argument('--showcmdfor', type=str, nargs='*', help='Shows how sm will try start the service')
 
     args = parser.parse_args()
 
@@ -122,6 +123,17 @@ def _process_command():
                 path = context.application.workspace + service_data["location"]
                 smrepo.clone_repo_if_required_raw(service_name, service_data["sources"]["repo"], path, context)
 
+    if args.showcmdfor:
+        print "The parameters you provided would start the following services using the following commands:"
+        for service_name in service_resolver.resolve_services_from_array(args.showcmdfor):
+            if context.has_service(service_name):
+                cmd = actions.get_start_cmd(context, service_name, args.fatjar, args.release, args.proxy, actions.overridden_port(args.showcmdfor, args.port))
+                print service_name
+                print " ".join(cmd)
+            else:
+                print "The requested service %s does not exist" % service_name
+
+
 try:
     _process_command()
 except ServiceManagerException as e:

servicemanager/actions/actions.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 import os
-import sys
 import time
 import calendar
 import glob
@@ -30,6 +29,16 @@ def start_one(context, service_name, fatjar, release, proxy, port=None):
 
     return False
 
+def get_start_cmd(context, service_name, fatjar, release, proxy, port=None):
+    if release:
+        run_from = "RELEASE"
+    elif fatjar:
+        run_from = "SNAPSHOT"
+    else:
+        run_from = "SOURCE"
+
+    starter = context.get_service_starter(service_name, run_from, proxy, None, None, port)
+    return starter.get_start_command(run_from)
 
 def stop_profile(context, profile):
     for service_name in context.application.services_for_profile(profile):
@@ -127,7 +136,6 @@ def _get_git_rev(context, service_name):
 
     return ""
 
-
 def display_info(context, service_name):
     arguments = _get_running_process_args(context, service_name)
     git_revision = _get_git_rev(context, service_name)

servicemanager/service/sbt-play-wrapper.py

@@ -5,7 +5,6 @@
 
 from servicemanager import subprocess
 
-
 try:
     args = json.loads(sys.argv[1])
     print u"Executing: %s" % " ".join(args)

servicemanager/service/smdropwizardservice.py

@@ -33,6 +33,30 @@ def __init__(self, context, service_name, run_from, port, admin_port, classifier
     def _get_jar_filename(self):
         return self.context.get_jar_filename(self.service_name, self.run_from)
 
+    def _get_binary_start_cmd(self):
+        binary_data = self.service_data["binary"]
+        filename = self._get_jar_filename()
+
+        if "configurationFile" not in binary_data:
+            print "ERROR: required config 'configurationFile' does not exist"
+            return None
+
+        configuration_file = binary_data["configurationFile"]
+        output_configuration_file = os.path.join(self.run_from, configuration_file)
+        microservice_target_path = self.context.get_microservice_target_path(self.service_name)
+        _java_bin = os.path.join(self.java_home, os.path.join("bin", "java"))
+        extra_params = self._build_extra_params()
+        cmd = [_java_bin] + self.java_options + extra_params
+        os.path.join(microservice_target_path, filename)
+        return cmd + ["-jar",  microservice_target_path + filename, "server", microservice_target_path + output_configuration_file]
+
+
+    def get_start_command(self, run_from):
+        if run_from == "SOURCE":
+            return self.sources["cmd"]
+        else:
+            return self._get_binary_start_cmd()
+
     def start_from_binary(self):
 
         microservice_target_path = self.context.get_microservice_target_path(self.service_name)
@@ -53,23 +77,12 @@ def start_from_binary(self):
             print "ERROR: required config 'configurationFile' does not exist"
             return None
 
-        output_configuration_file = os.path.join(self.run_from, configuration_file)
-
         binary_to_run = os.path.join(microservice_target_path, filename)
         if os.path.exists(binary_to_run):
             force_chdir(self.run_from)
             os.system("jar xvf " + microservice_target_path + filename + " " + configuration_file + " > /dev/null")
             force_chdir(microservice_target_path)
-
-            _java_bin = os.path.join(self.java_home, os.path.join("bin", "java"))
-
-            extra_params = self._build_extra_params()
-
-            cmd = [_java_bin] + self.java_options + extra_params
-
-            os.path.join(microservice_target_path, filename)
-
-            cmd = cmd + ["-jar",  microservice_target_path + filename, "server", microservice_target_path + output_configuration_file]
+            cmd = self.get_start_command("BINARY")
             process = Popen(cmd, env=os.environ.copy(), stdout=SmDropwizardServiceStarter.DEV_NULL, stderr=SmDropwizardServiceStarter.DEV_NULL, close_fds=True)
             if process.returncode == 1:
                 print b.fail + "ERROR: could not start '" + self.service_name + "' " + b.endc
@@ -117,7 +130,7 @@ def start_from_sources(self):
             # SBT is so specific should this be in configuration?
             new_env["SBT_EXTRA_PARAMS"] = " ".join(sbt_extra_params)
             os.chdir(base_dir)
-            process = Popen(cmd, env=new_env, stdout=SmDropwizardServiceStarter.DEV_NULL, stderr=SmDropwizardServiceStarter.DEV_NULL, close_fds=True)
+            process = Popen(self.get_start_command("SOURCE"), env=new_env, stdout=SmDropwizardServiceStarter.DEV_NULL, stderr=SmDropwizardServiceStarter.DEV_NULL, close_fds=True)
         except Exception, e:
             self.log("Could not start service in '%s' due to exception: %s" % (base_dir, str(e)))
 

servicemanager/service/smplayservice.py

@@ -65,6 +65,14 @@ def _build_extra_params(self):
 
         return extra_params
 
+    def get_start_command(self, run_from):
+        if run_from == "SOURCE":
+            source_cmd = copy.copy(self.service_data["sources"]["cmd"])
+            source_cmd[-1] = source_cmd[-1] + " " +  " ".join(self.sbt_extra_params())
+            return source_cmd
+        else:
+            return self.service_data["binary"]["cmd"] + self._build_extra_params()
+
     def start_from_binary(self):
         microservice_target_path = self.context.get_microservice_target_path(self.service_name)
         force_chdir(microservice_target_path)
@@ -77,7 +85,7 @@ def start_from_binary(self):
         parent, _ = os.path.split(unzip_dir)
         force_pushdir(parent)
 
-        cmd_with_params = self.service_data["binary"]["cmd"] + self._build_extra_params()
+        cmd_with_params = self.get_start_command("BINARY")
         if os.path.exists(cmd_with_params[0]):
             os.chmod(cmd_with_params[0], stat.S_IRWXU)
         else:
@@ -111,26 +119,29 @@ def _unzip_play_application(self):
 
         return target_dir
 
-    def start_from_sources(self):
+    def sbt_extra_params(self):
         sbt_extra_params = self._build_extra_params()
 
         if "extra_params" in self.service_data["sources"]:
             sbt_extra_params += self.service_data["sources"]["extra_params"]
 
+        return sbt_extra_params
+
+    def start_from_sources(self):
+        sbt_extra_params = self.sbt_extra_params()
+
         service_data = self.context.service_data(self.service_name)
         microservice_path = self.context.application.workspace + service_data["location"]
         curr_dir = force_pushdir(microservice_path)
         remove_if_exists("RUNNING_PID")
 
         env_copy = os.environ.copy()
-        env_copy["SBT_EXTRA_PARAMS"] = " ".join(sbt_extra_params)
+        env_copy["SBT_EXTRA_PARAMS"] = " ".join(sbt_extra_params) # TODO: not needed i think anymore...
 
         makedirs_if_not_exists("logs")
 
         with open("logs/stdout.txt", "wb") as out, open("logs/stderr.txt", "wb") as err:
-            cmd = copy.copy(self.service_data["sources"]["cmd"])
-            cmd[-1] = cmd[-1] + " " +  " ".join(sbt_extra_params)
-            serialised_cmd = json.dumps(cmd)
+            serialised_cmd = json.dumps(self.self.get_start_command("SOURCE"))
             Popen(["python", self.sbt_py_filename, serialised_cmd], env=env_copy, stdout=out, stderr=err)
 
         seconds_remaining = SmPlayServiceStarter.PLAY_PROCESS_STARTUP_TIMEOUT_SECONDS

servicemanager/service/smservice.py

@@ -66,6 +66,10 @@ def start(self):
     def process_arguments(self):
         pass
 
+    @abstractmethod
+    def get_start_command(self, context = None):
+        return ["get_start_command() not implemented for this type of service - fork and make a pull request :)"]
+
 
 class SmMicroServiceStarter(SmServiceStarter):
 

servicemanager/smcontext.py

@@ -312,15 +312,10 @@ def get_run_from_service_override_value_or_use_default(self, service, original_r
                 return service.service_data["always_run_from"]
         return original_runfrom
 
-    def start_service(self, service_name, run_from, proxy, classifier=None, service_mapping_ports=None, port=None, admin_port=None, version=None):
-
+    def get_service_starter(self, service_name, run_from, proxy, classifier=None, service_mapping_ports=None, port=None, admin_port=None, version=None):
         service = self.get_service(service_name)
         run_from = self.get_run_from_service_override_value_or_use_default(service, run_from)
 
-        feature_string = pretty_print_list(" with feature$s $list enabled", self.features)
-
-        self.log("Starting '%s' from %s%s..." % (service_name, run_from, feature_string))
-
         if not service_mapping_ports:
             service_mapping_ports = {}
 
@@ -341,7 +336,13 @@ def start_service(self, service_name, run_from, proxy, classifier=None, service_
         else:
             raise self.exception("Unknown service type '%s' for service '%s' - please check services.json" % (service_type, service_name))
 
-        service_process_id = starter.start()
+        return starter
+
+    def start_service(self, service_name, run_from, proxy, classifier=None, service_mapping_ports=None, port=None, admin_port=None, version=None):
+        service_starter = self.get_service_starter(service_name, run_from, proxy, classifier, service_mapping_ports, port, admin_port, version)
+        feature_string = pretty_print_list(" with feature$s $list enabled", self.features)
+        self.log("Starting '%s' from %s%s..." % (service_name, run_from, feature_string))
+        service_process_id = service_starter.start()
 
         if service_process_id:
             feature_string = pretty_print_list(" and feature$s $list enabled", self.features)

servicemanager/smprocess.py

@@ -43,6 +43,9 @@ def _is_system_or_smserver_or_test_process(pid):
     if _is_pycharm_test_process(pid):
         return True
 
+    if _is_pycharm_related_process(pid):
+        return True
+
     return False
 
 
@@ -95,6 +98,15 @@ def _is_pycharm_process(pid):
         return True
     return False
 
+def _is_pycharm_related_process(pid):
+    command = "ps -eo pid,args | grep %d | grep 'pycharm' | awk '{print $1}'" % pid
+    ps_command = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
+    ps_output = ps_command.stdout.read()
+    ps_pid_str = ps_output.strip()
+    if ps_pid_str and int(ps_pid_str) == pid:
+        return True
+    return False
+
 
 def kill_by_test_id(context, force):
     pids = _get_process_ids_for_test(context)

test/tests.py

@@ -12,6 +12,7 @@
 import shutil
 import unittest
 import smcontext
+from servicemanager.service.smplayservice import SmPlayServiceStarter
 
 from servicemanager.serviceresolver import ServiceResolver
 
@@ -143,10 +144,12 @@ def test_failing_play_from_jar(self):
         service_resolver = ServiceResolver(sm_application)
 
         context.kill_everything()
+        time.sleep(5)
 
         response1 = actions.start_one(context, "FAKE_NEXUS", True, False, None, port=None)
         self.assertTrue(response1)
         self.assertIsNotNone(context.get_service("FAKE_NEXUS").status())
+        time.sleep(5)
 
         try:
             servicetostart = ["BROKEN_PLAY_PROJECT"]
@@ -203,6 +206,93 @@ def test_python_server_offline(self):
         time.sleep(5)
         self.assertEqual(context.get_service("PYTHON_SIMPLE_SERVER_ASSETS_FRONTEND").status(), [])
 
+class TestStartCommands(unittest.TestCase):
+
+    def setUp(self):
+        set_up_and_clean_workspace()
+
+    def test_play_binary_config(self):
+        config_dir_override = os.path.join(os.path.dirname(__file__), "conf")
+        context = smcontext.SmContext(smcontext.SmApplication(config_dir_override), None, False, False)
+        starter = context.get_service_starter("PLAY_NEXUS_END_TO_END_TEST", True, False, None, port=None)
+        #starter = SmPlayServiceStarter(context, "PLAY_NEXUS_END_TO_END_TEST", True, False, None, None, None, None)
+        expected = [ './basicplayapp/bin/basicplayapp',
+                                '-DProd.microservice.whitelist.useWhitelist=false',
+                                '-DProd.mongodb.uri=mongodb://localhost:27017/auth',
+                                '-J-Xmx256m',
+                                '-J-Xms256m',
+                                '-J-XX:MaxPermSize=128m',
+                                '-Dhttp.port=8500',
+                                '-Dservice.manager.serviceName=PLAY_NEXUS_END_TO_END_TEST',
+                                '-Dservice.manager.runFrom=True']
+        self.assertEqual(starter.get_start_command("BINARY"), expected)
+
+    def test_play_source_config(self):
+        config_dir_override = os.path.join(os.path.dirname(__file__), "conf")
+        context = smcontext.SmContext(smcontext.SmApplication(config_dir_override), None, False, False)
+        starter = context.get_service_starter("PLAY_NEXUS_END_TO_END_TEST", True, False, None, port=None)
+        expected = [ 'play', 'start -Dhttp.port=8500 -Dservice.manager.serviceName=PLAY_NEXUS_END_TO_END_TEST -Dservice.manager.runFrom=True -DFoo=false']
+        self.assertEqual(starter.get_start_command("SOURCE"), expected)
+
+    def test_dropwizard_binary_config(self):
+        config_dir_override = os.path.join(os.path.dirname(__file__), "conf")
+        context = smcontext.SmContext(smcontext.SmApplication(config_dir_override), None, False, False)
+        starter = context.get_service_starter("DROPWIZARD_NEXUS_END_TO_END_TEST", "foo", proxy=None)
+        expected = [
+            '/Library/Java/JavaVirtualMachines/jdk1.7.0_45.jdk/Contents/Home/bin/java',
+            '-Dfile.encoding=UTF8',
+            '-Xmx64M',
+            '-XX:+CMSClassUnloadingEnabled',
+            '-XX:MaxPermSize=64m',
+            '-Ddw.http.port=8080',
+            '-Dservice.manager.serviceName=DROPWIZARD_NEXUS_END_TO_END_TEST',
+            '-Dservice.manager.serviceName=DROPWIZARD_NEXUS_END_TO_END_TEST',
+            '-Dservice.manager.runFrom=foo',
+            '-jar',
+            'dwtest-foo-shaded.jar',
+            'server',
+            'dev_config.yml']
+        cmd = starter.get_start_command("BINARY")
+        cmd[-1] = cmd[-1].split("/")[-1]
+        cmd[len(cmd) -3] = cmd[len(cmd) -3].split("/")[-1]
+        self.assertEqual(cmd, expected)
+
+    def test_dropwizard_source_config(self):
+        config_dir_override = os.path.join(os.path.dirname(__file__), "conf")
+        context = smcontext.SmContext(smcontext.SmApplication(config_dir_override), None, False, False)
+        starter = context.get_service_starter("DROPWIZARD_NEXUS_END_TO_END_TEST", "foo", proxy=None)
+        expected = ['./startappfromcode.sh']
+        self.assertEqual(starter.get_start_command("SOURCE"), expected)
+
+    def test_python_binary_config(self):
+        config_dir_override = os.path.join(os.path.dirname(__file__), "conf")
+        context = smcontext.SmContext(smcontext.SmApplication(config_dir_override), None, False, False)
+        starter = context.get_service_starter("PYTHON_SIMPLE_SERVER_ASSETS_FRONTEND", "foo", proxy=None)
+        expected = [ 'get_start_command() not implemented for this type of service - fork and make a pull request :)' ]
+        cmd = starter.get_start_command("BINARY")
+        self.assertEqual(cmd, expected)
+
+    def test_python_source_config(self):
+        config_dir_override = os.path.join(os.path.dirname(__file__), "conf")
+        context = smcontext.SmContext(smcontext.SmApplication(config_dir_override), None, False, False)
+        starter = context.get_service_starter("PYTHON_SIMPLE_SERVER_ASSETS_FRONTEND", "foo", proxy=None)
+        expected = ['get_start_command() not implemented for this type of service - fork and make a pull request :)']
+        self.assertEqual(starter.get_start_command("SOURCE"), expected)
+
+    def test_external_binary_config(self):
+        config_dir_override = os.path.join(os.path.dirname(__file__), "conf")
+        context = smcontext.SmContext(smcontext.SmApplication(config_dir_override), None, False, False)
+        starter = context.get_service_starter("FAKE_NEXUS", "foo", proxy=None)
+        expected = [ 'get_start_command() not implemented for this type of service - fork and make a pull request :)' ]
+        cmd = starter.get_start_command("BINARY")
+        self.assertEqual(cmd, expected)
+
+    def test_external_source_config(self):
+        config_dir_override = os.path.join(os.path.dirname(__file__), "conf")
+        context = smcontext.SmContext(smcontext.SmApplication(config_dir_override), None, False, False)
+        starter = context.get_service_starter("FAKE_NEXUS", "foo", proxy=None)
+        expected = ['get_start_command() not implemented for this type of service - fork and make a pull request :)']
+        self.assertEqual(starter.get_start_command("SOURCE"), expected)
 
 class TestServerFunctionality(unittest.TestCase):
     def setUp(self):