Test TravisCI

.travis.yml

@@ -0,0 +1,20 @@
+language: python
+python:
+  - "2.7"
+before_script:
+  - curl https://raw.githubusercontent.com/jrha/krb5-travis/master/install_krb5 | sudo bash
+  - curl https://raw.githubusercontent.com/jrha/krb5-travis/master/install_knc | bash
+  - wget https://github.com/google/protobuf/releases/download/v3.5.1/protoc-3.5.1-linux-x86_64.zip -O /tmp/protoc.zip && cd /tmp && unzip /tmp/protoc.zip && sudo cp bin/protoc /usr/bin/ && sudo chmod ugo+x /usr/bin/protoc
+  - cd /tmp && git clone https://github.com/quattor/aquilon-protocols.git && cd aquilon-protocols && git checkout upstream && ./setup.py build && sudo ./setup.py install --install-lib /usr/local/lib/aquilon/protocols/lib/python
+  - cd $TRAVIS_BUILD_DIR/tools/bootstrap_ms && python setup.py sdist && pip install dist/*.tar.gz
+  - cd $TRAVIS_BUILD_DIR && cp -R tests/fakemodules/ms/dsdb /home/travis/virtualenv/python$TRAVIS_PYTHON_VERSION/lib/python$TRAVIS_PYTHON_VERSION/site-packages/ms/; echo $?
+  - ls /home/travis/virtualenv/python$TRAVIS_PYTHON_VERSION/lib/python$TRAVIS_PYTHON_VERSION/site-packages/ms/
+script:
+  - cd $TRAVIS_BUILD_DIR && tests/runtests.py -c tests/unittest.conf.travis --assume_yes
+after_script:
+  - tail -n 64 /var/tmp/travis/aqtest/quattor/logs/aqd.log
+addons:
+  hosts:
+    - travis.dev
+  packages:
+    - libkrb5-dev

lib/aquilon/worker/processes.py

@@ -50,17 +50,18 @@
 
 DSDB_ENABLED = config.getboolean("dsdb", "enable")
 
-if DSDB_ENABLED:
-    # FIXME - this needs to be moved to depends.py after
-    # refactoring runtests.py and Config to allow override
-    # sys.path for python modules when running tests
-    # DSDB python client
+try:
     import ms.version
+except ImportError:
+    pass
+else:
+    ms.version.addpkg('setuptools', '0.6c11')
     ms.version.addpkg("requests", "2.7.0")
     ms.version.addpkg("requests-kerberos", "0.5-ms2")
     ms.version.addpkg("kerberos", "1.1.5")
     ms.version.addpkg("dns", "1.10.0")
     ms.version.addpkg('ms.dsdb', '6.0.30')
+if DSDB_ENABLED:
     import ms.dsdb.client
 
 # subprocess.Popen is not thread-safe in Python 2, so we need locking
@@ -1078,4 +1079,4 @@ def build_mako_lookup(config, kind, **kwargs):
     return TemplateLookup(directories=directories, **kwargs)
 
 
-DSDBRunner.snapshot_handlers['rack'] = DSDBRunner.snapshot_rack
+DSDBRunner.snapshot_handlers['rack'] = DSDBRunner.snapshot_rack

requirements.txt

@@ -0,0 +1,12 @@
+Twisted >= 12.2.0
+coverage
+ipaddress
+jsonschema
+lxml
+mako
+psycopg2 >= 2.5.1
+python-dateutil
+six >= 1.7.3
+sqlalchemy >= 0.9.7
+zope-interface
+protobuf >= 3.6.0

tests/broker/test_start.py

@@ -47,6 +47,7 @@ def testclonetemplateking(self):
 
         env = {}
         env["PATH"] = os.environ.get("PATH", "")
+        env["HOME"] = os.environ.get("HOME", "/home/travis")
         git = self.config.lookup_tool("git")
         if source.startswith("git://"):
             # Easy case - just clone the source repository
@@ -93,6 +94,7 @@ def testdisabletemplatetests(self):
         git = self.config.lookup_tool("git")
         env = {}
         env["PATH"] = os.environ.get("PATH", "")
+        env["HOME"] = os.environ.get("HOME", "/home/travis")
 
         tempdir = mkdtemp(prefix="fixup", dir=rundir)
 

tests/runtests.py

@@ -58,13 +58,14 @@
 
 def force_yes(msg):
     print(msg, file=sys.stderr)
-    print("""
-        Please confirm by typing yes (three letters) and pressing enter.
-        """, file=sys.stderr)
-    answer = sys.stdin.readline()
-    if not answer.startswith("yes"):
-        print("""Aborting.""", file=sys.stderr)
-        sys.exit(1)
+    if not opts.assume_yes:
+        print("""
+            Please confirm by typing yes (three letters) and pressing enter.
+            """, file=sys.stderr)
+        answer = sys.stdin.readline()
+        if not answer.startswith("yes"):
+            print("""Aborting.""", file=sys.stderr)
+            sys.exit(1)
 
 parser = argparse.ArgumentParser(description="Run the broker test suite.",
                                  epilog=epilog)
@@ -91,6 +92,8 @@ def force_yes(msg):
 parser.add_argument('-f', '--failfast', action='store_true',
                     help='Add failfast=True option to TestRunner. This will stop '
                          'unittests immediatelly if any failure.')
+parser.add_argument('--assume_yes', action='store_true',
+                    help='Assume yes to all questions, only use for continuous integration')
 
 
 opts = parser.parse_args()

tests/unittest.conf

@@ -69,6 +69,7 @@ checkedm = %(srcdir)s/tests/fakebin/fake_checkedm
 #aqd_checkedm = /ms/dist/aquilon/PROJ/edmtools/dev/bin/aqd_checkedm
 aqd_checkedm = %(srcdir)s/tests/fakebin/fake_aqd_checkedm
 location_uri_validator = /ms/dist/aquilon/PROJ/aqd-scripts/qa/bin/location_uri_validator
+klist = %(srcdir)s/tests/fakebin/fake_klist
 
 [location_feeds]
 building_feed = %(srcdir)s/tests/fakebin/user-data/location_feed.csv

tests/unittest.conf.travis

@@ -0,0 +1,156 @@
+# This file provides the default for the unittests.  To test against
+# an oracle database (or otherwise change the parameters), copy this
+# file and modify as needed.
+#
+# The config file etc/aqd.conf.defaults always gets read first, and the
+# values below override those.  To nullify a value from the defaults,
+# it must be explicitly set to an empty value here.
+
+[DEFAULT]
+basedir = /var/tmp/%(user)s/aqtest
+quattordir = %(basedir)s/quattor
+
+# By default takes the sqlite section from aqd.conf.defaults.
+[database]
+database_section = database_sqlite
+
+[database_sqlite]
+# We do not really care if the host crashes during the unittest...
+disable_fsync = yes
+
+[broker]
+servername = %(hostname)s
+umask = 0022
+kncport = 6902
+openport = 6903
+git_port = 9419
+# Testing the case when bind_address is set is more interesting than when it's not...
+# It would be better to use localhost (to make sure nothing goes out to the wire), but that would make Kerberos unhappy
+bind_address = %(hostname)s
+run_git_daemon = True
+git_author_name = travis
+git_author_email = travis@dev
+git_committer_name = travis
+git_committer_email = travis@dev
+trash_branch = unittest_trash
+dsdb_use_testdb = 1
+server_notifications = utnotify
+client_notifications = False
+sharedata = %(srcdir)s/tests/fakebin/nasobjects/testnasobjects.cdb
+run_knc = True
+grn_to_eonid_map_location = %(srcdir)s/tests/fakebin/eon-data
+esx_cluster_allow_cascaded_deco = True
+default_max_list_size = 1000
+reconfigure_max_list_size = 15
+pxeswitch_max_list_size = 15
+manage_max_list_size = 15
+reset_advertised_status_max_list_size = 15
+map_grn_max_list_size = 15
+unmap_grn_max_list_size = 15
+user_list_location = %(srcdir)s/tests/fakebin/user-data/passwd.byname
+user_delete_limit = 10
+service = aquilon
+keytab = /etc/krb5.keytab
+
+[change_management]
+# The contents of --justification will not be validated externally
+# (syntax checking will still be performed)
+enable = True
+extra_options = --mode warn --disable_edm --edm-instance qa
+
+[tool_locations]
+dsdb = %(srcdir)s/tests/fakebin/fake_dsdb
+aii_installfe = /bin/echo
+vlan2net = %(srcdir)s/tests/fakebin/fake_vlan2net
+mean = %(srcdir)s/tests/fakebin/fake_mean
+qip_dump_subnetdata = %(srcdir)s/tests/fakebin/fake_qip_dump_subnetdata
+ssh = %(srcdir)s/tests/fakebin/fake_ssh
+switch_discover = %(srcdir)s/tests/fakebin/fake_switchdata
+get_camtable = %(srcdir)s/tests/fakebin/fake_macdata
+checkedm = %(srcdir)s/tests/fakebin/fake_checkedm
+aqd_checkedm = %(srcdir)s/tests/fakebin/fake_aqd_checkedm
+location_uri_validator = %(srcdir)s/tests/fakebin/location_uri_validator
+git = /usr/bin/git
+klist = /usr/bin/klist
+knc = /usr/local/bin/knc
+
+# Alternative tool locations outside MS
+git_daemon = /usr/lib/git-core/git-daemon
+ant_contrib_jar = /usr/share/java/ant/ant-contrib.jar
+
+[location_feeds]
+building_feed = %(srcdir)s/tests/fakebin/user-data/location_feed.csv
+
+[unittest]
+last_success_db_snapshot = %(quattordir)s/aquilondb/aquilon_last_success_snapshot.db
+fake_module_location = %(srcdir)s/tests/fakemodules/
+scratchdir = %(basedir)s/scratch
+mirrordir = %(basedir)s/mirror
+
+# Update the template_king_host value
+template_king_host = localhost
+template_base =  %(srcdir)s/tests/templates/
+
+template_alternate_prod =
+fake_hosts_location = %(srcdir)s/tests/fakebin/dsdb.d/show_host_-host_name_
+datadir = %(srcdir)s/tests/broker/data
+
+# Alternative tool locations outside MS
+real_panc_location = /usr/lib/panc.jar
+
+# Versions for the Linux models. These are used both for Aquilon and Aurora.
+linux_version_curr = 6.1-x86_64
+linux_version_prev = 5.1-x86_64
+
+hostname = travis.dev
+
+[archetype_aquilon]
+default_grn_target = esp
+default_osversion = 6.1-x86_64
+default_osname = linux
+host_grn_targets = esp,hlmplus,atarget
+personality_grn_targets = esp,hlmplus,atarget
+
+[archetype_aurora]
+default_domain = unittest
+
+[archetype_f5]
+default_personality = generic
+
+[archetype_filer]
+default_personality = generic
+
+[archetype_metacluster]
+default_domain = unittest
+
+[archetype_hacluster]
+allow_cascaded_deco = False
+max_priority_order = 99
+min_priority_order = 1
+
+[archetype_netinfra]
+# Network devices should be compileable, but we're not quite there yet.
+# Stash them awain in a separate domain for now to avoid compilation
+# errors.
+default_domain = netinfra
+
+[archetype_windows]
+default_domain = ut-prod
+
+[panc]
+pan_compiler = %(basedir)s/panc-links/panc-%(version)s.jar
+include_pan = True
+
+[site]
+# Site specific settings
+default_hardware_label_regex = ^[a-z][a-z0-9]{,62}$
+# Allow Aquilon broker generate nextip for vip/localvip type
+# Temporary option until we can vary API by Aquilon broker instance
+ipfromtype = True
+
+[protocols]
+# Alternative tool locations outside MS
+directory = /usr/local/lib/aquilon/protocols/lib/python
+
+[dsdb]
+enable = False