From d3275b905afec496a668f90bcbee7f754596169f Mon Sep 17 00:00:00 2001 From: James Adams Date: Mon, 4 Jun 2018 13:00:27 +0100 Subject: [PATCH 1/7] Test travis config * Copy HOME from standard environment * Add dependency on protobuf 3.6.0 * Install knc * Install protocols to /usr/local/lib/... * Use locally installed knc * Force service name to be aquilon * Set keytab location Change-Id: I3dabb9e17d9ad4686a19336e121b1aceb925cf0d --- .travis.yml | 18 +++++ requirements.txt | 12 +++ tests/broker/test_start.py | 2 + tests/runtests.py | 17 ++-- tests/unittest.conf | 1 + tests/unittest.conf.travis | 156 +++++++++++++++++++++++++++++++++++++ 6 files changed, 199 insertions(+), 7 deletions(-) create mode 100644 .travis.yml create mode 100644 requirements.txt create mode 100644 tests/unittest.conf.travis diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..a833a8441 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,18 @@ +language: python +python: + - "2.7" +before_script: + - curl https://raw.githubusercontent.com/jrha/krb5-travis/master/install_krb5 | sudo bash + - curl https://raw.githubusercontent.com/jrha/krb5-travis/master/install_knc | bash + - wget https://github.com/google/protobuf/releases/download/v3.5.1/protoc-3.5.1-linux-x86_64.zip -O /tmp/protoc.zip && cd /tmp && unzip /tmp/protoc.zip && sudo cp bin/protoc /usr/bin/ && sudo chmod ugo+x /usr/bin/protoc + - cd /tmp && git clone https://github.com/quattor/aquilon-protocols.git && cd aquilon-protocols && git checkout upstream && ./setup.py build && sudo ./setup.py install --install-lib /usr/local/lib/aquilon/protocols/lib/python + - cd $TRAVIS_BUILD_DIR/tools/bootstrap_ms && python setup.py install +script: + - cd $TRAVIS_BUILD_DIR && tests/runtests.py -c tests/unittest.conf.travis -f --assume_yes +after_script: + - tail -n 64 /var/tmp/travis/aqtest/quattor/logs/aqd.log +addons: + hosts: + - travis.dev + packages: + - libkrb5-dev diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..d22e1b8f6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +Twisted >= 12.2.0 +coverage +ipaddress +jsonschema +lxml +mako +psycopg2 >= 2.5.1 +python-dateutil +six >= 1.7.3 +sqlalchemy >= 0.9.7 +zope-interface +protobuf >= 3.6.0 diff --git a/tests/broker/test_start.py b/tests/broker/test_start.py index 92b216b0f..c75994270 100755 --- a/tests/broker/test_start.py +++ b/tests/broker/test_start.py @@ -47,6 +47,7 @@ def testclonetemplateking(self): env = {} env["PATH"] = os.environ.get("PATH", "") + env["HOME"] = os.environ.get("HOME", "/home/travis") git = self.config.lookup_tool("git") if source.startswith("git://"): # Easy case - just clone the source repository @@ -93,6 +94,7 @@ def testdisabletemplatetests(self): git = self.config.lookup_tool("git") env = {} env["PATH"] = os.environ.get("PATH", "") + env["HOME"] = os.environ.get("HOME", "/home/travis") tempdir = mkdtemp(prefix="fixup", dir=rundir) diff --git a/tests/runtests.py b/tests/runtests.py index 3bf79e9fd..5224ca360 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -58,13 +58,14 @@ def force_yes(msg): print(msg, file=sys.stderr) - print(""" - Please confirm by typing yes (three letters) and pressing enter. - """, file=sys.stderr) - answer = sys.stdin.readline() - if not answer.startswith("yes"): - print("""Aborting.""", file=sys.stderr) - sys.exit(1) + if not opts.assume_yes: + print(""" + Please confirm by typing yes (three letters) and pressing enter. + """, file=sys.stderr) + answer = sys.stdin.readline() + if not answer.startswith("yes"): + print("""Aborting.""", file=sys.stderr) + sys.exit(1) parser = argparse.ArgumentParser(description="Run the broker test suite.", epilog=epilog) @@ -91,6 +92,8 @@ def force_yes(msg): parser.add_argument('-f', '--failfast', action='store_true', help='Add failfast=True option to TestRunner. This will stop ' 'unittests immediatelly if any failure.') +parser.add_argument('--assume_yes', action='store_true', + help='Assume yes to all questions, only use for continuous integration') opts = parser.parse_args() diff --git a/tests/unittest.conf b/tests/unittest.conf index 56f16b0ab..42c4b9c3b 100644 --- a/tests/unittest.conf +++ b/tests/unittest.conf @@ -69,6 +69,7 @@ checkedm = %(srcdir)s/tests/fakebin/fake_checkedm #aqd_checkedm = /ms/dist/aquilon/PROJ/edmtools/dev/bin/aqd_checkedm aqd_checkedm = %(srcdir)s/tests/fakebin/fake_aqd_checkedm location_uri_validator = /ms/dist/aquilon/PROJ/aqd-scripts/qa/bin/location_uri_validator +klist = %(srcdir)s/tests/fakebin/fake_klist [location_feeds] building_feed = %(srcdir)s/tests/fakebin/user-data/location_feed.csv diff --git a/tests/unittest.conf.travis b/tests/unittest.conf.travis new file mode 100644 index 000000000..4a8bca619 --- /dev/null +++ b/tests/unittest.conf.travis @@ -0,0 +1,156 @@ +# This file provides the default for the unittests. To test against +# an oracle database (or otherwise change the parameters), copy this +# file and modify as needed. +# +# The config file etc/aqd.conf.defaults always gets read first, and the +# values below override those. To nullify a value from the defaults, +# it must be explicitly set to an empty value here. + +[DEFAULT] +basedir = /var/tmp/%(user)s/aqtest +quattordir = %(basedir)s/quattor + +# By default takes the sqlite section from aqd.conf.defaults. +[database] +database_section = database_sqlite + +[database_sqlite] +# We do not really care if the host crashes during the unittest... +disable_fsync = yes + +[broker] +servername = %(hostname)s +umask = 0022 +kncport = 6902 +openport = 6903 +git_port = 9419 +# Testing the case when bind_address is set is more interesting than when it's not... +# It would be better to use localhost (to make sure nothing goes out to the wire), but that would make Kerberos unhappy +bind_address = %(hostname)s +run_git_daemon = True +git_author_name = travis +git_author_email = travis@dev +git_committer_name = travis +git_committer_email = travis@dev +trash_branch = unittest_trash +dsdb_use_testdb = 1 +server_notifications = utnotify +client_notifications = False +sharedata = %(srcdir)s/tests/fakebin/nasobjects/testnasobjects.cdb +run_knc = True +grn_to_eonid_map_location = %(srcdir)s/tests/fakebin/eon-data +esx_cluster_allow_cascaded_deco = True +default_max_list_size = 1000 +reconfigure_max_list_size = 15 +pxeswitch_max_list_size = 15 +manage_max_list_size = 15 +reset_advertised_status_max_list_size = 15 +map_grn_max_list_size = 15 +unmap_grn_max_list_size = 15 +user_list_location = %(srcdir)s/tests/fakebin/user-data/passwd.byname +user_delete_limit = 10 +service = aquilon +keytab = /etc/krb5.keytab + +[change_management] +# The contents of --justification will not be validated externally +# (syntax checking will still be performed) +enable = True +extra_options = --mode warn --disable_edm --edm-instance qa + +[tool_locations] +dsdb = %(srcdir)s/tests/fakebin/fake_dsdb +aii_installfe = /bin/echo +vlan2net = %(srcdir)s/tests/fakebin/fake_vlan2net +mean = %(srcdir)s/tests/fakebin/fake_mean +qip_dump_subnetdata = %(srcdir)s/tests/fakebin/fake_qip_dump_subnetdata +ssh = %(srcdir)s/tests/fakebin/fake_ssh +switch_discover = %(srcdir)s/tests/fakebin/fake_switchdata +get_camtable = %(srcdir)s/tests/fakebin/fake_macdata +checkedm = %(srcdir)s/tests/fakebin/fake_checkedm +aqd_checkedm = %(srcdir)s/tests/fakebin/fake_aqd_checkedm +location_uri_validator = %(srcdir)s/tests/fakebin/location_uri_validator +git = /usr/bin/git +klist = /usr/bin/klist +knc = /usr/local/bin/knc + +# Alternative tool locations outside MS +git_daemon = /usr/lib/git-core/git-daemon +ant_contrib_jar = /usr/share/java/ant/ant-contrib.jar + +[location_feeds] +building_feed = %(srcdir)s/tests/fakebin/user-data/location_feed.csv + +[unittest] +last_success_db_snapshot = %(quattordir)s/aquilondb/aquilon_last_success_snapshot.db +fake_module_location = %(srcdir)s/tests/fakemodules/ +scratchdir = %(basedir)s/scratch +mirrordir = %(basedir)s/mirror + +# Update the template_king_host value +template_king_host = localhost +template_base = %(srcdir)s/tests/templates/ + +template_alternate_prod = +fake_hosts_location = %(srcdir)s/tests/fakebin/dsdb.d/show_host_-host_name_ +datadir = %(srcdir)s/tests/broker/data + +# Alternative tool locations outside MS +real_panc_location = /usr/lib/panc.jar + +# Versions for the Linux models. These are used both for Aquilon and Aurora. +linux_version_curr = 6.1-x86_64 +linux_version_prev = 5.1-x86_64 + +hostname = travis.dev + +[archetype_aquilon] +default_grn_target = esp +default_osversion = 6.1-x86_64 +default_osname = linux +host_grn_targets = esp,hlmplus,atarget +personality_grn_targets = esp,hlmplus,atarget + +[archetype_aurora] +default_domain = unittest + +[archetype_f5] +default_personality = generic + +[archetype_filer] +default_personality = generic + +[archetype_metacluster] +default_domain = unittest + +[archetype_hacluster] +allow_cascaded_deco = False +max_priority_order = 99 +min_priority_order = 1 + +[archetype_netinfra] +# Network devices should be compileable, but we're not quite there yet. +# Stash them awain in a separate domain for now to avoid compilation +# errors. +default_domain = netinfra + +[archetype_windows] +default_domain = ut-prod + +[panc] +pan_compiler = %(basedir)s/panc-links/panc-%(version)s.jar +include_pan = True + +[site] +# Site specific settings +default_hardware_label_regex = ^[a-z][a-z0-9]{,62}$ +# Allow Aquilon broker generate nextip for vip/localvip type +# Temporary option until we can vary API by Aquilon broker instance +ipfromtype = True + +[protocols] +# Alternative tool locations outside MS +directory = /usr/local/lib/aquilon/protocols/lib/python + +[dsdb] +enable = False From 185c8e7b539253d719e9d63f3859f0532e7f3444 Mon Sep 17 00:00:00 2001 From: James Adams Date: Mon, 25 Jun 2018 13:45:10 +0100 Subject: [PATCH 2/7] Don't fail fast Let's see how far we get --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a833a8441..12254e482 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ before_script: - cd /tmp && git clone https://github.com/quattor/aquilon-protocols.git && cd aquilon-protocols && git checkout upstream && ./setup.py build && sudo ./setup.py install --install-lib /usr/local/lib/aquilon/protocols/lib/python - cd $TRAVIS_BUILD_DIR/tools/bootstrap_ms && python setup.py install script: - - cd $TRAVIS_BUILD_DIR && tests/runtests.py -c tests/unittest.conf.travis -f --assume_yes + - cd $TRAVIS_BUILD_DIR && tests/runtests.py -c tests/unittest.conf.travis --assume_yes after_script: - tail -n 64 /var/tmp/travis/aqtest/quattor/logs/aqd.log addons: From 59ece30e1c2640a01f369863b2aabc8fa04da15c Mon Sep 17 00:00:00 2001 From: James Adams Date: Thu, 28 Jun 2018 11:53:11 +0100 Subject: [PATCH 3/7] Catch import errors as per @urbonegi's suggestion Change-Id: I9c010847d3b14e10d8c2f7946a3164f44011c9d0 --- lib/aquilon/worker/processes.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/aquilon/worker/processes.py b/lib/aquilon/worker/processes.py index b217c570a..70c5b62db 100644 --- a/lib/aquilon/worker/processes.py +++ b/lib/aquilon/worker/processes.py @@ -50,17 +50,18 @@ DSDB_ENABLED = config.getboolean("dsdb", "enable") -if DSDB_ENABLED: - # FIXME - this needs to be moved to depends.py after - # refactoring runtests.py and Config to allow override - # sys.path for python modules when running tests - # DSDB python client +try: import ms.version +except ImportError: + pass +else: + ms.version.addpkg('setuptools', '0.6c11') ms.version.addpkg("requests", "2.7.0") ms.version.addpkg("requests-kerberos", "0.5-ms2") ms.version.addpkg("kerberos", "1.1.5") ms.version.addpkg("dns", "1.10.0") ms.version.addpkg('ms.dsdb', '6.0.30') +if DSDB_ENABLED: import ms.dsdb.client # subprocess.Popen is not thread-safe in Python 2, so we need locking @@ -1078,4 +1079,4 @@ def build_mako_lookup(config, kind, **kwargs): return TemplateLookup(directories=directories, **kwargs) -DSDBRunner.snapshot_handlers['rack'] = DSDBRunner.snapshot_rack \ No newline at end of file +DSDBRunner.snapshot_handlers['rack'] = DSDBRunner.snapshot_rack From 1e8431769c14437a77b25e4f8c11ea7e5858b0b4 Mon Sep 17 00:00:00 2001 From: James Adams Date: Wed, 25 Jul 2018 15:16:37 +0100 Subject: [PATCH 4/7] Inspect virtualenv Change-Id: I193a4a302c71195ef4b53c060e3ef782beacfe5f --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 12254e482..b487925e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,9 @@ before_script: - curl https://raw.githubusercontent.com/jrha/krb5-travis/master/install_knc | bash - wget https://github.com/google/protobuf/releases/download/v3.5.1/protoc-3.5.1-linux-x86_64.zip -O /tmp/protoc.zip && cd /tmp && unzip /tmp/protoc.zip && sudo cp bin/protoc /usr/bin/ && sudo chmod ugo+x /usr/bin/protoc - cd /tmp && git clone https://github.com/quattor/aquilon-protocols.git && cd aquilon-protocols && git checkout upstream && ./setup.py build && sudo ./setup.py install --install-lib /usr/local/lib/aquilon/protocols/lib/python - - cd $TRAVIS_BUILD_DIR/tools/bootstrap_ms && python setup.py install + - cd $TRAVIS_BUILD_DIR/tools/bootstrap_ms && python setup.py sdist && pip install dist/*.tar.gz + - cd $TRAVIS_BUILD_DIR && cp -R tests/fakemodules/ms/dsdb /home/travis/virtualenv/python$TRAVIS_PYTHON_VERSION/lib/python$TRAVIS_PYTHON_VERSION/site-packages/ms/; echo $? + - ls /home/travis/virtualenv/python$TRAVIS_PYTHON_VERSION/lib/python$TRAVIS_PYTHON_VERSION/site-packages/ms/ script: - cd $TRAVIS_BUILD_DIR && tests/runtests.py -c tests/unittest.conf.travis --assume_yes after_script: From c0066e8b6df0cb887775fab6e53fdda26af62393 Mon Sep 17 00:00:00 2001 From: James Adams Date: Thu, 26 Jul 2018 09:18:58 +0100 Subject: [PATCH 5/7] Inspect scratch directory and fake DSDB logs Change-Id: I05db6b3f2ea1743d68753ef4d8b4a743588d77a6 --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b487925e1..494605637 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,10 @@ before_script: script: - cd $TRAVIS_BUILD_DIR && tests/runtests.py -c tests/unittest.conf.travis --assume_yes after_script: - - tail -n 64 /var/tmp/travis/aqtest/quattor/logs/aqd.log + - tail -n 128 /var/tmp/travis/aqtest/quattor/logs/aqd.log + - ls /var/tmp/travis/aqtest/scratch + - ls /var/tmp/travis/aqtest/scratch/dsdb_coverage + - tail -n 128 /var/tmp/travis/aqtest/scratch/dsdb_coverage/issued_dsdb_cmds addons: hosts: - travis.dev From 9b9af10ef77f4930bee28b853a626f5f0247981b Mon Sep 17 00:00:00 2001 From: James Adams Date: Thu, 26 Jul 2018 10:39:47 +0100 Subject: [PATCH 6/7] Fudge the environment passed to git Change-Id: I7b1e4f2d9c747f4560745a1d1cad65317182a63b --- tests/broker/brokertest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/broker/brokertest.py b/tests/broker/brokertest.py index 2ee6b0067..b3b6a07a7 100755 --- a/tests/broker/brokertest.py +++ b/tests/broker/brokertest.py @@ -586,6 +586,10 @@ def gitenv(cls, env=None): # Some reasonable defaults... path = ["/bin", "/usr/bin"] + # Allow tests running in a CI environment (e.g. travis) to find local git config + if "HOME" not in newenv: + newenv["HOME"] = os.environ.get("HOME", "/home/%(USER)s" % newenv) + # The 'aq' command need to run some external tools, so make sure those # will be found for exe in [sys.executable, @@ -608,6 +612,7 @@ def gitcommand_raw(self, command, **kwargs): args = [command] args.insert(0, self.config.lookup_tool("git")) env = self.gitenv(kwargs.pop("env", None)) + p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, **kwargs) return p From 7c83bb7d29ceb61d3f78cb226625a09b175f7821 Mon Sep 17 00:00:00 2001 From: James Adams Date: Thu, 26 Jul 2018 10:59:11 +0100 Subject: [PATCH 7/7] Test fake DSDB Change-Id: I6f7ba5a9da3c1341bfa56788ee48510853ce0b61 --- .travis.yml | 5 ++++- tests/broker/brokertest.py | 2 ++ tests/fakebin/fake_dsdb | 7 +++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 494605637..b2fd3ec9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,10 +12,13 @@ before_script: script: - cd $TRAVIS_BUILD_DIR && tests/runtests.py -c tests/unittest.conf.travis --assume_yes after_script: - - tail -n 128 /var/tmp/travis/aqtest/quattor/logs/aqd.log + - cat /tmp/fake_dsdb.log + - cat /var/tmp/travis/aqtest/quattor/logs/aqd.log - ls /var/tmp/travis/aqtest/scratch - ls /var/tmp/travis/aqtest/scratch/dsdb_coverage + - tail -n 128 /var/tmp/travis/aqtest/scratch/dsdb_coverage/expected_dsdb_cmds - tail -n 128 /var/tmp/travis/aqtest/scratch/dsdb_coverage/issued_dsdb_cmds + - tail -n 128 /var/tmp/travis/aqtest/scratch/dsdb_coverage/failed_dsdb_cmds addons: hosts: - travis.dev diff --git a/tests/broker/brokertest.py b/tests/broker/brokertest.py index b3b6a07a7..c398f0f0c 100755 --- a/tests/broker/brokertest.py +++ b/tests/broker/brokertest.py @@ -702,6 +702,8 @@ def dsdb_expect(self, command, fail=False, errstr=""): filename = DSDB_EXPECT_SUCCESS_FILE expected_name = os.path.join(self.dsdb_coverage_dir, filename) + print expected_name + with open(expected_name, "a") as fp: if isinstance(command, list): fp.write(" ".join(str(cmd) for cmd in command)) diff --git a/tests/fakebin/fake_dsdb b/tests/fakebin/fake_dsdb index 00c0fcd54..3430ea7c3 100755 --- a/tests/fakebin/fake_dsdb +++ b/tests/fakebin/fake_dsdb @@ -14,14 +14,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +echo "HELLO" >> /tmp/fake_dsdb.log + DATADIR=$(dirname "$0")/dsdb.d if [ -z "$AQTEST_SCRATCHDIR" ]; then + echo "WHOOPS" >> /tmp/fake_dsdb.log exit 0 fi +echo "AQTEST_SCRATCHDIR=$AQTEST_SCRATCHDIR" >> /tmp/fake_dsdb.log + AQTEST_DSDB_COVERAGE_DIR="$AQTEST_SCRATCHDIR/dsdb_coverage" +echo "AQTEST_DSDB_COVERAGE_DIR=$AQTEST_DSDB_COVERAGE_DIR" >> /tmp/fake_dsdb.log + echo "$*" >> "${AQTEST_DSDB_COVERAGE_DIR}/issued_dsdb_cmds" if grep -q "^$*\$" "${AQTEST_DSDB_COVERAGE_DIR}/expected_dsdb_cmds" 2>/dev/null; then