diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..b2fd3ec9d --- /dev/null +++ b/.travis.yml @@ -0,0 +1,26 @@ +language: python +python: + - "2.7" +before_script: + - curl https://raw.githubusercontent.com/jrha/krb5-travis/master/install_krb5 | sudo bash + - curl https://raw.githubusercontent.com/jrha/krb5-travis/master/install_knc | bash + - wget https://github.com/google/protobuf/releases/download/v3.5.1/protoc-3.5.1-linux-x86_64.zip -O /tmp/protoc.zip && cd /tmp && unzip /tmp/protoc.zip && sudo cp bin/protoc /usr/bin/ && sudo chmod ugo+x /usr/bin/protoc + - cd /tmp && git clone https://github.com/quattor/aquilon-protocols.git && cd aquilon-protocols && git checkout upstream && ./setup.py build && sudo ./setup.py install --install-lib /usr/local/lib/aquilon/protocols/lib/python + - cd $TRAVIS_BUILD_DIR/tools/bootstrap_ms && python setup.py sdist && pip install dist/*.tar.gz + - cd $TRAVIS_BUILD_DIR && cp -R tests/fakemodules/ms/dsdb /home/travis/virtualenv/python$TRAVIS_PYTHON_VERSION/lib/python$TRAVIS_PYTHON_VERSION/site-packages/ms/; echo $? + - ls /home/travis/virtualenv/python$TRAVIS_PYTHON_VERSION/lib/python$TRAVIS_PYTHON_VERSION/site-packages/ms/ +script: + - cd $TRAVIS_BUILD_DIR && tests/runtests.py -c tests/unittest.conf.travis --assume_yes +after_script: + - cat /tmp/fake_dsdb.log + - cat /var/tmp/travis/aqtest/quattor/logs/aqd.log + - ls /var/tmp/travis/aqtest/scratch + - ls /var/tmp/travis/aqtest/scratch/dsdb_coverage + - tail -n 128 /var/tmp/travis/aqtest/scratch/dsdb_coverage/expected_dsdb_cmds + - tail -n 128 /var/tmp/travis/aqtest/scratch/dsdb_coverage/issued_dsdb_cmds + - tail -n 128 /var/tmp/travis/aqtest/scratch/dsdb_coverage/failed_dsdb_cmds +addons: + hosts: + - travis.dev + packages: + - libkrb5-dev diff --git a/lib/aquilon/worker/processes.py b/lib/aquilon/worker/processes.py index b217c570a..70c5b62db 100644 --- a/lib/aquilon/worker/processes.py +++ b/lib/aquilon/worker/processes.py @@ -50,17 +50,18 @@ DSDB_ENABLED = config.getboolean("dsdb", "enable") -if DSDB_ENABLED: - # FIXME - this needs to be moved to depends.py after - # refactoring runtests.py and Config to allow override - # sys.path for python modules when running tests - # DSDB python client +try: import ms.version +except ImportError: + pass +else: + ms.version.addpkg('setuptools', '0.6c11') ms.version.addpkg("requests", "2.7.0") ms.version.addpkg("requests-kerberos", "0.5-ms2") ms.version.addpkg("kerberos", "1.1.5") ms.version.addpkg("dns", "1.10.0") ms.version.addpkg('ms.dsdb', '6.0.30') +if DSDB_ENABLED: import ms.dsdb.client # subprocess.Popen is not thread-safe in Python 2, so we need locking @@ -1078,4 +1079,4 @@ def build_mako_lookup(config, kind, **kwargs): return TemplateLookup(directories=directories, **kwargs) -DSDBRunner.snapshot_handlers['rack'] = DSDBRunner.snapshot_rack \ No newline at end of file +DSDBRunner.snapshot_handlers['rack'] = DSDBRunner.snapshot_rack diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..d22e1b8f6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +Twisted >= 12.2.0 +coverage +ipaddress +jsonschema +lxml +mako +psycopg2 >= 2.5.1 +python-dateutil +six >= 1.7.3 +sqlalchemy >= 0.9.7 +zope-interface +protobuf >= 3.6.0 diff --git a/tests/broker/brokertest.py b/tests/broker/brokertest.py index 2ee6b0067..c398f0f0c 100755 --- a/tests/broker/brokertest.py +++ b/tests/broker/brokertest.py @@ -586,6 +586,10 @@ def gitenv(cls, env=None): # Some reasonable defaults... path = ["/bin", "/usr/bin"] + # Allow tests running in a CI environment (e.g. travis) to find local git config + if "HOME" not in newenv: + newenv["HOME"] = os.environ.get("HOME", "/home/%(USER)s" % newenv) + # The 'aq' command need to run some external tools, so make sure those # will be found for exe in [sys.executable, @@ -608,6 +612,7 @@ def gitcommand_raw(self, command, **kwargs): args = [command] args.insert(0, self.config.lookup_tool("git")) env = self.gitenv(kwargs.pop("env", None)) + p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, **kwargs) return p @@ -697,6 +702,8 @@ def dsdb_expect(self, command, fail=False, errstr=""): filename = DSDB_EXPECT_SUCCESS_FILE expected_name = os.path.join(self.dsdb_coverage_dir, filename) + print expected_name + with open(expected_name, "a") as fp: if isinstance(command, list): fp.write(" ".join(str(cmd) for cmd in command)) diff --git a/tests/broker/test_start.py b/tests/broker/test_start.py index 92b216b0f..c75994270 100755 --- a/tests/broker/test_start.py +++ b/tests/broker/test_start.py @@ -47,6 +47,7 @@ def testclonetemplateking(self): env = {} env["PATH"] = os.environ.get("PATH", "") + env["HOME"] = os.environ.get("HOME", "/home/travis") git = self.config.lookup_tool("git") if source.startswith("git://"): # Easy case - just clone the source repository @@ -93,6 +94,7 @@ def testdisabletemplatetests(self): git = self.config.lookup_tool("git") env = {} env["PATH"] = os.environ.get("PATH", "") + env["HOME"] = os.environ.get("HOME", "/home/travis") tempdir = mkdtemp(prefix="fixup", dir=rundir) diff --git a/tests/fakebin/fake_dsdb b/tests/fakebin/fake_dsdb index 00c0fcd54..3430ea7c3 100755 --- a/tests/fakebin/fake_dsdb +++ b/tests/fakebin/fake_dsdb @@ -14,14 +14,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +echo "HELLO" >> /tmp/fake_dsdb.log + DATADIR=$(dirname "$0")/dsdb.d if [ -z "$AQTEST_SCRATCHDIR" ]; then + echo "WHOOPS" >> /tmp/fake_dsdb.log exit 0 fi +echo "AQTEST_SCRATCHDIR=$AQTEST_SCRATCHDIR" >> /tmp/fake_dsdb.log + AQTEST_DSDB_COVERAGE_DIR="$AQTEST_SCRATCHDIR/dsdb_coverage" +echo "AQTEST_DSDB_COVERAGE_DIR=$AQTEST_DSDB_COVERAGE_DIR" >> /tmp/fake_dsdb.log + echo "$*" >> "${AQTEST_DSDB_COVERAGE_DIR}/issued_dsdb_cmds" if grep -q "^$*\$" "${AQTEST_DSDB_COVERAGE_DIR}/expected_dsdb_cmds" 2>/dev/null; then diff --git a/tests/runtests.py b/tests/runtests.py index 3bf79e9fd..5224ca360 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -58,13 +58,14 @@ def force_yes(msg): print(msg, file=sys.stderr) - print(""" - Please confirm by typing yes (three letters) and pressing enter. - """, file=sys.stderr) - answer = sys.stdin.readline() - if not answer.startswith("yes"): - print("""Aborting.""", file=sys.stderr) - sys.exit(1) + if not opts.assume_yes: + print(""" + Please confirm by typing yes (three letters) and pressing enter. + """, file=sys.stderr) + answer = sys.stdin.readline() + if not answer.startswith("yes"): + print("""Aborting.""", file=sys.stderr) + sys.exit(1) parser = argparse.ArgumentParser(description="Run the broker test suite.", epilog=epilog) @@ -91,6 +92,8 @@ def force_yes(msg): parser.add_argument('-f', '--failfast', action='store_true', help='Add failfast=True option to TestRunner. This will stop ' 'unittests immediatelly if any failure.') +parser.add_argument('--assume_yes', action='store_true', + help='Assume yes to all questions, only use for continuous integration') opts = parser.parse_args() diff --git a/tests/unittest.conf b/tests/unittest.conf index 56f16b0ab..42c4b9c3b 100644 --- a/tests/unittest.conf +++ b/tests/unittest.conf @@ -69,6 +69,7 @@ checkedm = %(srcdir)s/tests/fakebin/fake_checkedm #aqd_checkedm = /ms/dist/aquilon/PROJ/edmtools/dev/bin/aqd_checkedm aqd_checkedm = %(srcdir)s/tests/fakebin/fake_aqd_checkedm location_uri_validator = /ms/dist/aquilon/PROJ/aqd-scripts/qa/bin/location_uri_validator +klist = %(srcdir)s/tests/fakebin/fake_klist [location_feeds] building_feed = %(srcdir)s/tests/fakebin/user-data/location_feed.csv diff --git a/tests/unittest.conf.travis b/tests/unittest.conf.travis new file mode 100644 index 000000000..4a8bca619 --- /dev/null +++ b/tests/unittest.conf.travis @@ -0,0 +1,156 @@ +# This file provides the default for the unittests. To test against +# an oracle database (or otherwise change the parameters), copy this +# file and modify as needed. +# +# The config file etc/aqd.conf.defaults always gets read first, and the +# values below override those. To nullify a value from the defaults, +# it must be explicitly set to an empty value here. + +[DEFAULT] +basedir = /var/tmp/%(user)s/aqtest +quattordir = %(basedir)s/quattor + +# By default takes the sqlite section from aqd.conf.defaults. +[database] +database_section = database_sqlite + +[database_sqlite] +# We do not really care if the host crashes during the unittest... +disable_fsync = yes + +[broker] +servername = %(hostname)s +umask = 0022 +kncport = 6902 +openport = 6903 +git_port = 9419 +# Testing the case when bind_address is set is more interesting than when it's not... +# It would be better to use localhost (to make sure nothing goes out to the wire), but that would make Kerberos unhappy +bind_address = %(hostname)s +run_git_daemon = True +git_author_name = travis +git_author_email = travis@dev +git_committer_name = travis +git_committer_email = travis@dev +trash_branch = unittest_trash +dsdb_use_testdb = 1 +server_notifications = utnotify +client_notifications = False +sharedata = %(srcdir)s/tests/fakebin/nasobjects/testnasobjects.cdb +run_knc = True +grn_to_eonid_map_location = %(srcdir)s/tests/fakebin/eon-data +esx_cluster_allow_cascaded_deco = True +default_max_list_size = 1000 +reconfigure_max_list_size = 15 +pxeswitch_max_list_size = 15 +manage_max_list_size = 15 +reset_advertised_status_max_list_size = 15 +map_grn_max_list_size = 15 +unmap_grn_max_list_size = 15 +user_list_location = %(srcdir)s/tests/fakebin/user-data/passwd.byname +user_delete_limit = 10 +service = aquilon +keytab = /etc/krb5.keytab + +[change_management] +# The contents of --justification will not be validated externally +# (syntax checking will still be performed) +enable = True +extra_options = --mode warn --disable_edm --edm-instance qa + +[tool_locations] +dsdb = %(srcdir)s/tests/fakebin/fake_dsdb +aii_installfe = /bin/echo +vlan2net = %(srcdir)s/tests/fakebin/fake_vlan2net +mean = %(srcdir)s/tests/fakebin/fake_mean +qip_dump_subnetdata = %(srcdir)s/tests/fakebin/fake_qip_dump_subnetdata +ssh = %(srcdir)s/tests/fakebin/fake_ssh +switch_discover = %(srcdir)s/tests/fakebin/fake_switchdata +get_camtable = %(srcdir)s/tests/fakebin/fake_macdata +checkedm = %(srcdir)s/tests/fakebin/fake_checkedm +aqd_checkedm = %(srcdir)s/tests/fakebin/fake_aqd_checkedm +location_uri_validator = %(srcdir)s/tests/fakebin/location_uri_validator +git = /usr/bin/git +klist = /usr/bin/klist +knc = /usr/local/bin/knc + +# Alternative tool locations outside MS +git_daemon = /usr/lib/git-core/git-daemon +ant_contrib_jar = /usr/share/java/ant/ant-contrib.jar + +[location_feeds] +building_feed = %(srcdir)s/tests/fakebin/user-data/location_feed.csv + +[unittest] +last_success_db_snapshot = %(quattordir)s/aquilondb/aquilon_last_success_snapshot.db +fake_module_location = %(srcdir)s/tests/fakemodules/ +scratchdir = %(basedir)s/scratch +mirrordir = %(basedir)s/mirror + +# Update the template_king_host value +template_king_host = localhost +template_base = %(srcdir)s/tests/templates/ + +template_alternate_prod = +fake_hosts_location = %(srcdir)s/tests/fakebin/dsdb.d/show_host_-host_name_ +datadir = %(srcdir)s/tests/broker/data + +# Alternative tool locations outside MS +real_panc_location = /usr/lib/panc.jar + +# Versions for the Linux models. These are used both for Aquilon and Aurora. +linux_version_curr = 6.1-x86_64 +linux_version_prev = 5.1-x86_64 + +hostname = travis.dev + +[archetype_aquilon] +default_grn_target = esp +default_osversion = 6.1-x86_64 +default_osname = linux +host_grn_targets = esp,hlmplus,atarget +personality_grn_targets = esp,hlmplus,atarget + +[archetype_aurora] +default_domain = unittest + +[archetype_f5] +default_personality = generic + +[archetype_filer] +default_personality = generic + +[archetype_metacluster] +default_domain = unittest + +[archetype_hacluster] +allow_cascaded_deco = False +max_priority_order = 99 +min_priority_order = 1 + +[archetype_netinfra] +# Network devices should be compileable, but we're not quite there yet. +# Stash them awain in a separate domain for now to avoid compilation +# errors. +default_domain = netinfra + +[archetype_windows] +default_domain = ut-prod + +[panc] +pan_compiler = %(basedir)s/panc-links/panc-%(version)s.jar +include_pan = True + +[site] +# Site specific settings +default_hardware_label_regex = ^[a-z][a-z0-9]{,62}$ +# Allow Aquilon broker generate nextip for vip/localvip type +# Temporary option until we can vary API by Aquilon broker instance +ipfromtype = True + +[protocols] +# Alternative tool locations outside MS +directory = /usr/local/lib/aquilon/protocols/lib/python + +[dsdb] +enable = False