diff --git a/hera_librarian/tests/__init__.py b/hera_librarian/tests/__init__.py deleted file mode 100644 index 2911d77..0000000 --- a/hera_librarian/tests/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- mode: python; coding: utf-8 -*- -# Copyright 2019 the HERA Collaboration -# Licensed under the 2-clause BSD License - -"""Define test data files and attributes -""" - -import pytest -import os - - -# define where to find the data and their properties -DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "test_data") - -ALL_FILES = pytest.mark.datafiles( - os.path.join(DATA_DIR, "zen.2458043.12552.xx.HH.uvA"), - os.path.join(DATA_DIR, "zen.2458432.34569.uvh5"), - keep_top_dir=True, -) - -filetypes = ["uvA", "uvh5"] - -obsids = [1192201262, 1225829886] # miriad, uvh5 - -md5sums = [ - "ab038eee080348eaa5abd221ec702a67", # miriad - "291a451139cf16e73d880437270dd0ed", # uvh5 -] - -pathsizes = [983251, 224073] # uvh5, miriad diff --git a/hera_librarian/tests/test_base_store.py b/hera_librarian/tests/test_base_store.py deleted file mode 100644 index c8614d8..0000000 --- a/hera_librarian/tests/test_base_store.py +++ /dev/null @@ -1,217 +0,0 @@ -# -*- mode: python; coding: utf-8 -*- -# Copyright 2019 the HERA Collaboration -# Licensed under the 2-clause BSD License - -"""Test code in hera_librarian/store.py - -""" - -from __future__ import print_function, division, absolute_import -import pytest -import os -import tempfile -import json -import shutil -from hera_librarian import base_store, RPCError - -from . import ALL_FILES, filetypes, obsids, md5sums, pathsizes - - -@pytest.fixture -def local_store(): - tempdir = tempfile.mkdtemp(dir="/tmp") - return base_store.BaseStore("local_store", tempdir, "localhost"), tempdir - - -def test_path(local_store): - # test that store path is prepended - dirpath = os.path.join(local_store[1], "my_dir") - assert local_store[0]._path("my_dir") == dirpath - - # test passing in an absolute path - with pytest.raises(ValueError): - local_store[0]._path("/tmp/my_dir") - - # clean up - shutil.rmtree(os.path.join(local_store[1])) - - return - - -def test_ssh_slurp(local_store): - # test simple command - assert local_store[0]._ssh_slurp("echo hello world").decode("utf-8") == "hello world\n" - - # try a bogus command - with pytest.raises(RPCError): - local_store[0]._ssh_slurp("foo") - - # clean up - shutil.rmtree(os.path.join(local_store[1])) - - return - - -def test_copy_to_store(tmpdir, local_store): - # make a fake file in our tmpdir - tmppath = os.path.join(str(tmpdir), "my_file.txt") - with open(tmppath, "w") as f: - print("hello world", file=f) - - # copy it over - local_store[0].copy_to_store(str(tmpdir), "test_directory") - - # check that it exists - dirpath = os.path.join(local_store[1], "test_directory") - assert local_store[0]._ssh_slurp("ls {}".format(dirpath)).decode("utf-8") == "my_file.txt\n" - - # clean up - shutil.rmtree(os.path.join(local_store[1])) - - return - - -def test_chmod(local_store): - # make a small test file on the store, then change permissions - tempdir = local_store[1] - temppath = os.path.join(tempdir, "my_empty_file") - local_store[0]._ssh_slurp("touch {}".format(temppath)) - local_store[0]._chmod("my_empty_file", "664") - - # make sure permissions are correct - output = local_store[0]._ssh_slurp("ls -l {}".format(temppath)).decode("utf-8") - perms = output.split(" ")[0] - assert perms == "-rw-rw-r--" - - # clean up - shutil.rmtree(os.path.join("/tmp", local_store[1])) - - return - - -def test_move(local_store): - # test moving a file - temppath = os.path.join(local_store[1], "my_empty_file") - local_store[0]._ssh_slurp("touch {}".format(temppath)) - local_store[0]._move("my_empty_file", "my_moved_file") - temppath2 = os.path.join("/tmp", local_store[1], "my_moved_file") - assert local_store[0]._ssh_slurp("ls {}".format(temppath2)).decode("utf-8") == "{}\n".format(temppath2) - - # test trying to overwrite a file that already exists - with pytest.raises(RPCError): - local_store[0]._ssh_slurp("touch {}".format(temppath)) - local_store[0]._move("my_empty_file", "my_moved_file") - - # remove existing files; test using chmod - local_store[0]._ssh_slurp("rm -f {} {}".format(temppath, temppath2)) - local_store[0]._ssh_slurp("touch {}".format(temppath)) - local_store[0]._move("my_empty_file", "my_moved_file", chmod_spec=664) - output = local_store[0]._ssh_slurp("ls -l {}".format(temppath2)).decode("utf-8") - perms = output.split(" ")[0] - assert perms == "-rw-rw-r--" - - # clean up - shutil.rmtree(os.path.join(local_store[1])) - - return - - -def test_delete(local_store): - # test removing a file - temppath = os.path.join(local_store[1], "my_empty_file") - local_store[0]._ssh_slurp("touch {}".format(temppath)) - local_store[0]._delete("my_empty_file") - assert ( - local_store[0]._ssh_slurp( - "if [ -f {} ]; then echo file_still_exists; fi".format(temppath) - ).decode("utf-8") - == "" - ) - - # test deleting a write-protected file - tempdir = os.path.join(local_store[1], "my_empty_dir") - local_store[0]._ssh_slurp("mkdir {0}; chmod 755 {0}".format(tempdir)) - local_store[0]._delete("my_empty_dir", chmod_before=True) - assert ( - local_store[0]._ssh_slurp( - "if [ -d {} ]; then echo dir_still_exists; fi".format(tempdir) - ).decode("utf-8") - == "" - ) - - # clean up - shutil.rmtree(os.path.join(local_store[1])) - - return - - -def test_create_tempdir(local_store): - # make sure no temp dirs currently exist on host - tempdir = os.path.join(local_store[1]) - local_store[0]._ssh_slurp("rm -rf {}/libtmp.*".format(tempdir)) - tmppath = local_store[0]._create_tempdir() - # we don't know exactly what the directory name will be, because a random - # 6-digit string is appended to the end - assert tmppath.startswith("libtmp.") - assert len(tmppath) == len("libtmp.") + 6 - # make sure it exists on the host - assert ( - local_store[0]._ssh_slurp("ls -d1 {}/{}".format(tempdir, tmppath)).decode("utf-8") - == "{}/{}\n".format(tempdir, tmppath) - ) - - # clean up - shutil.rmtree(os.path.join(local_store[1])) - - return - - -@ALL_FILES -def test_get_info_for_path(local_store, datafiles): - # copy a datafile to store directory, so we can get its info - filepaths = sorted(list(map(str, datafiles.listdir()))) - filename = os.path.basename(filepaths[0]) - local_store[0].copy_to_store(filepaths[0], filename) - - # get the file info and check that it's right - info = local_store[0].get_info_for_path(filename) - # make a dict of the correct answers - # the uvh5 properties are first in these lists - correct_dict = { - "md5": md5sums[0], - "obsid": obsids[0], - "type": filetypes[0], - "size": pathsizes[0], - } - assert info == correct_dict - - # clean up - shutil.rmtree(os.path.join(local_store[1])) - - return - - -def test_get_space_info(local_store): - # get the disk information of the store - info = local_store[0].get_space_info() - assert "used" in info.keys() - assert "available" in info.keys() - assert "total" in info.keys() - assert info["used"] + info["available"] == info["total"] - - # test using the cache -- make sure the info is the same - info_cached = local_store[0].get_space_info() - assert info == info_cached - - # we also test the capacity, space_left, and usage_percentage properties - capacity = local_store[0].capacity - space_left = local_store[0].space_left - usage_percentage = local_store[0].usage_percentage - assert capacity == info["total"] - assert space_left == info["available"] - assert usage_percentage == pytest.approx(100.0 * info["used"] / info["total"]) - - # clean up - shutil.rmtree(os.path.join(local_store[1])) - - return diff --git a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/flags b/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/flags deleted file mode 100644 index 2853c1e..0000000 Binary files a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/flags and /dev/null differ diff --git a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/header b/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/header deleted file mode 100644 index ce345b7..0000000 Binary files a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/header and /dev/null differ diff --git a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/history b/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/history deleted file mode 100644 index a6c150a..0000000 --- a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/history +++ /dev/null @@ -1,3 +0,0 @@ -CORR-DACQ: created file. - Read/written with pyuvdata version: 1.1. Git origin: https://github.com/HERA-Team/pyuvdata.git. Git hash: 8c65e5432ad70a38102b6c09ad5851e4de10f76a. Git branch: h1c. Git description: v1.1-533-g8c65e54. Downselected to specific antennas using pyuvdata. Hera Hex antennas selected with hera_cal/scripts/extract_hh.py, hera_cal version: {'git_hash': '8de90f0f2afe01adec5a5e317716af5a8e7811b4', 'version': '1.0', 'git_description': 'v1.0', 'git_branch': 'h1c', 'git_origin': 'https://github.com/HERA-Team/hera_cal.git'}. - Read/written with pyuvdata version: 1.2.1. Git origin: https://github.com/HERA-Team/pyuvdata. Git hash: 1a3bf03ea8859d02a5c5e22fe5290f17fad653b4. Git branch: master. Git description: v1.2-49-g1a3bf03. Downselected to specific antennas, frequencies using pyuvdata. diff --git a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/vartable b/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/vartable deleted file mode 100644 index da99589..0000000 --- a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/vartable +++ /dev/null @@ -1,39 +0,0 @@ -r corr -i nchan -i npol -i nspect -d inttime -d sdf -a source -a telescop -d latitud -d longitu -d antdiam -i nschan -i ischan -i nants -d antpos -d sfreq -i ntimes -i nbls -i nblts -a visunits -a instrume -d altitude -a cminfo -a st_type -d stopt -d startt -i obsid -a cmver -d duration -d antnums -a antnames -d lst -d ra -d dec -i pol -d cnt -d coord -d time -r baseline diff --git a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/visdata b/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/visdata deleted file mode 100644 index 9e1a3cf..0000000 Binary files a/hera_librarian/tests/test_data/zen.2458043.12552.xx.HH.uvA/visdata and /dev/null differ diff --git a/hera_librarian/tests/test_data/zen.2458432.34569.uvh5 b/hera_librarian/tests/test_data/zen.2458432.34569.uvh5 deleted file mode 100644 index a4bf6b9..0000000 Binary files a/hera_librarian/tests/test_data/zen.2458432.34569.uvh5 and /dev/null differ diff --git a/hera_librarian/tests/test_utils.py b/hera_librarian/tests/test_utils.py deleted file mode 100644 index 5aadfe0..0000000 --- a/hera_librarian/tests/test_utils.py +++ /dev/null @@ -1,159 +0,0 @@ -# -*- mode: python; coding: utf-8 -*- -# Copyright 2019 the HERA Collaboration -# Licensed under the 2-clause BSD License - -"""Test code in hera_librarian/utils.py - -""" - -import pytest -import os -import six -import sys -import json -from contextlib import contextmanager -from hera_librarian import utils - -# import test data attributes from __init__.py -from . import ALL_FILES, obsids, filetypes, md5sums, pathsizes - - -def test_get_type_from_path(): - """Test type checking from path""" - path = "/some/long/file.name.txt" - assert utils.get_type_from_path(path) == "txt" - - return - - -def test_get_pol_from_path(): - """Test polarization extraction from filename""" - filename = "zen.2458000.12345.xx.uvh5" - assert utils.get_pol_from_path(filename) == "xx" - - filename = "zen.2458000.12345.uvh5" - assert utils.get_pol_from_path(filename) is None - - return - - -@pytest.mark.filterwarnings("ignore:numpy.ufunc size changed") -@ALL_FILES -def test_get_obsid_from_path(datafiles): - """Test extracting obsid values from datasets""" - filepaths = sorted(list(map(str, datafiles.listdir()))) - for obsid, path in zip(obsids, filepaths): - assert utils.get_obsid_from_path(path) == obsid - - return - - -def test_normalize_and_validate_md5(): - """Test md5sum normalization""" - md5sum = "d41d8cd98f00b204e9800998ecf8427e" - # function does not do anything for text already lowercase - assert utils.normalize_and_validate_md5(md5sum) == md5sum - - md5sum_padded = md5sum + " " - assert utils.normalize_and_validate_md5(md5sum_padded) == md5sum - - md5sum_upper = md5sum.upper() + " " - assert utils.normalize_and_validate_md5(md5sum_upper) == md5sum - - # make sure error is raised when length is incorrect - with pytest.raises(ValueError): - utils.normalize_and_validate_md5(md5sum[:-1]) - - return - - -@ALL_FILES -def test_md5_of_file(datafiles): - """Test generating md5sum of file""" - filepaths = sorted(list(map(str, datafiles.listdir()))) - assert utils._md5_of_file(filepaths[1]) == md5sums[1] - - return - - -@ALL_FILES -def test_get_md5_from_path(datafiles): - """Test getting the md5sum for both a flat file and directory""" - filepaths = sorted(list(map(str, datafiles.listdir()))) - # test normal execution - for md5sum, path in zip(md5sums, filepaths): - assert utils.get_md5_from_path(path) == md5sum - - # test adding funny bits to the ends of the directory names - datafile_miriad = filepaths[0] + "//." - assert utils.get_md5_from_path(datafile_miriad) == md5sums[0] - - return - - -@ALL_FILES -def test_get_size_from_path(datafiles): - """Test computing filesize from path""" - filepaths = sorted(list(map(str, datafiles.listdir()))) - for pathsize, path in zip(pathsizes, filepaths): - assert utils.get_size_from_path(path) == pathsize - - return - - -@ALL_FILES -def test_gather_info_for_path(datafiles): - """Test getting all info for a given path""" - filepaths = sorted(list(map(str, datafiles.listdir()))) - for filetype, md5, size, obsid, path in zip( - filetypes, md5sums, pathsizes, obsids, filepaths - ): - info = utils.gather_info_for_path(path) - assert info["type"] == filetype - assert info["md5"] == md5 - assert info["size"] == size - assert info["obsid"] == obsid - - return - - -@ALL_FILES -def test_print_info_for_path(datafiles, capsys): - """Test printing file info to stdout""" - filepaths = sorted(list(map(str, datafiles.listdir()))) - for filetype, md5, size, obsid, path in zip( - filetypes, md5sums, pathsizes, obsids, filepaths - ): - utils.print_info_for_path(path) - out, err = capsys.readouterr() - # convert from json to dict - out_dict = json.loads(out) - - # build up correct dict - correct_info = {"type": filetype, "md5": md5, "size": size, "obsid": obsid} - assert out_dict == correct_info - - return - - -def test_format_jd_as_calendar_date(): - """Test converting JD to calendar date""" - jd = 2456000 - assert utils.format_jd_as_calendar_date(jd) == "2012-03-13" - - return - - -def test_format_jd_as_iso_date_time(): - """Test converting JD to ISO datetime""" - jd = 2456000 - assert utils.format_jd_as_iso_date_time(jd) == "2012-03-13 12:00:00" - - return - - -def test_format_obsid_as_calendar_date(): - """Test converting obsid to calendar date""" - assert utils.format_obsid_as_calendar_date(obsids[1]) == "2018-11-09" - - return diff --git a/pyproject.toml b/pyproject.toml index be826a5..17fb4d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,9 +68,10 @@ Legacy = "https://github.com/HERA-Team/librarian/" [tool.pytest.ini_options] testpaths = [ - "tests/integration_test", "tests/server_unit_test", "tests/background_unit_test", + "tests/client_unit_test", + "tests/integration_test", ] [tool.coverage.run] diff --git a/tests/client_unit_test/__init__.py b/tests/client_unit_test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hera_librarian/tests/test_cli.py b/tests/client_unit_test/test_cli.py similarity index 94% rename from hera_librarian/tests/test_cli.py rename to tests/client_unit_test/test_cli.py index e9bfb50..6c727a1 100644 --- a/hera_librarian/tests/test_cli.py +++ b/tests/client_unit_test/test_cli.py @@ -125,10 +125,4 @@ def test_generate_parser(): assert "stage-files" in available_subparsers assert "upload" in available_subparsers - return - - -def test_main(script_runner): - version = hera_librarian.__version__ - ret = script_runner.run("librarian", "-V") - assert ret.stdout == "librarian {}\n".format(version) + return \ No newline at end of file