diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index ddfe31c..790b15e 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -37,7 +37,9 @@ jobs: - name: Lint with Tox if: ${{ matrix.python-version == env.PYTHON_MAIN_VERSION }} shell: bash -l {0} - run: tox -e lint + run: | + tox -e lint + tox -e lint-security - name: Test with Tox shell: bash -l {0} diff --git a/Pipfile b/Pipfile index 07de281..ab3bd21 100644 --- a/Pipfile +++ b/Pipfile @@ -11,6 +11,7 @@ autopep8 = "*" tox = "*" notebook = "*" sphinx-rtd-theme = "*" +bandit = "*" [packages] numpy = "*" diff --git a/pystog/cli.py b/pystog/cli.py index 2069e98..071f8ec 100755 --- a/pystog/cli.py +++ b/pystog/cli.py @@ -11,13 +11,10 @@ import json from pystog import StoG +from pystog.stog import NoInputFilesException from pystog.io import get_cli_parser, parse_cli_args -class NoInputFilesException(Exception): - """Exception when no files are given to process""" - - def pystog_cli(kwargs=None): """ Main entry point for PyStoG CLI tool @@ -26,8 +23,9 @@ def pystog_cli(kwargs=None): If None, parsed from command line via get_cli_parser :type kwargs: dict """ + parser = get_cli_parser() + if not kwargs: - parser = get_cli_parser() args = parser.parse_args() if args.json: print("loading config from '%s'" % args.json) diff --git a/pystog/stog.py b/pystog/stog.py index 10277d4..256bf5c 100644 --- a/pystog/stog.py +++ b/pystog/stog.py @@ -17,6 +17,10 @@ from pystog.fourier_filter import FourierFilter +class NoInputFilesException(Exception): + """Exception when no files are given to process""" + + class StoG(object): """ The StoG class is used to put together @@ -543,8 +547,8 @@ def reciprocal_individuals(self): return self.__reciprocal_individuals @reciprocal_individuals.setter - def reciprocal_individuals(self, df): - self.__reciprocal_individuals = df + def reciprocal_individuals(self, individuals): + self.__reciprocal_individuals = individuals @property def sq_individuals(self): @@ -562,8 +566,8 @@ def sq_individuals(self): return self.__sq_individuals @sq_individuals.setter - def sq_individuals(self, df): - self.__sq_individuals = df + def sq_individuals(self, individuals): + self.__sq_individuals = individuals @property def sq_master(self): @@ -580,8 +584,8 @@ def sq_master(self): return self.__sq_master @sq_master.setter - def sq_master(self, df): - self.__sq_master = df + def sq_master(self, sq): + self.__sq_master = sq @property def gr_master(self): @@ -598,8 +602,8 @@ def gr_master(self): return self.__gr_master @gr_master.setter - def gr_master(self, df): - self.__gr_master = df + def gr_master(self, gr): + self.__gr_master = gr @property def q_master(self): @@ -616,8 +620,8 @@ def q_master(self): return self.__q_master @q_master.setter - def q_master(self, df): - self.__q_master = df + def q_master(self, q): + self.__q_master = q @property def r_master(self): @@ -634,8 +638,8 @@ def r_master(self): return self.__r_master @r_master.setter - def r_master(self, df): - self.__r_master = df + def r_master(self, r): + self.__r_master = r @property def real_space_function(self): @@ -801,9 +805,11 @@ def read_all_data(self, **kwargs): the **sq_individuals** numpy storage array in **add_dataset** method via **read_dataset** method. """ - assert self.files is not None - assert len(self.files) != 0 + # Check that we have files to operate on + if not self.files: + raise NoInputFilesException("No input files given in arguments") + # Read in all the data files for i, file_info in enumerate(self.files): self.read_dataset(file_info, **kwargs) @@ -824,10 +830,12 @@ def read_dataset( :param info: Dict with information for dataset (filename, manipulations, etc.) :type info: dict - :param xcol: The column in the data file that contains the X-axis + :param xcol: Column in data file for X-axis :type xcol: int - :param ycol: The column in the data file that contains the Y-axis + :param ycol: Column in data file for Y-axis :type ycol: int + :param dycol: Column in data file for Y uncertainty + :type dycol: int :param sep: Separator for the file used by numpy.loadtxt :type sep: raw string :param skiprows: Number of rows to skip. Passed to numpy.loadtxt @@ -905,10 +913,11 @@ def add_dataset( xoffset = info['X']['Offset'] if adjusting: - x, y, dy = self._apply_scales_and_offset(x, y, dy=dy, - yscale=yscale, - yoffset=yoffset, - xoffset=xoffset) + x, y, dy = self.apply_scales_and_offset( + x, y, dy=dy, + yscale=yscale, + yoffset=yoffset, + xoffset=xoffset) # Save overal x-axis min and max self.xmin = min(self.xmin, xmin) @@ -953,8 +962,8 @@ def add_dataset( array_seq = (self.sq_individuals, np.stack((x, y, dy))) self.sq_individuals = np.concatenate(array_seq, axis=1) - def _apply_scales_and_offset( - self, + @staticmethod + def apply_scales_and_offset( x, y, dy=None, @@ -977,42 +986,14 @@ def _apply_scales_and_offset( :return: X and Y vectors after scales and offsets applied :rtype: numpy.array pair """ - y = self._scale(y, yscale) - y = self._offset(y, yoffset) - x = self._offset(x, xoffset) + y = y * yscale + y = y + yoffset + x = x + xoffset if dy is None: dy = np.zeros_like(y) - dy = self._scale(dy, yscale) + dy = dy * yscale return x, y, dy - def _offset(self, data, offset): - """ - Applies offset to data - - :param data: Input data - :type data: numpy.array or list - :param offset: Offset to apply to data - :type offset: float - :return: Data with offset applied - :rtype: numpy.array - """ - data = data + offset - return data - - def _scale(self, data, scale): - """ - Applies scale to data - - :param data: Input data - :type data: numpy.array or list - :param offset: Scale to apply to data - :type offset: float - :return: Data with scale applied - :rtype: numpy.array - """ - data = scale * data - return data - def merge_data(self): """ Merges the reciprocal space data stored in the @@ -1095,7 +1076,7 @@ def merge_data(self): sq = data_merged[1] dsq = data_merged[2] - q, sq, dsq = self._apply_scales_and_offset( + q, sq, dsq = self.apply_scales_and_offset( q, sq, yscale=self.merged_opts['Y']['Scale'], yoffset=self.merged_opts['Y']['Offset'], diff --git a/requirements-dev.txt b/requirements-dev.txt index 8cad28b..16be0d3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ +bandit mock pytest flake8 diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..beb2f44 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,8 @@ +import pytest +from pystog.cli import pystog_cli +from pystog.stog import NoInputFilesException + + +def test_pystog_cli_no_files_exception(): + with pytest.raises(NoInputFilesException): + pystog_cli({"cat": "meow"}) diff --git a/tests/test_converter.py b/tests/test_converter.py index cd925fa..ca140cf 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -280,10 +280,18 @@ def F_to_S(self): atol=self.atol) assert_allclose(dsq, np.ones_like(self.q) / self.q) + def F_to_S_with_no_dfq(self): + sq, dsq = self.converter.F_to_S(self.q, self.fq, **self.kwargs) + assert_allclose( + sq[self.first:self.last], + self.sq_target, + rtol=self.rtol, + atol=self.atol) + assert_allclose(dsq, np.zeros_like(dsq)) + def F_to_FK(self): - fq_keen, dfq_keen = self.converter.F_to_FK(self.q, self.fq, - np.ones_like(self.q), - **self.kwargs) + fq_keen, dfq_keen = self.converter.F_to_FK( + self.q, self.fq, np.ones_like(self.q), **self.kwargs) assert_allclose( fq_keen[self.first:self.last], self.fq_keen_target, @@ -293,6 +301,16 @@ def F_to_FK(self): dfq_keen, np.ones_like(self.q) * self.kwargs['^2'] / self.q) + def F_to_FK_with_no_dfq(self): + fq_keen, dfq_keen = self.converter.F_to_FK( + self.q, self.fq, **self.kwargs) + assert_allclose( + fq_keen[self.first:self.last], + self.fq_keen_target, + rtol=self.rtol, + atol=self.atol) + assert_allclose(dfq_keen, np.zeros_like(dfq_keen)) + def F_to_DCS(self): dcs, ddcs = self.converter.F_to_DCS(self.q, self.fq, np.ones_like(self.q), @@ -395,9 +413,15 @@ def test_S_to_DCS(self): def test_F_to_S(self): self.F_to_S() + def test_F_to_S_with_no_dfq(self): + self.F_to_S_with_no_dfq() + def test_F_to_FK(self): self.F_to_FK() + def test_F_to_FK_with_no_dfq(self): + self.F_to_FK_with_no_dfq() + def test_F_to_DCS(self): self.F_to_DCS() @@ -438,9 +462,15 @@ def test_S_to_DCS(self): def test_F_to_S(self): self.F_to_S() + def test_F_to_S_with_no_dfq(self): + self.F_to_S_with_no_dfq() + def test_F_to_FK(self): self.F_to_FK() + def test_F_to_FK_with_no_dfq(self): + self.F_to_FK_with_no_dfq() + def test_F_to_DCS(self): self.F_to_DCS() diff --git a/tests/test_stog.py b/tests/test_stog.py index 26f0bc6..b57daa5 100644 --- a/tests/test_stog.py +++ b/tests/test_stog.py @@ -4,11 +4,14 @@ import unittest from tests.utils import \ - get_data_path, load_data, get_index_of_function + get_data_path, \ + get_index_of_function, \ + load_data from tests.materials import Argon from pystog.utils import \ - RealSpaceHeaders, ReciprocalSpaceHeaders -from pystog.stog import StoG + RealSpaceHeaders, \ + ReciprocalSpaceHeaders +from pystog.stog import NoInputFilesException, StoG class TestStogBase(unittest.TestCase): @@ -36,6 +39,7 @@ def initialize_material(self): # targets for 1st peaks self.sq_target = self.material.sq_target + self.dsq_target = 3.087955 self.fq_target = self.material.fq_target self.fq_keen_target = self.material.fq_keen_target self.dcs_target = self.material.dcs_target @@ -151,8 +155,10 @@ def test_stog_init(self): "Offset": 0.0, "Scale": 1.0}}) self.assertEqual(stog.stem_name, "out") self.assertEqual(stog.reciprocal_individuals.size, 0) + self.assertEqual(stog.q_master, {}) self.assertEqual(stog.sq_master, {}) self.assertEqual(stog.sq_individuals.size, 0) + self.assertEqual(stog.r_master, {}) self.assertEqual(stog.gr_master, {}) def test_stog_init_kwargs_files(self): @@ -339,11 +345,21 @@ def test_stog_sq_individuals_setter(self): stog.sq_individuals = self.target np.testing.assert_allclose(stog.sq_individuals, self.target) + def test_stog_q_master_setter(self): + stog = StoG() + stog.q_master = self.target + np.testing.assert_allclose(stog.q_master, self.target) + def test_stog_sq_master_setter(self): stog = StoG() stog.sq_master = self.target np.testing.assert_allclose(stog.sq_master, self.target) + def test_stog_r_master_setter(self): + stog = StoG() + stog.r_master = self.target + np.testing.assert_allclose(stog.r_master, self.target) + def test_stog_gr_master_setter(self): stog = StoG() stog.gr_master = self.target @@ -371,6 +387,39 @@ class TestStogDatasetSpecificMethods(TestStogBase): def setUp(self): super(TestStogDatasetSpecificMethods, self).setUp() + def test_stog_apply_scales_and_offset(self): + q, sq, dq = StoG.apply_scales_and_offset(self.q, self.sq) + np.testing.assert_allclose(q, self.q) + np.testing.assert_allclose(sq, self.sq) + np.testing.assert_allclose(dq, np.zeros_like(sq)) + + def test_stog_apply_scales_and_offset_with_dy(self): + q, sq, dq = StoG.apply_scales_and_offset(self.q, self.sq, dy=self.sq) + np.testing.assert_allclose(q, self.q) + np.testing.assert_allclose(sq, self.sq) + np.testing.assert_allclose(dq, self.sq) + + def test_stog_apply_scales_and_offset_with_yscale(self): + q, sq, dq = StoG.apply_scales_and_offset( + self.q, self.sq, dy=self.sq, yscale=2.0) + np.testing.assert_allclose(q, self.q) + np.testing.assert_allclose(sq, self.sq * 2.0) + np.testing.assert_allclose(dq, self.sq * 2.0) + + def test_stog_apply_scales_and_offset_with_yoffset(self): + q, sq, dq = StoG.apply_scales_and_offset( + self.q, self.sq, dy=self.sq, yoffset=2.0) + np.testing.assert_allclose(q, self.q) + np.testing.assert_allclose(sq, self.sq + 2.0) + np.testing.assert_allclose(dq, self.sq) + + def test_stog_apply_scales_and_offset_with_xoffset(self): + q, sq, dq = StoG.apply_scales_and_offset( + self.q, self.sq, dy=self.sq, xoffset=2.0) + np.testing.assert_allclose(q, self.q + 2.0) + np.testing.assert_allclose(sq, self.sq) + np.testing.assert_allclose(dq, self.sq) + def test_stog_add_dataset(self): # Number of decimal places for precision places = 5 @@ -391,10 +440,12 @@ def test_stog_add_dataset(self): stog.reciprocal_individuals[1][self.first], self.sq_target[0], places=places) + self.assertEqual(stog.reciprocal_individuals[2][self.first], 0.0) self.assertAlmostEqual( stog.sq_individuals[1][self.first], self.sq_target[0], places=places) + self.assertEqual(stog.sq_individuals[2][self.first], 0.0) # Add the Q[S(Q)-1] data set and check values for it and S(Q) against # targets @@ -404,14 +455,19 @@ def test_stog_add_dataset(self): 'ReciprocalFunction': 'Q[S(Q)-1]' } stog.add_dataset(info) + self.assertAlmostEqual( stog.reciprocal_individuals[1][self.first + stride], self.fq_target[0], places=places) + self.assertEqual( + stog.reciprocal_individuals[2][self.first + stride], + 0.0) self.assertAlmostEqual( stog.sq_individuals[1][self.first], self.sq_target[0], places=places) + self.assertEqual(stog.sq_individuals[2][self.first], 0.0) # Add the FK(Q) data set and check values for it and S(Q) against # targets @@ -424,10 +480,14 @@ def test_stog_add_dataset(self): stog.reciprocal_individuals[1][self.first + stride], self.fq_keen_target[0], places=places) + self.assertEqual( + stog.reciprocal_individuals[2][self.first + stride], + 0.0) self.assertAlmostEqual( stog.sq_individuals[1][self.first], self.sq_target[0], places=places) + self.assertEqual(stog.sq_individuals[2][self.first], 0.0) # Add the DCS(Q) data set and check values for it and S(Q) against # targets @@ -440,10 +500,14 @@ def test_stog_add_dataset(self): stog.reciprocal_individuals[1][self.first + stride], self.dcs_target[0], places=places) + self.assertEqual( + stog.reciprocal_individuals[2][self.first + stride], + 0.0) self.assertAlmostEqual( stog.sq_individuals[1][self.first], self.sq_target[0], places=places) + self.assertEqual(stog.sq_individuals[2][self.first], 0.0) def test_stog_add_dataset_yscale(self): # Scale S(Q) and make sure it does not equal original target values @@ -456,9 +520,11 @@ def test_stog_add_dataset_yscale(self): self.assertNotEqual( stog.reciprocal_individuals[1][self.first], self.sq_target[0]) + self.assertEqual(stog.reciprocal_individuals[2][self.first], 0.0) self.assertNotEqual( stog.sq_individuals[1][self.first], self.sq_target[0]) + self.assertEqual(stog.sq_individuals[2][self.first], 0.0) def test_stog_add_dataset_yoffset(self): # Offset S(Q) and make sure it does not equal original target values @@ -471,10 +537,60 @@ def test_stog_add_dataset_yoffset(self): self.assertNotEqual( stog.reciprocal_individuals[1][self.first], self.sq_target[0]) + self.assertEqual(stog.reciprocal_individuals[2][self.first], 0.0) + self.assertNotEqual( + stog.sq_individuals[1][self.first], + self.sq_target[0]) + self.assertEqual(stog.sq_individuals[2][self.first], 0.0) + + def test_stog_add_dataset_yscale_with_dy(self): + # Scale S(Q) and make sure it does not equal original target values + stog = StoG() + info = { + 'data': [self.q, self.sq, self.sq], + 'ReciprocalFunction': 'S(Q)', + 'Y': {'Scale': 2.0}} + stog.add_dataset(info) + + self.assertNotEqual( + stog.reciprocal_individuals[1][self.first], + self.sq_target[0]) + + dsq_target = info['Y']['Scale'] * 2.59173 + self.assertEqual( + stog.reciprocal_individuals[2][self.first], + dsq_target) + + self.assertNotEqual( + stog.sq_individuals[1][self.first], + self.sq_target[0]) + + self.assertEqual(stog.sq_individuals[2][self.first], dsq_target) + + def test_stog_add_dataset_yoffset_with_dy(self): + # Offset S(Q) and make sure it does not equal original target values + stog = StoG() + info = { + 'data': [self.q, self.sq, self.sq], + 'ReciprocalFunction': 'S(Q)', + 'Y': {'Offset': 2.0}} + stog.add_dataset(info) + + self.assertNotEqual( + stog.reciprocal_individuals[1][self.first], + self.sq_target[0]) + + dsq_target = 2.59173 + self.assertEqual( + stog.reciprocal_individuals[2][self.first], + dsq_target) + self.assertNotEqual( stog.sq_individuals[1][self.first], self.sq_target[0]) + self.assertEqual(stog.sq_individuals[2][self.first], dsq_target) + def test_stog_add_dataset_xoffset(self): # Offset Q from 1.96 -> 2.14 stog = StoG() @@ -505,10 +621,12 @@ def test_stog_add_dataset_default_reciprocal_space_function(self): self.assertEqual( stog.reciprocal_individuals[0][self.first], self.reciprocal_xtarget) + self.assertEqual(stog.reciprocal_individuals[2][self.first], 0.0) self.assertAlmostEqual( stog.reciprocal_individuals[1][self.first], self.sq_target[0], places=places) + self.assertEqual(stog.sq_individuals[2][self.first], 0.0) def test_stog_add_dataset_wrong_reciprocal_space_function_exception(self): # Check qmin and qmax apply cropping @@ -548,6 +666,62 @@ def test_stog_read_dataset(self): stog.reciprocal_individuals[1][self.first], self.sq_target[0], places=places) + self.assertEqual( + stog.reciprocal_individuals[2][self.first], + self.dsq_target) + self.assertAlmostEqual( + stog.sq_individuals[1][self.first], + self.sq_target[0], + places=places) + self.assertEqual( + stog.sq_individuals[2][self.first], + self.dsq_target) + + def test_stog_read_dataset_xcol_data_format_exception(self): + stog = StoG() + filename = get_data_path(self.material.reciprocal_space_filename) + with self.assertRaises(RuntimeError): + stog.read_dataset({'Filename': filename}, xcol=99) + + def test_stog_read_dataset_ycol_data_format_exception(self): + stog = StoG() + filename = get_data_path(self.material.reciprocal_space_filename) + with self.assertRaises(RuntimeError): + stog.read_dataset({'Filename': filename}, ycol=99) + + def test_stog_read_dataset_dycol_too_large(self): + # Number of decimal places for precision + places = 5 + + # Load S(Q) for Argon from test data + stog = StoG(**{'^2': self.kwargs['^2'], + '': self.kwargs['']}) + info = { + 'Filename': get_data_path( + self.material.reciprocal_space_filename), + 'ReciprocalFunction': 'S(Q)', + 'Qmin': 0.02, + 'Qmax': 35.2, + 'Y': { + 'Offset': 0.0, + 'Scale': 1.0}, + 'X': { + 'Offset': 0.0}} + + info['index'] = 0 + stog.read_dataset(info, dycol=99) + + # Check S(Q) data against targets + self.assertEqual( + stog.reciprocal_individuals[0][self.first], + self.reciprocal_xtarget) + self.assertAlmostEqual( + stog.reciprocal_individuals[1][self.first], + self.sq_target[0], + places=places) + self.assertEqual( + stog.reciprocal_individuals[2][self.first], + 0.0) self.assertAlmostEqual( stog.sq_individuals[1][self.first], self.sq_target[0], @@ -555,11 +729,11 @@ def test_stog_read_dataset(self): def test_stog_read_all_data_assertion(self): stog = StoG() - with self.assertRaises(AssertionError): + with self.assertRaises(NoInputFilesException): stog.read_all_data() stog.files = list() - with self.assertRaises(AssertionError): + with self.assertRaises(NoInputFilesException): stog.read_all_data() def test_stog_read_all_data_for_files_length(self): diff --git a/tox.ini b/tox.ini index a0d3597..84ad303 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,10 @@ commands = deps = flake8 commands = flake8 pystog/ tests/ setup.py --count +[testenv:lint-security] +deps = bandit +commands = bandit -r pystog/ -x pystog/_version.py + [testenv:coverage] deps = pytest-cov commands = pytest --cov=pystog --cov-report=term-missing tests/