From ced26beb9bb0ca952334c0798a7d265da88257c2 Mon Sep 17 00:00:00 2001 From: Marshall McDonnell Date: Wed, 6 Jan 2021 08:47:00 -0500 Subject: [PATCH 01/16] Add bandit dev dependency --- Pipfile | 3 ++- requirements-dev.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index 07de281..40b746d 100644 --- a/Pipfile +++ b/Pipfile @@ -11,10 +11,11 @@ autopep8 = "*" tox = "*" notebook = "*" sphinx-rtd-theme = "*" +bandit = "*" [packages] numpy = "*" -pystog = {editable = true,path = "."} +pystog = {path = "."} [requires] python_version = "3.6" diff --git a/requirements-dev.txt b/requirements-dev.txt index 8cad28b..16be0d3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ +bandit mock pytest flake8 From 643e400fa6f5040a81d6e01c6cd9bc829a583b6a Mon Sep 17 00:00:00 2001 From: Marshall McDonnell Date: Wed, 6 Jan 2021 08:47:12 -0500 Subject: [PATCH 02/16] Add lint-security to tox --- tox.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tox.ini b/tox.ini index a0d3597..84ad303 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,10 @@ commands = deps = flake8 commands = flake8 pystog/ tests/ setup.py --count +[testenv:lint-security] +deps = bandit +commands = bandit -r pystog/ -x pystog/_version.py + [testenv:coverage] deps = pytest-cov commands = pytest --cov=pystog --cov-report=term-missing tests/ From 2b00706e2e1a2b05bfb3fed65f842eb55e510bd8 Mon Sep 17 00:00:00 2001 From: Marshall McDonnell Date: Wed, 6 Jan 2021 08:47:25 -0500 Subject: [PATCH 03/16] Add lint-security to CI --- .github/workflows/actions.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index ddfe31c..790b15e 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -37,7 +37,9 @@ jobs: - name: Lint with Tox if: ${{ matrix.python-version == env.PYTHON_MAIN_VERSION }} shell: bash -l {0} - run: tox -e lint + run: | + tox -e lint + tox -e lint-security - name: Test with Tox shell: bash -l {0} From d9fffd503ca95bcbe3480c696d8c373b2c3002ed Mon Sep 17 00:00:00 2001 From: Marshall McDonnell Date: Wed, 6 Jan 2021 08:57:06 -0500 Subject: [PATCH 04/16] Move NoInputFilesException to pystog.stog + fix asserts --- pystog/cli.py | 5 +---- pystog/stog.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pystog/cli.py b/pystog/cli.py index 2069e98..3248ff9 100755 --- a/pystog/cli.py +++ b/pystog/cli.py @@ -11,13 +11,10 @@ import json from pystog import StoG +from pystog.stog import NoInputFilesException from pystog.io import get_cli_parser, parse_cli_args -class NoInputFilesException(Exception): - """Exception when no files are given to process""" - - def pystog_cli(kwargs=None): """ Main entry point for PyStoG CLI tool diff --git a/pystog/stog.py b/pystog/stog.py index 10277d4..3ceb7da 100644 --- a/pystog/stog.py +++ b/pystog/stog.py @@ -17,6 +17,10 @@ from pystog.fourier_filter import FourierFilter +class NoInputFilesException(Exception): + """Exception when no files are given to process""" + + class StoG(object): """ The StoG class is used to put together @@ -801,9 +805,14 @@ def read_all_data(self, **kwargs): the **sq_individuals** numpy storage array in **add_dataset** method via **read_dataset** method. """ - assert self.files is not None - assert len(self.files) != 0 + # Check that we have files to operate on + if not self.files: + raise NoInputFilesException("No input files given in arguments") + + if len(self.files) == 0: + raise NoInputFilesException("No input files given in arguments") + # Read in all the data files for i, file_info in enumerate(self.files): self.read_dataset(file_info, **kwargs) From 9cdd38d2660ed08673c02cb9e3a5dcb0f9b67ad2 Mon Sep 17 00:00:00 2001 From: Marshall McDonnell Date: Wed, 6 Jan 2021 09:04:47 -0500 Subject: [PATCH 05/16] Fix exception tests for pystog.stog --- tests/test_stog.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_stog.py b/tests/test_stog.py index 26f0bc6..11ca36c 100644 --- a/tests/test_stog.py +++ b/tests/test_stog.py @@ -4,11 +4,14 @@ import unittest from tests.utils import \ - get_data_path, load_data, get_index_of_function + get_data_path, \ + get_index_of_function, \ + load_data from tests.materials import Argon from pystog.utils import \ - RealSpaceHeaders, ReciprocalSpaceHeaders -from pystog.stog import StoG + RealSpaceHeaders, \ + ReciprocalSpaceHeaders +from pystog.stog import NoInputFilesException, StoG class TestStogBase(unittest.TestCase): @@ -555,11 +558,11 @@ def test_stog_read_dataset(self): def test_stog_read_all_data_assertion(self): stog = StoG() - with self.assertRaises(AssertionError): + with self.assertRaises(NoInputFilesException): stog.read_all_data() stog.files = list() - with self.assertRaises(AssertionError): + with self.assertRaises(NoInputFilesException): stog.read_all_data() def test_stog_read_all_data_for_files_length(self): From 0d5cc6f80eab1a2bf73745d17a0dc6df2009e8aa Mon Sep 17 00:00:00 2001 From: Marshall McDonnell Date: Wed, 6 Jan 2021 09:46:42 -0500 Subject: [PATCH 06/16] Added tests for pystog.converter when no dfq given --- tests/test_converter.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/tests/test_converter.py b/tests/test_converter.py index cd925fa..ca140cf 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -280,10 +280,18 @@ def F_to_S(self): atol=self.atol) assert_allclose(dsq, np.ones_like(self.q) / self.q) + def F_to_S_with_no_dfq(self): + sq, dsq = self.converter.F_to_S(self.q, self.fq, **self.kwargs) + assert_allclose( + sq[self.first:self.last], + self.sq_target, + rtol=self.rtol, + atol=self.atol) + assert_allclose(dsq, np.zeros_like(dsq)) + def F_to_FK(self): - fq_keen, dfq_keen = self.converter.F_to_FK(self.q, self.fq, - np.ones_like(self.q), - **self.kwargs) + fq_keen, dfq_keen = self.converter.F_to_FK( + self.q, self.fq, np.ones_like(self.q), **self.kwargs) assert_allclose( fq_keen[self.first:self.last], self.fq_keen_target, @@ -293,6 +301,16 @@ def F_to_FK(self): dfq_keen, np.ones_like(self.q) * self.kwargs['^2'] / self.q) + def F_to_FK_with_no_dfq(self): + fq_keen, dfq_keen = self.converter.F_to_FK( + self.q, self.fq, **self.kwargs) + assert_allclose( + fq_keen[self.first:self.last], + self.fq_keen_target, + rtol=self.rtol, + atol=self.atol) + assert_allclose(dfq_keen, np.zeros_like(dfq_keen)) + def F_to_DCS(self): dcs, ddcs = self.converter.F_to_DCS(self.q, self.fq, np.ones_like(self.q), @@ -395,9 +413,15 @@ def test_S_to_DCS(self): def test_F_to_S(self): self.F_to_S() + def test_F_to_S_with_no_dfq(self): + self.F_to_S_with_no_dfq() + def test_F_to_FK(self): self.F_to_FK() + def test_F_to_FK_with_no_dfq(self): + self.F_to_FK_with_no_dfq() + def test_F_to_DCS(self): self.F_to_DCS() @@ -438,9 +462,15 @@ def test_S_to_DCS(self): def test_F_to_S(self): self.F_to_S() + def test_F_to_S_with_no_dfq(self): + self.F_to_S_with_no_dfq() + def test_F_to_FK(self): self.F_to_FK() + def test_F_to_FK_with_no_dfq(self): + self.F_to_FK_with_no_dfq() + def test_F_to_DCS(self): self.F_to_DCS() From cfe242ded38858ee6b714e8aa48c0f2577006d9d Mon Sep 17 00:00:00 2001 From: Marshall McDonnell Date: Wed, 6 Jan 2021 09:48:40 -0500 Subject: [PATCH 07/16] Change variables names from df in pystog.stog --- pystog/stog.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pystog/stog.py b/pystog/stog.py index 3ceb7da..e3c1218 100644 --- a/pystog/stog.py +++ b/pystog/stog.py @@ -547,8 +547,8 @@ def reciprocal_individuals(self): return self.__reciprocal_individuals @reciprocal_individuals.setter - def reciprocal_individuals(self, df): - self.__reciprocal_individuals = df + def reciprocal_individuals(self, individuals): + self.__reciprocal_individuals = individuals @property def sq_individuals(self): @@ -566,8 +566,8 @@ def sq_individuals(self): return self.__sq_individuals @sq_individuals.setter - def sq_individuals(self, df): - self.__sq_individuals = df + def sq_individuals(self, individuals): + self.__sq_individuals = individuals @property def sq_master(self): @@ -584,8 +584,8 @@ def sq_master(self): return self.__sq_master @sq_master.setter - def sq_master(self, df): - self.__sq_master = df + def sq_master(self, sq): + self.__sq_master = sq @property def gr_master(self): @@ -602,8 +602,8 @@ def gr_master(self): return self.__gr_master @gr_master.setter - def gr_master(self, df): - self.__gr_master = df + def gr_master(self, gr): + self.__gr_master = gr @property def q_master(self): @@ -620,8 +620,8 @@ def q_master(self): return self.__q_master @q_master.setter - def q_master(self, df): - self.__q_master = df + def q_master(self, q): + self.__q_master = q @property def r_master(self): @@ -638,8 +638,8 @@ def r_master(self): return self.__r_master @r_master.setter - def r_master(self, df): - self.__r_master = df + def r_master(self, r): + self.__r_master = r @property def real_space_function(self): From adb1ca7e47655685f3bc564634d009c100625584 Mon Sep 17 00:00:00 2001 From: Marshall McDonnell Date: Wed, 6 Jan 2021 10:28:52 -0500 Subject: [PATCH 08/16] Removed useless line from pystog.stog --- pystog/stog.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pystog/stog.py b/pystog/stog.py index e3c1218..157931e 100644 --- a/pystog/stog.py +++ b/pystog/stog.py @@ -809,9 +809,6 @@ def read_all_data(self, **kwargs): if not self.files: raise NoInputFilesException("No input files given in arguments") - if len(self.files) == 0: - raise NoInputFilesException("No input files given in arguments") - # Read in all the data files for i, file_info in enumerate(self.files): self.read_dataset(file_info, **kwargs) From 51612e1d65bba0756a726d569576081a69b19cdb Mon Sep 17 00:00:00 2001 From: Marshall McDonnell Date: Wed, 6 Jan 2021 10:31:31 -0500 Subject: [PATCH 09/16] Added setter tests for r and q master --- tests/test_stog.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_stog.py b/tests/test_stog.py index 11ca36c..acc18a4 100644 --- a/tests/test_stog.py +++ b/tests/test_stog.py @@ -154,8 +154,10 @@ def test_stog_init(self): "Offset": 0.0, "Scale": 1.0}}) self.assertEqual(stog.stem_name, "out") self.assertEqual(stog.reciprocal_individuals.size, 0) + self.assertEqual(stog.q_master, {}) self.assertEqual(stog.sq_master, {}) self.assertEqual(stog.sq_individuals.size, 0) + self.assertEqual(stog.r_master, {}) self.assertEqual(stog.gr_master, {}) def test_stog_init_kwargs_files(self): @@ -342,11 +344,21 @@ def test_stog_sq_individuals_setter(self): stog.sq_individuals = self.target np.testing.assert_allclose(stog.sq_individuals, self.target) + def test_stog_q_master_setter(self): + stog = StoG() + stog.q_master = self.target + np.testing.assert_allclose(stog.q_master, self.target) + def test_stog_sq_master_setter(self): stog = StoG() stog.sq_master = self.target np.testing.assert_allclose(stog.sq_master, self.target) + def test_stog_r_master_setter(self): + stog = StoG() + stog.r_master = self.target + np.testing.assert_allclose(stog.r_master, self.target) + def test_stog_gr_master_setter(self): stog = StoG() stog.gr_master = self.target From ef10bebd12a97544fd7612a9182d0896f0cbdc03 Mon Sep 17 00:00:00 2001 From: Marshall McDonnell Date: Wed, 6 Jan 2021 10:52:28 -0500 Subject: [PATCH 10/16] Add exception tests for read_dataset --- tests/test_stog.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_stog.py b/tests/test_stog.py index acc18a4..4fca0d2 100644 --- a/tests/test_stog.py +++ b/tests/test_stog.py @@ -568,6 +568,18 @@ def test_stog_read_dataset(self): self.sq_target[0], places=places) + def test_stog_read_dataset_xcol_data_format_exception(self): + stog = StoG() + filename = get_data_path(self.material.reciprocal_space_filename) + with self.assertRaises(RuntimeError): + stog.read_dataset({'Filename': filename}, xcol=99) + + def test_stog_read_dataset_ycol_data_format_exception(self): + stog = StoG() + filename = get_data_path(self.material.reciprocal_space_filename) + with self.assertRaises(RuntimeError): + stog.read_dataset({'Filename': filename}, ycol=99) + def test_stog_read_all_data_assertion(self): stog = StoG() with self.assertRaises(NoInputFilesException): From 93e7f64713ed9e9d183774e114ece01cc9068cfb Mon Sep 17 00:00:00 2001 From: Marshall McDonnell Date: Wed, 6 Jan 2021 13:43:36 -0500 Subject: [PATCH 11/16] Switched apply_scales_and_offset to static method --- Pipfile | 2 +- pystog/stog.py | 50 ++++++++++++-------------------------------------- 2 files changed, 13 insertions(+), 39 deletions(-) diff --git a/Pipfile b/Pipfile index 40b746d..ab3bd21 100644 --- a/Pipfile +++ b/Pipfile @@ -15,7 +15,7 @@ bandit = "*" [packages] numpy = "*" -pystog = {path = "."} +pystog = {editable = true,path = "."} [requires] python_version = "3.6" diff --git a/pystog/stog.py b/pystog/stog.py index 157931e..31fc1c9 100644 --- a/pystog/stog.py +++ b/pystog/stog.py @@ -830,10 +830,12 @@ def read_dataset( :param info: Dict with information for dataset (filename, manipulations, etc.) :type info: dict - :param xcol: The column in the data file that contains the X-axis + :param xcol: Column in data file for X-axis :type xcol: int - :param ycol: The column in the data file that contains the Y-axis + :param ycol: Column in data file for Y-axis :type ycol: int + :param dycol: Column in data file for Y uncertainty + :type dycol: int :param sep: Separator for the file used by numpy.loadtxt :type sep: raw string :param skiprows: Number of rows to skip. Passed to numpy.loadtxt @@ -911,7 +913,7 @@ def add_dataset( xoffset = info['X']['Offset'] if adjusting: - x, y, dy = self._apply_scales_and_offset(x, y, dy=dy, + x, y, dy = self.apply_scales_and_offset(x, y, dy=dy, yscale=yscale, yoffset=yoffset, xoffset=xoffset) @@ -959,8 +961,8 @@ def add_dataset( array_seq = (self.sq_individuals, np.stack((x, y, dy))) self.sq_individuals = np.concatenate(array_seq, axis=1) - def _apply_scales_and_offset( - self, + @staticmethod + def apply_scales_and_offset( x, y, dy=None, @@ -983,42 +985,14 @@ def _apply_scales_and_offset( :return: X and Y vectors after scales and offsets applied :rtype: numpy.array pair """ - y = self._scale(y, yscale) - y = self._offset(y, yoffset) - x = self._offset(x, xoffset) + y = y * yscale + y = y + yoffset + x = x + xoffset if dy is None: dy = np.zeros_like(y) - dy = self._scale(dy, yscale) + dy = dy * yscale return x, y, dy - def _offset(self, data, offset): - """ - Applies offset to data - - :param data: Input data - :type data: numpy.array or list - :param offset: Offset to apply to data - :type offset: float - :return: Data with offset applied - :rtype: numpy.array - """ - data = data + offset - return data - - def _scale(self, data, scale): - """ - Applies scale to data - - :param data: Input data - :type data: numpy.array or list - :param offset: Scale to apply to data - :type offset: float - :return: Data with scale applied - :rtype: numpy.array - """ - data = scale * data - return data - def merge_data(self): """ Merges the reciprocal space data stored in the @@ -1101,7 +1075,7 @@ def merge_data(self): sq = data_merged[1] dsq = data_merged[2] - q, sq, dsq = self._apply_scales_and_offset( + q, sq, dsq = self.apply_scales_and_offset( q, sq, yscale=self.merged_opts['Y']['Scale'], yoffset=self.merged_opts['Y']['Offset'], From 586a6997dfce23d359919aee10706e996303c376 Mon Sep 17 00:00:00 2001 From: Marshall McDonnell Date: Wed, 6 Jan 2021 13:44:12 -0500 Subject: [PATCH 12/16] Added tests for apply_scales_and_offset --- tests/test_stog.py | 141 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/tests/test_stog.py b/tests/test_stog.py index 4fca0d2..ad503e5 100644 --- a/tests/test_stog.py +++ b/tests/test_stog.py @@ -39,6 +39,7 @@ def initialize_material(self): # targets for 1st peaks self.sq_target = self.material.sq_target + self.dsq_target = 3.087955 self.fq_target = self.material.fq_target self.fq_keen_target = self.material.fq_keen_target self.dcs_target = self.material.dcs_target @@ -386,6 +387,39 @@ class TestStogDatasetSpecificMethods(TestStogBase): def setUp(self): super(TestStogDatasetSpecificMethods, self).setUp() + def test_stog_apply_scales_and_offset(self): + q, sq, dq = StoG.apply_scales_and_offset(self.q, self.sq) + np.testing.assert_allclose(q, self.q) + np.testing.assert_allclose(sq, self.sq) + np.testing.assert_allclose(dq, np.zeros_like(sq)) + + def test_stog_apply_scales_and_offset_with_dy(self): + q, sq, dq = StoG.apply_scales_and_offset(self.q, self.sq, dy=self.sq) + np.testing.assert_allclose(q, self.q) + np.testing.assert_allclose(sq, self.sq) + np.testing.assert_allclose(dq, self.sq) + + def test_stog_apply_scales_and_offset_with_yscale(self): + q, sq, dq = StoG.apply_scales_and_offset( + self.q, self.sq, dy=self.sq, yscale=2.0) + np.testing.assert_allclose(q, self.q) + np.testing.assert_allclose(sq, self.sq * 2.0) + np.testing.assert_allclose(dq, self.sq * 2.0) + + def test_stog_apply_scales_and_offset_with_yoffset(self): + q, sq, dq = StoG.apply_scales_and_offset( + self.q, self.sq, dy=self.sq, yoffset=2.0) + np.testing.assert_allclose(q, self.q) + np.testing.assert_allclose(sq, self.sq + 2.0) + np.testing.assert_allclose(dq, self.sq) + + def test_stog_apply_scales_and_offset_with_yoffset(self): + q, sq, dq = StoG.apply_scales_and_offset( + self.q, self.sq, dy=self.sq, xoffset=2.0) + np.testing.assert_allclose(q, self.q + 2.0) + np.testing.assert_allclose(sq, self.sq) + np.testing.assert_allclose(dq, self.sq) + def test_stog_add_dataset(self): # Number of decimal places for precision places = 5 @@ -406,10 +440,12 @@ def test_stog_add_dataset(self): stog.reciprocal_individuals[1][self.first], self.sq_target[0], places=places) + self.assertEqual(stog.reciprocal_individuals[2][self.first], 0.0) self.assertAlmostEqual( stog.sq_individuals[1][self.first], self.sq_target[0], places=places) + self.assertEqual(stog.sq_individuals[2][self.first], 0.0) # Add the Q[S(Q)-1] data set and check values for it and S(Q) against # targets @@ -419,14 +455,17 @@ def test_stog_add_dataset(self): 'ReciprocalFunction': 'Q[S(Q)-1]' } stog.add_dataset(info) + self.assertAlmostEqual( stog.reciprocal_individuals[1][self.first + stride], self.fq_target[0], places=places) + self.assertEqual(stog.reciprocal_individuals[2][self.first + stride], 0.0) self.assertAlmostEqual( stog.sq_individuals[1][self.first], self.sq_target[0], places=places) + self.assertEqual(stog.sq_individuals[2][self.first], 0.0) # Add the FK(Q) data set and check values for it and S(Q) against # targets @@ -439,10 +478,12 @@ def test_stog_add_dataset(self): stog.reciprocal_individuals[1][self.first + stride], self.fq_keen_target[0], places=places) + self.assertEqual(stog.reciprocal_individuals[2][self.first + stride], 0.0) self.assertAlmostEqual( stog.sq_individuals[1][self.first], self.sq_target[0], places=places) + self.assertEqual(stog.sq_individuals[2][self.first], 0.0) # Add the DCS(Q) data set and check values for it and S(Q) against # targets @@ -455,10 +496,12 @@ def test_stog_add_dataset(self): stog.reciprocal_individuals[1][self.first + stride], self.dcs_target[0], places=places) + self.assertEqual(stog.reciprocal_individuals[2][self.first + stride], 0.0) self.assertAlmostEqual( stog.sq_individuals[1][self.first], self.sq_target[0], places=places) + self.assertEqual(stog.sq_individuals[2][self.first], 0.0) def test_stog_add_dataset_yscale(self): # Scale S(Q) and make sure it does not equal original target values @@ -471,9 +514,11 @@ def test_stog_add_dataset_yscale(self): self.assertNotEqual( stog.reciprocal_individuals[1][self.first], self.sq_target[0]) + self.assertEqual(stog.reciprocal_individuals[2][self.first], 0.0) self.assertNotEqual( stog.sq_individuals[1][self.first], self.sq_target[0]) + self.assertEqual(stog.sq_individuals[2][self.first], 0.0) def test_stog_add_dataset_yoffset(self): # Offset S(Q) and make sure it does not equal original target values @@ -486,10 +531,60 @@ def test_stog_add_dataset_yoffset(self): self.assertNotEqual( stog.reciprocal_individuals[1][self.first], self.sq_target[0]) + self.assertEqual(stog.reciprocal_individuals[2][self.first], 0.0) + self.assertNotEqual( + stog.sq_individuals[1][self.first], + self.sq_target[0]) + self.assertEqual(stog.sq_individuals[2][self.first], 0.0) + + def test_stog_add_dataset_yscale_with_dy(self): + # Scale S(Q) and make sure it does not equal original target values + stog = StoG() + info = { + 'data': [self.q, self.sq, self.sq], + 'ReciprocalFunction': 'S(Q)', + 'Y': {'Scale': 2.0}} + stog.add_dataset(info) + + self.assertNotEqual( + stog.reciprocal_individuals[1][self.first], + self.sq_target[0]) + + dsq_target = info['Y']['Scale'] * 2.59173 + self.assertEqual( + stog.reciprocal_individuals[2][self.first], + dsq_target) + + self.assertNotEqual( + stog.sq_individuals[1][self.first], + self.sq_target[0]) + + self.assertEqual(stog.sq_individuals[2][self.first], dsq_target) + + def test_stog_add_dataset_yoffset_with_dy(self): + # Offset S(Q) and make sure it does not equal original target values + stog = StoG() + info = { + 'data': [self.q, self.sq, self.sq], + 'ReciprocalFunction': 'S(Q)', + 'Y': {'Offset': 2.0}} + stog.add_dataset(info) + + self.assertNotEqual( + stog.reciprocal_individuals[1][self.first], + self.sq_target[0]) + + dsq_target = 2.59173 + self.assertEqual( + stog.reciprocal_individuals[2][self.first], + dsq_target) + self.assertNotEqual( stog.sq_individuals[1][self.first], self.sq_target[0]) + self.assertEqual(stog.sq_individuals[2][self.first], dsq_target) + def test_stog_add_dataset_xoffset(self): # Offset Q from 1.96 -> 2.14 stog = StoG() @@ -520,10 +615,12 @@ def test_stog_add_dataset_default_reciprocal_space_function(self): self.assertEqual( stog.reciprocal_individuals[0][self.first], self.reciprocal_xtarget) + self.assertEqual(stog.reciprocal_individuals[2][self.first], 0.0) self.assertAlmostEqual( stog.reciprocal_individuals[1][self.first], self.sq_target[0], places=places) + self.assertEqual(stog.sq_individuals[2][self.first], 0.0) def test_stog_add_dataset_wrong_reciprocal_space_function_exception(self): # Check qmin and qmax apply cropping @@ -563,10 +660,16 @@ def test_stog_read_dataset(self): stog.reciprocal_individuals[1][self.first], self.sq_target[0], places=places) + self.assertEqual( + stog.reciprocal_individuals[2][self.first], + self.dsq_target) self.assertAlmostEqual( stog.sq_individuals[1][self.first], self.sq_target[0], places=places) + self.assertEqual( + stog.sq_individuals[2][self.first], + self.dsq_target) def test_stog_read_dataset_xcol_data_format_exception(self): stog = StoG() @@ -580,6 +683,44 @@ def test_stog_read_dataset_ycol_data_format_exception(self): with self.assertRaises(RuntimeError): stog.read_dataset({'Filename': filename}, ycol=99) + def test_stog_read_dataset_dycol_too_large(self): + # Number of decimal places for precision + places = 5 + + # Load S(Q) for Argon from test data + stog = StoG(**{'^2': self.kwargs['^2'], + '': self.kwargs['']}) + info = { + 'Filename': get_data_path( + self.material.reciprocal_space_filename), + 'ReciprocalFunction': 'S(Q)', + 'Qmin': 0.02, + 'Qmax': 35.2, + 'Y': { + 'Offset': 0.0, + 'Scale': 1.0}, + 'X': { + 'Offset': 0.0}} + + info['index'] = 0 + stog.read_dataset(info, dycol=99) + + # Check S(Q) data against targets + self.assertEqual( + stog.reciprocal_individuals[0][self.first], + self.reciprocal_xtarget) + self.assertAlmostEqual( + stog.reciprocal_individuals[1][self.first], + self.sq_target[0], + places=places) + self.assertEqual( + stog.reciprocal_individuals[2][self.first], + 0.0) + self.assertAlmostEqual( + stog.sq_individuals[1][self.first], + self.sq_target[0], + places=places) + def test_stog_read_all_data_assertion(self): stog = StoG() with self.assertRaises(NoInputFilesException): From 7d5885b711f105c68dc28cbfbecfbae4fd08b59c Mon Sep 17 00:00:00 2001 From: Marshall McDonnell Date: Wed, 6 Jan 2021 13:51:11 -0500 Subject: [PATCH 13/16] Fix lint errors --- pystog/stog.py | 9 +++++---- tests/test_stog.py | 14 ++++++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/pystog/stog.py b/pystog/stog.py index 31fc1c9..256bf5c 100644 --- a/pystog/stog.py +++ b/pystog/stog.py @@ -913,10 +913,11 @@ def add_dataset( xoffset = info['X']['Offset'] if adjusting: - x, y, dy = self.apply_scales_and_offset(x, y, dy=dy, - yscale=yscale, - yoffset=yoffset, - xoffset=xoffset) + x, y, dy = self.apply_scales_and_offset( + x, y, dy=dy, + yscale=yscale, + yoffset=yoffset, + xoffset=xoffset) # Save overal x-axis min and max self.xmin = min(self.xmin, xmin) diff --git a/tests/test_stog.py b/tests/test_stog.py index ad503e5..b57daa5 100644 --- a/tests/test_stog.py +++ b/tests/test_stog.py @@ -413,7 +413,7 @@ def test_stog_apply_scales_and_offset_with_yoffset(self): np.testing.assert_allclose(sq, self.sq + 2.0) np.testing.assert_allclose(dq, self.sq) - def test_stog_apply_scales_and_offset_with_yoffset(self): + def test_stog_apply_scales_and_offset_with_xoffset(self): q, sq, dq = StoG.apply_scales_and_offset( self.q, self.sq, dy=self.sq, xoffset=2.0) np.testing.assert_allclose(q, self.q + 2.0) @@ -460,7 +460,9 @@ def test_stog_add_dataset(self): stog.reciprocal_individuals[1][self.first + stride], self.fq_target[0], places=places) - self.assertEqual(stog.reciprocal_individuals[2][self.first + stride], 0.0) + self.assertEqual( + stog.reciprocal_individuals[2][self.first + stride], + 0.0) self.assertAlmostEqual( stog.sq_individuals[1][self.first], self.sq_target[0], @@ -478,7 +480,9 @@ def test_stog_add_dataset(self): stog.reciprocal_individuals[1][self.first + stride], self.fq_keen_target[0], places=places) - self.assertEqual(stog.reciprocal_individuals[2][self.first + stride], 0.0) + self.assertEqual( + stog.reciprocal_individuals[2][self.first + stride], + 0.0) self.assertAlmostEqual( stog.sq_individuals[1][self.first], self.sq_target[0], @@ -496,7 +500,9 @@ def test_stog_add_dataset(self): stog.reciprocal_individuals[1][self.first + stride], self.dcs_target[0], places=places) - self.assertEqual(stog.reciprocal_individuals[2][self.first + stride], 0.0) + self.assertEqual( + stog.reciprocal_individuals[2][self.first + stride], + 0.0) self.assertAlmostEqual( stog.sq_individuals[1][self.first], self.sq_target[0], From 1d50f46d303dca6297dde167d8278de927569be9 Mon Sep 17 00:00:00 2001 From: Marshall McDonnell Date: Wed, 6 Jan 2021 14:18:06 -0500 Subject: [PATCH 14/16] Added initial pystog_cli test --- tests/test_cli.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tests/test_cli.py diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..2244c6b --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,8 @@ +import pytest +from pystog.cli import pystog_cli +from pystog.stog import NoInputFilesException + +def test_pystog_cli_no_files_exception(): + with pytest.raises(NoInputFilesException): + pystog_cli({"cat": "meow"}) + From e780f488abea16209c57d92b145a931a24d363fd Mon Sep 17 00:00:00 2001 From: Marshall McDonnell Date: Wed, 6 Jan 2021 14:18:32 -0500 Subject: [PATCH 15/16] Fix need for parser in pystog cli if no Files --- pystog/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pystog/cli.py b/pystog/cli.py index 3248ff9..071f8ec 100755 --- a/pystog/cli.py +++ b/pystog/cli.py @@ -23,8 +23,9 @@ def pystog_cli(kwargs=None): If None, parsed from command line via get_cli_parser :type kwargs: dict """ + parser = get_cli_parser() + if not kwargs: - parser = get_cli_parser() args = parser.parse_args() if args.json: print("loading config from '%s'" % args.json) From c02aaee841b1c4c929168b8569d1b6e1db482db5 Mon Sep 17 00:00:00 2001 From: Marshall McDonnell Date: Wed, 6 Jan 2021 14:19:09 -0500 Subject: [PATCH 16/16] Fix lint errors in cli test --- tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2244c6b..beb2f44 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,7 +2,7 @@ from pystog.cli import pystog_cli from pystog.stog import NoInputFilesException + def test_pystog_cli_no_files_exception(): with pytest.raises(NoInputFilesException): pystog_cli({"cat": "meow"}) -