From d405e8839cf6279837e9740a095f0c5082cca5d9 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 22 Feb 2022 15:05:30 +0000 Subject: [PATCH 1/4] Add timestamps to test logging --- tests/conftest.py | 9 ++++++++- tests/test_record_values.py | 16 ++++++++-------- tests/test_records.py | 25 +++++++++++++------------ 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3d30eb64..e0b6729a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +from datetime import datetime import atexit import os import random @@ -20,7 +21,7 @@ WAVEFORM_LENGTH = 40 # Default timeout for many operations across testing -TIMEOUT = 5 # Seconds +TIMEOUT = 10 # Seconds # Address for multiprocessing Listener/Client pair ADDRESS = ("localhost", 2345) @@ -29,6 +30,12 @@ def create_random_prefix(): """Create 12-character random string, for generating unique Device Names""" return "".join(random.choice(string.ascii_uppercase) for _ in range(12)) +# Can't use logging as it's not multiprocess safe, and +# alteratives are overkill +def log(*args): + print(datetime.now().strftime("%H:%M:%S"), *args) + + class SubprocessIOC: def __init__(self, ioc_py): self.pv_prefix = create_random_prefix() diff --git a/tests/test_record_values.py b/tests/test_record_values.py index 15f77b90..f1dbab7c 100644 --- a/tests/test_record_values.py +++ b/tests/test_record_values.py @@ -9,6 +9,7 @@ from conftest import ( requires_cothread, WAVEFORM_LENGTH, + log, select_and_recv, TIMEOUT ) @@ -30,7 +31,6 @@ "string and so lets test it and prove that shall we?" - def record_func_names(fixture_value): """Provide a nice name for the record_func fixture""" return fixture_value.__name__ @@ -461,7 +461,7 @@ def run_test_function( creation_func in [builder.WaveformOut, builder.WaveformIn] and expected_value.dtype in [numpy.float64, numpy.int32] ): - print( + log( "caget cannot distinguish between a waveform with 1 " "element and a scalar value, and so always returns a " "scalar. Therefore we skip this check.") @@ -760,16 +760,16 @@ def none_value_test_func(self, record_func, queue): builder.LoadDatabase() softioc.iocInit(dispatcher) - print("CHILD: Soft IOC started, about to .set(None)") + log("CHILD: Soft IOC started, about to .set(None)") try: record.set(None) - print("CHILD: Uh-OH! No exception thrown when setting None!") + log("CHILD: Uh-OH! No exception thrown when setting None!") except Exception as e: - print("CHILD: Putting exception into queue", e) + log("CHILD: Putting exception into queue %s", e) queue.put(e) else: - print("CHILD: No exception raised when using None as value!") + log("CHILD: No exception raised when using None as value!") queue.put(Exception("FAIL: No exception raised during .set()")) @requires_cothread @@ -785,14 +785,14 @@ def test_value_none_rejected_set_after_init(self, record_func_reject_none): process.start() - print("PARENT: Child process started, waiting for returned exception") + log("PARENT: Child process started, waiting for returned exception") try: exception = queue.get(timeout=TIMEOUT) assert isinstance(exception, self.expected_exceptions) finally: - print("PARENT: Issuing terminate to child process") + log("PARENT: Issuing terminate to child process") process.terminate() process.join(timeout=TIMEOUT) if process.exitcode is None: diff --git a/tests/test_records.py b/tests/test_records.py index 210f6327..ebb54da7 100644 --- a/tests/test_records.py +++ b/tests/test_records.py @@ -4,6 +4,7 @@ import pytest from conftest import ( + log, create_random_prefix, requires_cothread, _clear_records, @@ -376,14 +377,14 @@ def on_update_done(_): conn.send("R") # "Ready" - print("CHILD: Sent R over Connection to Parent") + log("CHILD: Sent R over Connection to Parent") # Keep process alive while main thread runs CAGET if conn.poll(TIMEOUT): val = conn.recv() assert val == "D", "Did not receive expected Done character" - print("CHILD: Received exit command, child exiting") + log("CHILD: Received exit command, child exiting") def on_update_runner(self, creation_func, always_update, put_same_value): parent_conn, child_conn = multiprocessing.Pipe() @@ -397,7 +398,7 @@ def on_update_runner(self, creation_func, always_update, put_same_value): process.start() - print("PARENT: Child started, waiting for R command") + log("PARENT: Child started, waiting for R command") from cothread.catools import caget, caput, _channel_cache @@ -405,7 +406,7 @@ def on_update_runner(self, creation_func, always_update, put_same_value): # Wait for message that IOC has started select_and_recv(parent_conn, "R") - print("PARENT: received R command") + log("PARENT: received R command") # Suppress potential spurious warnings @@ -416,7 +417,7 @@ def on_update_runner(self, creation_func, always_update, put_same_value): # value to force processing to occur count = 1 - print("PARENT: begining While loop") + log("PARENT: begining While loop") while count < 4: put_ret = caput( @@ -426,11 +427,11 @@ def on_update_runner(self, creation_func, always_update, put_same_value): ) assert put_ret.ok, f"caput did not succeed: {put_ret.errorcode}" - print(f"PARENT: completed caput with count {count}") + log(f"PARENT: completed caput with count {count}") count += 1 - print("PARENT: Put'ing to DONE record") + log("PARENT: Put'ing to DONE record") caput( device_name + ":ON-UPDATE-DONE", @@ -438,14 +439,14 @@ def on_update_runner(self, creation_func, always_update, put_same_value): wait=True, ) - print("PARENT: Waiting for C command") + log("PARENT: Waiting for C command") # Wait for action record to process, so we know all the callbacks # have finished processing (This assumes record callbacks are not # re-ordered, and will run in the same order as the caputs we sent) select_and_recv(parent_conn, "C") - print("PARENT: Received C command") + log("PARENT: Received C command") ret_val = caget( device_name + ":ON-UPDATE-COUNTER-RECORD", @@ -454,7 +455,7 @@ def on_update_runner(self, creation_func, always_update, put_same_value): assert ret_val.ok, \ f"caget did not succeed: {ret_val.errorcode}, {ret_val}" - print(f"PARENT: Received val from COUNTER: {ret_val}") + log(f"PARENT: Received val from COUNTER: {ret_val}") # Expected value is either 3 (incremented once per caput) @@ -469,10 +470,10 @@ def on_update_runner(self, creation_func, always_update, put_same_value): # Suppress potential spurious warnings _channel_cache.purge() - print("PARENT:Sending Done command to child") + log("PARENT:Sending Done command to child") parent_conn.send("D") # "Done" process.join(timeout=TIMEOUT) - print(f"PARENT: Join completed with exitcode {process.exitcode}") + log(f"PARENT: Join completed with exitcode {process.exitcode}") if process.exitcode is None: pytest.fail("Process did not terminate") From 574d582ccb5f47fcc2ad63acabce1043f623d3ac Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 23 Feb 2022 11:41:34 +0000 Subject: [PATCH 2/4] Test sdist installs and works on all platforms --- .github/workflows/code.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 823f3194..29ef3609 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -86,6 +86,8 @@ jobs: CIBW_BUILD: ${{ matrix.python }}*64 CIBW_TEST_EXTRAS: dev CIBW_TEST_COMMAND: pytest {project}/tests --cov-report xml:${{ matrix.cov_file }} --junit-xml=${{ matrix.results_file }} + # Run with faulthandler and -s in the hope we get a stack trace on seg fault on windows... + CIBW_TEST_COMMAND_WINDOWS: python -X faulthandler -m pytest -s {project}/tests --cov-report xml:${{ matrix.cov_file }} --junit-xml=${{ matrix.results_file }} # Disable auditwheel as it isn't compatible with setuptools_dso approach # https://github.com/mdavidsaver/setuptools_dso/issues/17 CIBW_REPAIR_WHEEL_COMMAND: "" @@ -129,8 +131,28 @@ jobs: with: files: artifacts/**/*.xml - release: + sdist: needs: [build] + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python: [cp36, cp37, cp38, cp39, cp310] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/download-artifact@v2 + with: + name: dist + path: dist + + - name: Install sdist in a venv and check cli works + run: pipx run --spec dist/*.tar.gz pythonSoftIOC --version + shell: bash + + release: + needs: [build, sdist] runs-on: ubuntu-latest # upload to PyPI and make a release on every tag if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') From 4c1a9f07e625626936b887e636f082d1b72c5539 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 23 Feb 2022 15:32:00 +0000 Subject: [PATCH 3/4] Fix sdist included files --- setup.cfg | 9 +++++++++ setup.py | 18 +++--------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/setup.cfg b/setup.cfg index 3004b03d..3088277c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,6 +48,15 @@ dev = cothread; sys_platform != "win32" p4p +# Include the OS specific files in the sdist, even +# if not packages by setup.py (as we only build sdist on one arch) +[options.data_files] +iocStats/devIocStats = iocStats/devIocStats/*.c, iocStats/devIocStats/*.h +iocStats/devIocStats/os/default = iocStats/devIocStats/os/default/* +iocStats/devIocStats/os/Darwin = iocStats/devIocStats/os/Darwin/* +iocStats/devIocStats/os/WIN32 = iocStats/devIocStats/os/WIN32/* +iocStats/devIocStats/os/Linux = iocStats/devIocStats/os/Linux/* + [flake8] max-line-length = 80 extend-ignore = diff --git a/setup.py b/setup.py index c6305ab6..7d7bc88c 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,6 @@ "devIocStatsWaveform.c", "devIocStatsSub.c", "devIocStatsTest.c", - "devIocStats.h", ] devIocStats_OSD = [ @@ -39,30 +38,19 @@ "osdSystemInfo.c", "osdHostInfo.c", "osdPIDInfo.c", - "devIocStatsOSD.h", ] devIocStats_src = os.path.join("iocStats", "devIocStats") devIocStats_os = os.path.join(devIocStats_src, "os", get_config_var('OS_CLASS')) devIocStats_default = os.path.join(devIocStats_src, "os", "default") -def _add_file(f): - if f.endswith(".h"): - # Only add header files if making an sdist - # https://github.com/pypa/packaging-problems/issues/84#issuecomment-383718492 - should_add = "sdist" in sys.argv - else: - should_add = True - if should_add: - sources.append(f) - for f in devIocStats_OSI: - _add_file(os.path.join(devIocStats_src, f)) + sources.append(os.path.join(devIocStats_src, f)) for f in devIocStats_OSD: if os.path.exists(os.path.join(devIocStats_os, f)): - _add_file(os.path.join(devIocStats_os, f)) + sources.append(os.path.join(devIocStats_os, f)) else: - _add_file(os.path.join(devIocStats_default, f)) + sources.append(os.path.join(devIocStats_default, f)) # Extension with all our C code ext = Extension( From 10192bbf0ff335f61777ff0af3715edd5bac2647 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 23 Feb 2022 16:23:52 +0000 Subject: [PATCH 4/4] Separate out sdist job --- .github/workflows/code.yml | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 29ef3609..99fb4a82 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -30,6 +30,25 @@ jobs: - name: Lint run: flake8 + sdist: + runs-on: "ubuntu-latest" + steps: + - name: Checkout Source + uses: actions/checkout@v2 + with: + # require history to get back to last tag for version number of branches + fetch-depth: 0 + submodules: true + + - name: Build Sdist + run: pipx run build --sdist . + + - name: Upload Sdist + uses: actions/upload-artifact@v2 + with: + name: dist + path: dist/* + build: strategy: fail-fast: false @@ -50,9 +69,6 @@ jobs: - os: ubuntu-latest cov_file: /output/coverage.xml results_file: /output/pytest-results.xml - # Build an sdist on linux so it has the right line endings - - os: ubuntu-latest - sdist: true name: build/${{ matrix.os }}/${{ matrix.python }} runs-on: ${{ matrix.os }} @@ -74,10 +90,6 @@ jobs: # Pin cibuildwheel due to https://github.com/pypa/cibuildwheel/issues/962 run: pip install build cibuildwheel>=2.3.1 - - name: Build Sdist - if: matrix.sdist - run: python -m build --sdist . - - name: Build Wheel run: cibuildwheel --output-dir dist env: @@ -95,7 +107,7 @@ jobs: CIBW_ENVIRONMENT_LINUX: SETUPTOOLS_DSO_PLAT_NAME=manylinux2014_x86_64 CIBW_SKIP: "*-musllinux*" # epicscorelibs doesn't build on musllinux platforms - - name: Upload Wheel and Sdist + - name: Upload Wheel uses: actions/upload-artifact@v2 with: name: dist @@ -131,8 +143,8 @@ jobs: with: files: artifacts/**/*.xml - sdist: - needs: [build] + test-sdist: + needs: [sdist] strategy: fail-fast: false matrix: