From 3b0bef0b2d6cf9edd6759739b7bfdb4c1f658eee Mon Sep 17 00:00:00 2001 From: jsouter <107045742+jsouter@users.noreply.github.com> Date: Fri, 13 Sep 2024 11:46:52 +0100 Subject: [PATCH] Autosave support (#163) Add support for saving and restoring pv fields with autosave --- .github/workflows/code.yml | 38 +- CHANGELOG.rst | 2 +- Pipfile.lock | 90 +++++ docs/examples/example_autosave_ioc.py | 34 ++ docs/how-to/use-autosave-in-an-ioc.rst | 76 ++++ docs/index.rst | 1 + docs/reference/api.rst | 102 +++++ setup.py | 3 +- softioc/autosave.py | 335 +++++++++++++++++ softioc/builder.py | 2 + softioc/device.py | 7 +- softioc/pythonSoftIoc.py | 4 +- softioc/softioc.py | 3 +- tests/conftest.py | 8 +- tests/sim_records.py | 2 +- tests/test_autosave.py | 495 +++++++++++++++++++++++++ 16 files changed, 1181 insertions(+), 21 deletions(-) create mode 100644 docs/examples/example_autosave_ioc.py create mode 100644 docs/how-to/use-autosave-in-an-ioc.rst create mode 100644 softioc/autosave.py create mode 100644 tests/test_autosave.py diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 68d590bf..2d407c93 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -17,10 +17,10 @@ jobs: runs-on: "ubuntu-latest" steps: - name: Checkout Source - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Install Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: "3.7" @@ -34,7 +34,7 @@ jobs: runs-on: "ubuntu-latest" steps: - name: Checkout Source - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: # require history to get back to last tag for version number of branches fetch-depth: 0 @@ -44,7 +44,7 @@ jobs: run: pipx run build --sdist . - name: Upload Sdist - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: dist path: dist/* @@ -56,6 +56,11 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] python: [cp37, cp38, cp39, cp310] + exclude: + # MacOS 14.4.1 for arm64 doesn't support Python < 3.8 + - os: macos-latest + python: "cp37" + include: # Put coverage and results files in the project directory for mac - os: macos-latest @@ -69,22 +74,27 @@ jobs: - os: ubuntu-latest cov_file: /output/coverage.xml results_file: /output/pytest-results.xml + # MacOS 13 required for Python < 3.8 + - os: macos-13 + python: "cp37" + cov_file: "{project}/dist/coverage.xml" + results_file: "{project}/dist/pytest-results.xml" name: build/${{ matrix.os }}/${{ matrix.python }} runs-on: ${{ matrix.os }} steps: - name: Checkout Source - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: # require history to get back to last tag for version number of branches fetch-depth: 0 submodules: true - name: Install Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: "3.7" + python-version: "3.12" - name: Install Python Dependencies # Pin cibuildwheel due to https://github.com/pypa/cibuildwheel/issues/962 @@ -106,20 +116,20 @@ jobs: CIBW_SKIP: "*-musllinux*" # epicscorelibs doesn't build on musllinux platforms - name: Upload Wheel - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: - name: dist + name: dist-${{ matrix.os }}-${{ matrix.python }} path: dist/softioc* - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v4 with: name: ${{ matrix.os }}/${{ matrix.python }} directory: dist - name: Upload Unit Test Results if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: Unit Test Results (${{ matrix.os }}-${{ matrix.python }}) path: dist/pytest-results.xml @@ -132,7 +142,7 @@ jobs: steps: - name: Download Artifacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: path: artifacts @@ -152,7 +162,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 with: name: dist path: dist @@ -167,7 +177,7 @@ jobs: # upload to PyPI and make a release on every tag if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 with: name: dist path: dist diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0b472331..c3da414a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,7 +11,7 @@ Unreleased_ ----------- Added: - +- `Add autosave support to all records and record fields <../../pull/163>`_ - `Add int64In/Out record support <../../pull/161>`_ - `Enable setting alarm status of Out records <../../pull/157>`_ - `Adding the non_interactive_ioc function <../../pull/156>`_ diff --git a/Pipfile.lock b/Pipfile.lock index 474775f3..ae346b24 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -152,6 +152,51 @@ ], "version": "==1.3.1" }, + "pyyaml": { + "hashes": [ + "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf", + "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293", + "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", + "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57", + "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b", + "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4", + "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07", + "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba", + "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9", + "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", + "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513", + "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", + "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782", + "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", + "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92", + "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f", + "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", + "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc", + "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1", + "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", + "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86", + "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4", + "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c", + "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", + "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b", + "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d", + "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c", + "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb", + "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7", + "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737", + "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", + "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d", + "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358", + "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53", + "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78", + "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", + "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a", + "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f", + "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", + "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5" + ], + "version": "==6.0" + }, "scipy": { "hashes": [ "sha256:01b38dec7e9f897d4db04f8de4e20f0f5be3feac98468188a0f47a991b796055", @@ -691,6 +736,51 @@ ], "version": "==2022.4" }, + "pyyaml": { + "hashes": [ + "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf", + "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293", + "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", + "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57", + "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b", + "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4", + "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07", + "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba", + "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9", + "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", + "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513", + "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", + "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782", + "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", + "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92", + "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f", + "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", + "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc", + "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1", + "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", + "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86", + "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4", + "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c", + "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", + "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b", + "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d", + "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c", + "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb", + "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7", + "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737", + "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", + "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d", + "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358", + "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53", + "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78", + "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", + "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a", + "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f", + "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", + "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5" + ], + "version": "==6.0" + }, "requests": { "hashes": [ "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", diff --git a/docs/examples/example_autosave_ioc.py b/docs/examples/example_autosave_ioc.py new file mode 100644 index 00000000..dab84165 --- /dev/null +++ b/docs/examples/example_autosave_ioc.py @@ -0,0 +1,34 @@ +from softioc import autosave, builder, softioc +import cothread + +# Set the record prefix +builder.SetDeviceName("MY-DEVICE-PREFIX") + +# Create records, set some of them to autosave, also save some of their fields + +builder.aOut("AO", autosave=True) +builder.aIn("AI", autosave=["PREC", "EGU"]) +builder.boolIn("BO") +builder.WaveformIn("WAVEFORMIN", [0, 0, 0, 0], autosave=True) +with autosave.Autosave(["VAL", "LOPR", "HOPR"]): + builder.aOut("AUTOMATIC-AO", autosave=["EGU"]) +seconds = builder.longOut("SECONDSRUN", autosave=True) + +autosave.configure( + directory="/tmp/autosave-data", + name="MY-DEVICE-PREFIX", + save_period=5.0 +) + +builder.LoadDatabase() +softioc.iocInit() + +# Start processes required to be run after iocInit +def update(): + while True: + cothread.Sleep(1) + seconds.set(seconds.get() + 1) + +cothread.Spawn(update) + +softioc.interactive_ioc(globals()) diff --git a/docs/how-to/use-autosave-in-an-ioc.rst b/docs/how-to/use-autosave-in-an-ioc.rst new file mode 100644 index 00000000..8a41c91d --- /dev/null +++ b/docs/how-to/use-autosave-in-an-ioc.rst @@ -0,0 +1,76 @@ +Use `softioc.autosave` in an IOC +================================ + +`../tutorials/creating-an-ioc` shows how to create a pythonSoftIOC. + + +Example IOC +----------- + +.. literalinclude:: ../examples/example_autosave_ioc.py + +Records are instantiated as normal and configured for automatic loading and +periodic saving to a backup file with use of the keyword argument ``autosave``. +``autosave`` resolves to a list of strings, which are the names of fields to be +tracked by autosave. By default ``autosave=False``, which disables autosave for that PV. +Setting ``autosave=True`` is equivalent to passing ``["VAL"]``. Note that ``"VAL"`` must be +explicitly passed when tracking other fields, e.g. ``["VAL", "LOPR", "HOPR"]``. +``autosave`` can also accept a single string field name as an argument. + +The field values get written into a yaml-formatted file containing key-value pairs. +By default the keys are the same as the full PV name, including any device name specified +in :func:`~softioc.builder.SetDeviceName()`. + +Autosave is disabled until :func:`~softioc.autosave.configure()` is called. The first two arguments, +``directory`` and ``name`` are required. Backup files are periodically written into +``directory`` with the name ``.softsav`` every ``save_period`` seconds, +set to 30.0 by default. The directory must exist, and should be configured with the appropriate +read/write permissions for the user running the IOC. + +IOC developers should only need to interface with autosave via the :func:`~softioc.autosave.configure()` +method and the ``autosave`` keyword argument. Alternatively, +PVs can be instantiated inside the :class:`~softioc.autosave.Autosave()` context manager, which +automatically passes the ``autosave`` argument to any PVs created +inside the context manager. If any fields are already specified by the ``autosave`` keyword +argument of a PV's initialisation call the lists of fields to track get combined. +All other module members are intended for internal use only. + +In normal operation, loading from a backup is performed once during the +:func:`~softioc.builder.LoadDatabase()` call and periodic saving to the backup file begins when +:func:`~softioc.softioc.iocInit()` is called, provided that any PVs are configured to be saved. +Currently, manual loading from a backup at runtime after ioc initialisation is not supported. +Saving only occurs when any of the saved field values have changed since the last save. +Users are discouraged from manually editing the backup files while the +IOC is running so that the internal state of the autosave thread is consistent with +the backup file. + +If autosave is enabled and active, a timestamped copy of the latest existing autosave backup file is created +when the IOC is restarted, e.g. ``.softsav_240717-095004`` (timestamps are in the format yymmdd-HHMMSS). +If you only wish to store one backup of the autosave file at a time, ``timestamped_backups=False`` +can be passed to :func:`~softioc.autosave.configure()` when it is called, this will create a backup file +named ``.softsav.bu``. To disable any autosaving, comment out the +:func:`~softioc.autosave.configure()` call or pass it the keyword argument +``enabled=False``. + +The resulting backup file after running the example IOC for about 30 seconds is the following: + +.. code-block:: + + MY-DEVICE-PREFIX:AI.EGU: '' + MY-DEVICE-PREFIX:AI.PREC: '0' + MY-DEVICE-PREFIX:AO: 0.0 + MY-DEVICE-PREFIX:AUTOMATIC-AO: 0.0 + MY-DEVICE-PREFIX:AUTOMATIC-AO.EGU: '' + MY-DEVICE-PREFIX:AUTOMATIC-AO.HOPR: '0' + MY-DEVICE-PREFIX:AUTOMATIC-AO.LOPR: '0' + MY-DEVICE-PREFIX:SECONDSRUN: 29 + MY-DEVICE-PREFIX:WAVEFORMIN: [0, 0, 0, 0] + + +If the IOC is stopped and restarted, the SECONDSRUN record will load its saved +value of 29 from the backup. +All non-VAL fields are stored as strings. Waveform type records holding arrays +are cast into lists before saving. + +This example IOC uses cothread, but autosave works identically when using +an asyncio dispatcher. diff --git a/docs/index.rst b/docs/index.rst index 7d99fc9e..b15f5ab2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -59,6 +59,7 @@ Table Of Contents :maxdepth: 1 how-to/use-asyncio-in-an-ioc + how-to/use-autosave-in-an-ioc how-to/make-publishable-ioc how-to/read-data-from-ioc how-to/use-soft-records diff --git a/docs/reference/api.rst b/docs/reference/api.rst index d5d9aa97..6bbee443 100644 --- a/docs/reference/api.rst +++ b/docs/reference/api.rst @@ -142,6 +142,80 @@ and stderr streams, is sent directly to the terminal. .. autoclass:: softioc.asyncio_dispatcher.AsyncioDispatcher +.. automodule:: softioc.autosave + + Configuring saving and loading of record fields with `softioc.autosave.configure` + --------------------------------------------------------------------------------- + + .. function:: configure(directory, name, save_period=30, timestamped_backups=True, enabled=True) + + Used to configure settings for Autosave. + Backups are disabled by default unless this method is called. It must be + called prior to :func:`~softioc.builder.LoadDatabase`. + + It has the following arguments: + + .. _directory: + + `directory` + ~~~~~~~~~~~ + The directory where backup files should be saved to and loaded + from. This argument is required. + + .. _name: + + + `name` + ~~~~~~ + The file prefix used for naming the backup files. This is typically set to + be the same as the device prefix. The resulting file name will be + ``name``.softsav. This argument is required. + + .. _save_period: + + `save_period` + ~~~~~~~~~~~~~ + The period in seconds between each backup attempt, 30.0 by default. + Backup files are only overwritten if any of the field values have changed + since the last backup. + + .. _timestamped_backups: + + `timestamped_backups` + ~~~~~~~~~~~~~~~~~~~~~ + A boolean that is `True` by default, creates a backup of the latest existing + autosave file that is timestamped at the time that the autosave thread is + started. If set to `False`, the backup is not timestamped and gets overwritten + every time the IOC restarts. + + .. _enabled: + + `enabled` + ~~~~~~~~~ + A boolean that is `True` by default, if `False` then no loading will occur + at IOC startup, and no values with be saved to any backup files. + + .. seealso:: + + :ref:`autosave`, the builder keyword argument used to designate PV fields for autosave + + :class:`Autosave` for how to add fields to autosave inside a context manager. + + .. class:: Autosave + + .. method:: __init__(autosave=True) + + To be called as a context manager. Any PVs that are created inside + the context manager have the fields passed to the ``autosave`` argument of + the context manager added to autosave tracking. The options for ``autosave`` + are identical to the ones described in the builder keyword argument + :ref:`autosave`. If a PV already has :ref:`autosave` set, the two lists of fields + get combined into a single set. If the PV's :ref:`autosave` keyword is set + explicitly to ``False``, the fields specified in the context manager's argument + are not tracked. + + + .. automodule:: softioc.builder Creating Records: `softioc.builder` @@ -260,6 +334,34 @@ and stderr streams, is sent directly to the terminal. .. seealso:: `SetBlocking` for configuring a global default blocking value + .. _autosave: + + :ref:`autosave` + ~~~~~~~~~~~~~~~ + + Available on all record types. + Resolves to a list of string field names. When not empty it marks the record + fields for automatic periodic backing up to a file. Set to `None` by + default. When the IOC is restarted and a backup file exists, the saved values are + loaded from this file when :func:`~softioc.builder.LoadDatabase` is called. + The saved values takes priority over any initial field value passed to the PV + in `initial_value` or ``**fields``. No backing up will occur unless autosave is + enabled and configured with :func:`~softioc.autosave.configure`. + + The options for the argument are: + + * ``True``, which is equivalent to ``["VAL"]`` + * ``False``, which is equivalent to ``[]`` and disables all autosave tracking for the PV, even inside an :class:`~softioc.autosave.Autosave` context manager + * ``None``, similar to ``False`` but does not overload any fields specified in an :class:`~softioc.autosave.Autosave` context manager + * A list of field names such as ``["VAL", "LOPR", "HOPR"]``, note that ``"VAL"`` must be explicitly provided + * A single field name such as ``"EGU"`` which is equivalent to passing ``["EGU"]`` + + .. seealso:: + :func:`~softioc.autosave.configure` for discussion on how to configure saving. + + :class:`~softioc.autosave.Autosave` for how to track PVs with autosave inside a context manager. + + For all of these functions any EPICS database field can be assigned a value by passing it as a keyword argument for the corresponding field name (in upper case) or by assigning to the corresponding field of the returned record object. diff --git a/setup.py b/setup.py index 480d5c21..117387ab 100644 --- a/setup.py +++ b/setup.py @@ -106,7 +106,8 @@ def install_for_development(self): epicscorelibs.version.abi_requires(), "pvxslibs>=1.2.4", "numpy<2.0", - "epicsdbbuilder>=1.4" + "epicsdbbuilder>=1.4", + "pyyaml>=6.0" ], zip_safe = False, # setuptools_dso is not compatible with eggs! ) diff --git a/softioc/autosave.py b/softioc/autosave.py new file mode 100644 index 00000000..48576f02 --- /dev/null +++ b/softioc/autosave.py @@ -0,0 +1,335 @@ +import atexit +import sys +import threading +import traceback +from datetime import datetime +from os import rename +from pathlib import Path +from shutil import copy2 + +import numpy +import yaml + +SAV_SUFFIX = "softsav" +SAVB_SUFFIX = "softsavB" +DEFAULT_SAVE_PERIOD = 30.0 + + +def _ndarray_representer(dumper, array): + return dumper.represent_sequence( + "tag:yaml.org,2002:seq", array.tolist(), flow_style=True + ) + + +yaml.add_representer(numpy.ndarray, _ndarray_representer, Dumper=yaml.Dumper) + + +def configure( + directory, + name, + save_period=DEFAULT_SAVE_PERIOD, + timestamped_backups=True, + enabled=True, +): + """This should be called before initialising the IOC. Configures the + autosave thread for periodic backing up of PV values. + + Args: + directory: string or Path giving directory path where autosave backup + files are saved and loaded. + name: string name of the root used for naming backup files. This + is usually the same as the device name. + save_period: time in seconds between backups. Backups are only performed + if PV values have changed. + timestamped_backups: boolean which determines if backups of existing + autosave files are timestamped on IOC restart. True by default, if + False then backups get overwritten on each IOC restart. + enabled: boolean which enables or disables autosave, set to True by + default, or False if configure not called. + """ + directory_path = Path(directory) + if not directory_path.is_dir(): + raise FileNotFoundError( + f"{directory} is not a valid autosave directory" + ) + AutosaveConfig.directory = directory_path + AutosaveConfig.timestamped_backups = timestamped_backups + AutosaveConfig.save_period = save_period + AutosaveConfig.enabled = enabled + AutosaveConfig.device_name = name + + +class AutosaveConfig: + directory = None + device_name = None + timestamped_backups = True + save_period = DEFAULT_SAVE_PERIOD + enabled = False + + +def start_autosave_thread(): + worker = threading.Thread( + target=Autosave._loop, + ) + worker.start() + atexit.register(_shutdown_autosave_thread, worker) + + +def _shutdown_autosave_thread(worker): + Autosave._stop() + worker.join() + + +def _parse_autosave_fields(fields): + if not fields: + return [] + elif fields is True: + return ["VAL"] + elif isinstance(fields, list): + return fields + elif isinstance(fields, str): + return [fields] + else: + raise ValueError(f"Could not parse autosave fields argument: {fields}") + + +def add_pv_to_autosave(pv, name, fields): + """Configures a PV for autosave + + Args: + pv: a PV object inheriting ProcessDeviceSupportCore + name: the key by which the PV value is saved to and loaded from a + backup. This is typically the same as the PV name. + fields: used to determine which fields of a PV are tracked by autosave. + The allowed options are a single string such as "VAL" or "EGU", + a list of strings such as ["VAL", "EGU"], a boolean True which + evaluates to ["VAL"] or False to track no fields. If the PV is + created inside an Autosave context manager, the fields passed to the + context manager are also tracked by autosave. + """ + if fields is False: + # if autosave=False explicitly set, override context manager + return + fields = set(_parse_autosave_fields(fields)) + # instantiate context to get thread local class variables via instance + context = _AutosaveContext() + if context._in_cm: # _fields should always be a list if in context manager + fields.update(context._fields) + for field in fields: + field_name = name if field == "VAL" else f"{name}.{field}" + Autosave._pvs[field_name] = _AutosavePV(pv, field) + + +def load_autosave(): + Autosave._load() + + +class _AutosavePV: + def __init__(self, pv, field): + if field == "VAL": + self.get = pv.get + self.set = pv.set + else: + self.get = lambda: pv.get_field(field) + self.set = lambda val: setattr(pv, field, val) + + +def _get_current_sav_path(): + return ( + AutosaveConfig.directory / f"{AutosaveConfig.device_name}.{SAV_SUFFIX}" + ) + + +def _get_tmp_sav_path(): + return ( + AutosaveConfig.directory / f"{AutosaveConfig.device_name}.{SAVB_SUFFIX}" + ) + + +def _get_timestamped_backup_sav_path(timestamp): + sav_path = _get_current_sav_path() + return sav_path.parent / ( + sav_path.name + timestamp.strftime("_%y%m%d-%H%M%S") + ) + + +def _get_backup_sav_path(): + sav_path = _get_current_sav_path() + return sav_path.parent / (sav_path.name + ".bu") + + +class _AutosaveContext(threading.local): + _instance = None + _lock = threading.Lock() + _fields = None + _in_cm = False + + def __new__(cls, fields=None): + if cls._instance is None: + with cls._lock: + if not cls._instance: + cls._instance = super().__new__(cls) + if cls._instance._in_cm and fields is not None: + cls._instance._fields = fields or [] + return cls._instance + + def reset(self): + self._fields = None + self._in_cm = False + + +class Autosave: + _pvs = {} + _last_saved_state = {} + _last_saved_time = datetime.now() + _stop_event = threading.Event() + _loop_started = False + + def __init__(self, fields=True): + """ + When called as a context manager, any PVs created in the context have + the fields provided by the fields argument added to autosave backups. + + Args: + fields: a list of string field names to be periodically saved to a + backup file, which are loaded from on IOC restart. + The allowed options are a single string such as "VAL" or "EGU", + a list of strings such as ["VAL", "EGU"], a boolean True which + evaluates to ["VAL"] or False to track no additional fields. + If the autosave keyword is already specified in a PV's + initialisation, the list of fields to track are combined. + """ + context = _AutosaveContext() + if context._in_cm: + raise RuntimeError( + "Can not instantiate Autosave when already in context manager" + ) + fields = _parse_autosave_fields(fields) + context._fields = fields + + def __enter__(self): + context = _AutosaveContext() + context._in_cm = True + + def __exit__(self, A, B, C): + context = _AutosaveContext() + context.reset() + + @classmethod + def __backup_sav_file(cls): + if ( + not AutosaveConfig.directory + or not AutosaveConfig.directory.is_dir() + ): + print( + f"Could not back up autosave as {AutosaveConfig.directory} is" + " not a valid directory", + file=sys.stderr, + ) + return + sav_path = _get_current_sav_path() + if AutosaveConfig.timestamped_backups: + backup_path = _get_timestamped_backup_sav_path(cls._last_saved_time) + else: + backup_path = _get_backup_sav_path() + if sav_path.is_file(): + copy2(sav_path, backup_path) + else: + print( + f"Could not back up autosave, {sav_path} is not a file", + file=sys.stderr, + ) + + @classmethod + def __get_state(cls): + state = {} + for pv_field, pv in cls._pvs.items(): + try: + state[pv_field] = pv.get() + except Exception: + print(f"Exception getting {pv_field}", file=sys.stderr) + traceback.print_exc() + return state + + @classmethod + def __set_pvs_from_saved_state(cls): + for pv_field, value in cls._last_saved_state.items(): + try: + pv = cls._pvs[pv_field] + pv.set(value) + except Exception: + print( + f"Exception setting {pv_field} to {value}", + file=sys.stderr, + ) + traceback.print_exc() + + @classmethod + def __state_changed(cls, state): + return cls._last_saved_state.keys() != state.keys() or any( + # checks equality for builtins and numpy arrays + not numpy.array_equal(state[key], cls._last_saved_state[key]) + for key in state + ) + + @classmethod + def _save(cls): + state = cls.__get_state() + if cls.__state_changed(state): + sav_path = _get_current_sav_path() + tmp_path = _get_tmp_sav_path() + # write to temporary file first then use atomic os.rename + # to safely update stored state + with open(tmp_path, "w") as backup: + yaml.dump(state, backup, indent=4) + rename(tmp_path, sav_path) + cls._last_saved_state = state + cls._last_saved_time = datetime.now() + + @classmethod + def _load(cls): + if not AutosaveConfig.enabled or not cls._pvs: + return + if not AutosaveConfig.device_name: + raise RuntimeError( + "Device name is not known to autosave thread, " + "call autosave.configure() with keyword argument name" + ) + if not AutosaveConfig.directory: + raise RuntimeError( + "Autosave directory is not known, call " + "autosave.configure() with keyword argument directory" + ) + if not AutosaveConfig.directory.is_dir(): + raise FileNotFoundError( + f"{AutosaveConfig.directory} is not a valid autosave directory" + ) + cls.__backup_sav_file() + sav_path = _get_current_sav_path() + if not sav_path or not sav_path.is_file(): + print( + f"Could not load autosave values from file {sav_path}", + file=sys.stderr, + ) + return + with open(sav_path, "r") as f: + cls._last_saved_state = yaml.full_load(f) + cls.__set_pvs_from_saved_state() + + @classmethod + def _stop(cls): + cls._stop_event.set() + + @classmethod + def _loop(cls): + if not AutosaveConfig.enabled or not cls._pvs or cls._loop_started: + return + cls._loop_started = True + while True: + try: + cls._stop_event.wait(timeout=AutosaveConfig.save_period) + cls._save() + if cls._stop_event.is_set(): # Stop requested + return + except Exception: + traceback.print_exc() diff --git a/softioc/builder.py b/softioc/builder.py index e84e53a6..bcbd21d3 100644 --- a/softioc/builder.py +++ b/softioc/builder.py @@ -3,6 +3,7 @@ from .device_core import RecordLookup from .softioc import dbLoadDatabase +from .autosave import load_autosave from epicsdbbuilder import * @@ -299,6 +300,7 @@ def LoadDatabase(): '''This should be called after all the builder records have been created, but before calling iocInit(). The database is loaded into EPICS memory, ready for operation.''' + load_autosave() from tempfile import mkstemp fd, database = mkstemp('.db') os.close(fd) diff --git a/softioc/device.py b/softioc/device.py index 8c133fad..332de014 100644 --- a/softioc/device.py +++ b/softioc/device.py @@ -3,7 +3,7 @@ import ctypes from ctypes import * import numpy - +from . import autosave from . import alarm from . import fields from .imports import ( @@ -53,6 +53,11 @@ class ProcessDeviceSupportCore(DeviceSupportCore, RecordLookup): # from record init or processing _epics_rc_ = EPICS_OK + # all record types can support autosave + def __init__(self, name, **kargs): + autosave_fields = kargs.pop("autosave", None) + autosave.add_pv_to_autosave(self, name, autosave_fields) + self.__super.__init__(name, **kargs) # Most subclasses (all except waveforms) define a ctypes constructor for the # underlying EPICS compatible value. diff --git a/softioc/pythonSoftIoc.py b/softioc/pythonSoftIoc.py index 906ffa86..8d04606f 100644 --- a/softioc/pythonSoftIoc.py +++ b/softioc/pythonSoftIoc.py @@ -24,7 +24,9 @@ def __init__(self, builder, device, name, **fields): # have to maintain this separately from the corresponding device list. DeviceKeywords = [ 'on_update', 'on_update_name', 'validate', 'always_update', - 'initial_value', '_wf_nelm', '_wf_dtype', 'blocking'] + 'initial_value', '_wf_nelm', '_wf_dtype', 'blocking', + 'autosave' + ] device_kargs = {} for keyword in DeviceKeywords: if keyword in fields: diff --git a/softioc/softioc.py b/softioc/softioc.py index aa0a3968..7867d50f 100644 --- a/softioc/softioc.py +++ b/softioc/softioc.py @@ -4,7 +4,7 @@ from ctypes import * from tempfile import NamedTemporaryFile -from . import imports, device +from . import autosave, imports, device from . import cothread_dispatcher __all__ = ['dbLoadDatabase', 'iocInit', 'interactive_ioc'] @@ -34,6 +34,7 @@ def iocInit(dispatcher=None): # Set the dispatcher for record processing callbacks device.dispatcher = dispatcher imports.iocInit() + autosave.start_autosave_thread() def safeEpicsExit(code=0): diff --git a/tests/conftest.py b/tests/conftest.py index 301ccb01..66ab41ee 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,7 @@ import pytest from softioc import builder -from softioc.builder import ClearRecords +from softioc.builder import ClearRecords, SetDeviceName, GetRecordNames in_records = [ builder.aIn, @@ -98,12 +98,18 @@ def asyncio_ioc_override(): ioc.kill() aioca_cleanup() +def reset_device_name(): + if GetRecordNames().prefix: + SetDeviceName("") + @pytest.fixture(autouse=True) def clear_records(): """Deletes all records before and after every test""" ClearRecords() + reset_device_name() yield ClearRecords() + reset_device_name() @pytest.fixture(autouse=True) def enable_code_coverage(): diff --git a/tests/sim_records.py b/tests/sim_records.py index b882d58e..7d9daec4 100644 --- a/tests/sim_records.py +++ b/tests/sim_records.py @@ -14,7 +14,6 @@ ioc_name = names.prefix[0] else: ioc_name = 'TS-DI-TEST-01' - SetDeviceName(ioc_name) def on_update(value): print('on_update', repr(value)) @@ -29,6 +28,7 @@ def on_update_name(value, name): def create_records(): global t_ai, t_ao + SetDeviceName(ioc_name) t_ai = aIn('AI', initial_value=12.34) boolIn('BOOLIN', 'True', 'False', initial_value=False) diff --git a/tests/test_autosave.py b/tests/test_autosave.py new file mode 100644 index 00000000..a1dda923 --- /dev/null +++ b/tests/test_autosave.py @@ -0,0 +1,495 @@ +from conftest import get_multiprocessing_context, select_and_recv +from softioc import autosave, builder, softioc, device_core, asyncio_dispatcher +from unittest.mock import patch +import pytest +import threading +import numpy +import re +import yaml +import time + +DEVICE_NAME = "MY-DEVICE" + + +@pytest.fixture(autouse=True) +def reset_autosave_setup_teardown(): + default_save_period = autosave.AutosaveConfig.save_period + default_device_name = autosave.AutosaveConfig.device_name + default_directory = autosave.AutosaveConfig.directory + default_enabled = autosave.AutosaveConfig.enabled + default_tb = autosave.AutosaveConfig.timestamped_backups + default_pvs = autosave.Autosave._pvs.copy() + default_state = autosave.Autosave._last_saved_state.copy() + default_cm_save_fields = autosave._AutosaveContext._fields + default_instance = autosave._AutosaveContext._instance + yield + autosave.AutosaveConfig.save_period = default_save_period + autosave.AutosaveConfig.device_name = default_device_name + autosave.AutosaveConfig.directory = default_directory + autosave.AutosaveConfig.enabled = default_enabled + autosave.AutosaveConfig.timestamped_backups = default_tb + autosave.Autosave._pvs = default_pvs + autosave.Autosave._last_saved_state = default_state + autosave.Autosave._stop_event = threading.Event() + autosave._AutosaveContext._fields = default_cm_save_fields + autosave._AutosaveContext._instance = default_instance + + +@pytest.fixture +def existing_autosave_dir(tmp_path): + state = { + "SAVED-AO": 20.0, + "SAVED-AI": 20.0, + "SAVED-BO": 1, + "SAVED-BI": 1, + "SAVED-LONGIN": 20, + "SAVED-LONGOUT": 20, + "SAVED-INT64IN": 100, + "SAVED-INT64OUT": 100, + "SAVED-MBBI": 15, + "SAVED-MBBO": 15, + "SAVED-STRINGIN": "test string in", + "SAVED-STRINGOUT": "test string out", + "SAVED-LONGSTRINGIN": "test long string in", + "SAVED-LONGSTRINGOUT": "test long string out", + "SAVED-ACTION": 1, + "SAVED-WAVEFORMIN": [1, 2, 3, 4], + "SAVED-WAVEFORMOUT": [1, 2, 3, 4], + "SAVED-WAVEFORMIN-STRINGS": ["test", "waveform", "strings"], + "SAVED-WAVEFORMOUT-STRINGS": ["test", "waveform", "strings"], + } + with open(tmp_path / f"{DEVICE_NAME}.softsav", "w") as f: + yaml.dump(state, f, indent=4) + with open(tmp_path / f"{DEVICE_NAME}.softsav.bu", "w") as f: + yaml.dump({"OUT-OF-DATE-KEY": "out of date value"}, f, indent=4) + return tmp_path + + +def test_configure(tmp_path): + assert autosave.AutosaveConfig.enabled is False + autosave.configure(tmp_path, DEVICE_NAME) + assert autosave.AutosaveConfig.device_name == DEVICE_NAME + assert autosave.AutosaveConfig.directory == tmp_path + assert autosave.AutosaveConfig.enabled is True + assert autosave.AutosaveConfig.timestamped_backups is True + + +def test_autosave_defaults(): + assert autosave.Autosave._pvs == {} + assert autosave.Autosave._last_saved_state == {} + assert isinstance(autosave.Autosave._stop_event, threading.Event) + assert not autosave.Autosave._stop_event.is_set() + assert autosave.AutosaveConfig.save_period == 30.0 + assert autosave.AutosaveConfig.device_name is None + assert autosave.AutosaveConfig.directory is None + assert autosave.AutosaveConfig.enabled is False + assert autosave.AutosaveConfig.timestamped_backups is True + + +def test_configure_dir_doesnt_exist(tmp_path): + DEVICE_NAME = "MY_DEVICE" + builder.aOut("MY-RECORD", autosave=True) + with pytest.raises(FileNotFoundError): + autosave.configure(tmp_path / "subdir-doesnt-exist", DEVICE_NAME) + + +def test_returns_if_init_called_before_configure(): + autosave.Autosave() + assert autosave.AutosaveConfig.enabled is False + + +def test_all_record_types_saveable(tmp_path): + autosave.configure(tmp_path, DEVICE_NAME) + + number_types = [ + "aIn", + "aOut", + "boolIn", + "boolOut", + "longIn", + "longOut", + "int64In", + "int64Out", + "mbbIn", + "mbbOut", + "Action", + ] + string_types = ["stringIn", "stringOut", "longStringIn", "longStringOut"] + waveform_types = ["WaveformIn", "WaveformOut"] + for pv_type in number_types: + pv = getattr(builder, pv_type)(pv_type, autosave=True) + for pv_type in string_types: + pv = getattr(builder, pv_type)(pv_type, autosave=True) + pv.set("test string") + for pv_type in waveform_types: + getattr(builder, pv_type)(pv_type, numpy.zeros((100)), autosave=True) + getattr(builder, pv_type)( + f"{pv_type}_of_chars", "test waveform string", autosave=True + ) + getattr(builder, pv_type)( + f"{pv_type}_of_strings", ["array", "of", "strings"], autosave=True + ) + + autosaver = autosave.Autosave() + autosaver._save() + + with open(tmp_path / f"{DEVICE_NAME}.softsav", "r") as f: + saved = yaml.full_load(f) + for pv_type in number_types + string_types + waveform_types: + assert pv_type in saved + + +def test_can_save_fields(tmp_path): + builder.aOut("SAVEVAL", autosave=["VAL", "DISA"]) + builder.aOut("DONTSAVEVAL", autosave=["SCAN"]) + # we need to patch get_field as we can't call builder.LoadDatabase() + # and softioc.iocInit() in unit tests + with patch( + "softioc.device.ProcessDeviceSupportCore.get_field", return_value="0" + ): + autosave.configure(tmp_path, DEVICE_NAME) + autosaver = autosave.Autosave() + assert "SAVEVAL" in autosaver._pvs + assert "SAVEVAL.DISA" in autosaver._pvs + assert "DONTSAVEVAL" not in autosaver._pvs + assert "DONTSAVEVAL.SCAN" in autosaver._pvs + autosaver._save() + with open(tmp_path / f"{DEVICE_NAME}.softsav", "r") as f: + saved = yaml.full_load(f) + assert "SAVEVAL" in saved + assert "SAVEVAL.DISA" in saved + assert "DONTSAVEVAL" not in saved + assert "DONTSAVEVAL.SCAN" in saved + + +def test_stop_event(tmp_path): + autosave.configure(tmp_path, DEVICE_NAME) + builder.aOut("DUMMYRECORD", autosave=True) + worker = threading.Thread( + target=autosave.Autosave._loop, + ) + try: + worker.daemon = True + worker.start() + assert not autosave.Autosave._stop_event.is_set() + assert worker.is_alive() + autosave.Autosave._stop() + assert autosave.Autosave._stop_event.is_set() + finally: + worker.join(timeout=1) + + +@pytest.mark.parametrize( + "timestamped,regex", + [ + (False, r"^" + DEVICE_NAME + r"\.softsav_[0-9]{6}-[0-9]{6}$"), + (True, r"^" + DEVICE_NAME + r"\.softsav\.bu$"), + ], +) +def test_backup_on_load(existing_autosave_dir, timestamped, regex): + autosave.configure( + existing_autosave_dir, DEVICE_NAME, timestamped_backups=timestamped + ) + # backup only performed if there are any pvs to save + builder.aOut("SAVED-AO", autosave=True) + autosave.load_autosave() + backup_files = list(existing_autosave_dir.glob("*.softsav_*")) + # assert backup is .softsav_yymmdd-HHMMSS or .softsav.bu + any(re.match(regex, file.name) for file in backup_files) + if not timestamped: + # test that existing .bu file gets overwritten + with open(existing_autosave_dir / f"{DEVICE_NAME}.softsav.bu") as f: + state = yaml.full_load(f) + assert "OUT-OF-DATE-KEY" not in state + assert "SAVED-AO" in state + + +def test_autosave_key_names(tmp_path): + builder.aOut("DEFAULTNAME", autosave=True) + builder.aOut("DEFAULTNAMEAFTERPREFIXSET", autosave=True) + autosave.configure(tmp_path, DEVICE_NAME) + autosaver = autosave.Autosave() + autosaver._save() + with open(tmp_path / f"{DEVICE_NAME}.softsav", "r") as f: + saved = yaml.full_load(f) + assert "DEFAULTNAME" in saved + assert "DEFAULTNAMEAFTERPREFIXSET" in saved + + +def test_context_manager(tmp_path): + builder.aOut("MANUAL", autosave=["VAL", "EGU"]) + with autosave.Autosave(["VAL", "PINI"]): + builder.aOut("AUTOMATIC") + builder.aOut( + "AUTOMATIC-EXTRA-FIELD", autosave=["SCAN"] + ) + autosave.configure(tmp_path, DEVICE_NAME) + with patch( + "softioc.device.ProcessDeviceSupportCore.get_field", return_value="0" + ): + autosaver = autosave.Autosave() + autosaver._save() + with open(tmp_path / f"{DEVICE_NAME}.softsav", "r") as f: + saved = yaml.full_load(f) + assert "MANUAL" in saved + assert "MANUAL.EGU" in saved + assert "AUTOMATIC" in saved + assert "AUTOMATIC.PINI" in saved + assert "AUTOMATIC-EXTRA-FIELD" in saved + assert "AUTOMATIC-EXTRA-FIELD.SCAN" in saved + assert "AUTOMATIC-EXTRA-FIELD.PINI" in saved + + +def check_all_record_types_load_properly(device_name, autosave_dir, conn): + autosave.configure(autosave_dir, device_name) + pv_aOut = builder.aOut("SAVED-AO", autosave=True) + pv_aIn = builder.aIn("SAVED-AI", autosave=True) + pv_boolOut = builder.boolOut("SAVED-BO", autosave=True) + pv_boolIn = builder.boolIn("SAVED-BI", autosave=True) + pv_longIn = builder.longIn("SAVED-LONGIN", autosave=True) + pv_longOut = builder.longOut("SAVED-LONGOUT", autosave=True) + pv_int64In = builder.int64In("SAVED-INT64IN", autosave=True) + pv_int64Out = builder.int64Out("SAVED-INT64OUT", autosave=True) + pv_mbbIn = builder.mbbIn("SAVED-MBBI", autosave=True) + pv_mbbOut = builder.mbbOut("SAVED-MBBO", autosave=True) + pv_stringIn = builder.stringIn("SAVED-STRINGIN", autosave=True) + pv_stringOut = builder.stringOut("SAVED-STRINGOUT", autosave=True) + pv_longStringIn = builder.longStringIn("SAVED-LONGSTRINGIN", autosave=True) + pv_longStringOut = builder.longStringOut( + "SAVED-LONGSTRINGOUT", autosave=True + ) + pv_Action = builder.Action("SAVED-ACTION", autosave=True) + pv_WaveformIn = builder.WaveformIn( + "SAVED-WAVEFORMIN", numpy.zeros((4)), autosave=True + ) + pv_WaveformOut = builder.WaveformOut( + "SAVED-WAVEFORMOUT", numpy.zeros((4)), autosave=True + ) + pv_WaveformIn_strings = builder.WaveformIn( + "SAVED-WAVEFORMIN-STRINGS", + ["initial", "waveform", "strings"], + autosave=True, + ) + pv_WaveformOut_strings = builder.WaveformOut( + "SAVED-WAVEFORMOUT-STRINGS", + ["initial", "waveform", "strings"], + autosave=True, + ) + assert pv_aOut.get() == 0.0 + assert pv_aIn.get() == 0.0 + assert pv_boolOut.get() == 0 + assert pv_boolIn.get() == 0 + assert pv_longIn.get() == 0 + assert pv_longOut.get() == 0 + assert pv_int64In.get() == 0 + assert pv_int64Out.get() == 0 + assert pv_mbbIn.get() == 0 + assert pv_mbbOut.get() == 0 + assert pv_stringIn.get() == "" + assert pv_stringOut.get() == "" + assert pv_longStringIn.get() == "" + assert pv_longStringOut.get() == "" + assert pv_Action.get() == 0 + assert (pv_WaveformIn.get() == numpy.array([0, 0, 0, 0])).all() + assert (pv_WaveformOut.get() == numpy.array([0, 0, 0, 0])).all() + assert pv_WaveformIn_strings.get() == ["initial", "waveform", "strings"] + assert pv_WaveformOut_strings.get() == ["initial", "waveform", "strings"] + # load called automatically when LoadDatabase() called + builder.LoadDatabase() + assert pv_aOut.get() == 20.0 + assert pv_aIn.get() == 20.0 + assert pv_boolOut.get() == 1 + assert pv_boolIn.get() == 1 + assert pv_longIn.get() == 20 + assert pv_longOut.get() == 20 + assert pv_int64In.get() == 100 + assert pv_int64Out.get() == 100 + assert pv_mbbIn.get() == 15 + assert pv_mbbOut.get() == 15 + assert pv_stringIn.get() == "test string in" + assert pv_stringOut.get() == "test string out" + assert pv_longStringIn.get() == "test long string in" + assert pv_longStringOut.get() == "test long string out" + assert pv_Action.get() == 1 + assert (pv_WaveformIn.get() == numpy.array([1, 2, 3, 4])).all() + assert (pv_WaveformOut.get() == numpy.array([1, 2, 3, 4])).all() + assert pv_WaveformIn_strings.get() == ["test", "waveform", "strings"] + assert pv_WaveformOut_strings.get() == ["test", "waveform", "strings"] + conn.send("D") # "Done" + + +def test_actual_ioc_load(existing_autosave_dir): + ctx = get_multiprocessing_context() + parent_conn, child_conn = ctx.Pipe() + ioc_process = ctx.Process( + target=check_all_record_types_load_properly, + args=(DEVICE_NAME, existing_autosave_dir, child_conn), + ) + ioc_process.start() + # If we never receive D it probably means an assert failed + select_and_recv(parent_conn, "D") + + +def check_all_record_types_save_properly(device_name, autosave_dir, conn): + autosave.configure(autosave_dir, device_name, save_period=1) + builder.aOut("aOut", autosave=True, initial_value=20.0) + builder.aIn("aIn", autosave=True, initial_value=20.0) + builder.boolOut("boolOut", autosave=True, initial_value=1) + builder.boolIn("boolIn", autosave=True, initial_value=1) + builder.longIn("longIn", autosave=True, initial_value=20) + builder.longOut("longOut", autosave=True, initial_value=20) + builder.int64In("int64In", autosave=True, initial_value=100) + builder.int64Out("int64Out", autosave=True, initial_value=100) + builder.mbbIn("mbbIn", autosave=True, initial_value=15) + builder.mbbOut("mbbOut", autosave=True, initial_value=15) + builder.stringIn("stringIn", autosave=True, initial_value="test string in") + builder.stringOut( + "stringOut", autosave=True, initial_value="test string out" + ) + builder.longStringIn( + "longStringIn", autosave=True, initial_value="test long string in" + ) + builder.longStringOut( + "longStringOut", autosave=True, initial_value="test long string out" + ) + builder.Action("Action", autosave=True, initial_value=1) + builder.WaveformIn("WaveformIn", [1, 2, 3, 4], autosave=True) + builder.WaveformOut("WaveformOut", [1, 2, 3, 4], autosave=True) + builder.LoadDatabase() + dispatcher = asyncio_dispatcher.AsyncioDispatcher() + softioc.iocInit(dispatcher) + # wait long enough to ensure one save has occurred + time.sleep(2) + with open(autosave_dir / f"{device_name}.softsav", "r") as f: + saved = yaml.full_load(f) + assert saved["aOut"] == 20.0 + assert saved["aIn"] == 20.0 + assert saved["boolOut"] == 1 + assert saved["boolIn"] == 1 + assert saved["longIn"] == 20 + assert saved["longOut"] == 20 + assert saved["int64In"] == 100 + assert saved["int64Out"] == 100 + assert saved["mbbIn"] == 15 + assert saved["mbbOut"] == 15 + assert saved["stringIn"] == "test string in" + assert saved["stringOut"] == "test string out" + assert saved["longStringIn"] == "test long string in" + assert saved["longStringOut"] == "test long string out" + assert saved["Action"] == 1 + assert (saved["WaveformIn"] == numpy.array([1, 2, 3, 4])).all() + assert (saved["WaveformOut"] == numpy.array([1, 2, 3, 4])).all() + autosave.Autosave._stop() + # force autosave thread to stop to ensure pytest exits + conn.send("D") + + +def test_actual_ioc_save(tmp_path): + ctx = get_multiprocessing_context() + parent_conn, child_conn = ctx.Pipe() + ioc_process = ctx.Process( + target=check_all_record_types_save_properly, + args=(DEVICE_NAME, tmp_path, child_conn), + ) + ioc_process.start() + # If we never receive D it probably means an assert failed + select_and_recv(parent_conn, "D") + + +def check_autosave_field_names_contain_device_prefix( + device_name, tmp_path, conn +): + autosave.configure(tmp_path, device_name, save_period=1) + builder.aOut("BEFORE", autosave=["VAL", "EGU"]) + builder.SetDeviceName(device_name) + builder.aOut("AFTER", autosave=["VAL", "EGU"]) + builder.LoadDatabase() + dispatcher = asyncio_dispatcher.AsyncioDispatcher() + softioc.iocInit(dispatcher) + time.sleep(2) + with open(tmp_path / f"{device_name}.softsav", "r") as f: + saved = yaml.full_load(f) + assert "BEFORE" in saved.keys() + assert f"{device_name}:AFTER" in saved.keys() + autosave.Autosave._stop() + conn.send("D") + + +def test_autosave_field_names_contain_device_prefix(tmp_path): + ctx = get_multiprocessing_context() + parent_conn, child_conn = ctx.Pipe() + ioc_process = ctx.Process( + target=check_autosave_field_names_contain_device_prefix, + args=(DEVICE_NAME, tmp_path, child_conn), + ) + ioc_process.start() + # If we never receive D it probably means an assert failed + select_and_recv(parent_conn, "D") + +def test_context_manager_thread_safety(tmp_path): + autosave.configure(tmp_path, DEVICE_NAME) + in_cm_event = threading.Event() + + def create_pv_in_thread(name): + in_cm_event.wait() + builder.aOut(name, autosave=False) + pv_thread_before_cm = threading.Thread( + target=create_pv_in_thread, args=["PV-FROM-THREAD-BEFORE"]) + pv_thread_in_cm = threading.Thread( + target=create_pv_in_thread, args=["PV-FROM-THREAD-DURING"]) + pv_thread_before_cm.start() + with autosave.Autosave(["VAL", "EGU"]): + in_cm_event.set() + builder.aOut("PV-FROM-CM") + pv_thread_in_cm.start() + pv_thread_in_cm.join() + pv_thread_before_cm.join() + + assert "PV-FROM-THREAD-BEFORE" not in autosave.Autosave._pvs + assert "PV-FROM-THREAD-DURING" not in autosave.Autosave._pvs + assert device_core.LookupRecord("PV-FROM-THREAD-BEFORE") + assert device_core.LookupRecord("PV-FROM-THREAD-DURING") + +def test_nested_context_managers_raises(tmp_path): + autosave.configure(tmp_path, DEVICE_NAME) + with autosave.Autosave(["SCAN"]): + with pytest.raises(RuntimeError): + with autosave.Autosave(False): + builder.aOut("MY-PV") + with pytest.raises(RuntimeError): + autosave.Autosave() + +def test_autosave_arguments(tmp_path): + autosave.configure(tmp_path, DEVICE_NAME) + builder.aOut("TRUE", autosave=True) + builder.aOut("FIELDS", autosave=["LOPR", "HOPR"]) + builder.aOut("FALSE", autosave=False) + builder.aOut("SINGLE-FIELD", autosave="EGU") + builder.aOut("AUTO-VAL", autosave="VAL") + assert set(autosave.Autosave._pvs) == { + "TRUE", "FIELDS.LOPR", "FIELDS.HOPR", "SINGLE-FIELD.EGU", "AUTO-VAL"} + autosave.Autosave._pvs = {} + builder.ClearRecords() + with autosave.Autosave(): # True by default + builder.aOut("AUTO-TRUE", autosave=False) + builder.aOut("FIELDS", autosave=["LOPR", "HOPR"]) + assert set(autosave.Autosave._pvs) == { + "FIELDS", "FIELDS.LOPR", "FIELDS.HOPR"} + autosave.Autosave._pvs = {} + builder.ClearRecords() + with autosave.Autosave(["EGU"]): + builder.aOut("AUTO-FALSE") + builder.aOut("FIELDS", autosave=["PINI"]) + assert set(autosave.Autosave._pvs) == { + "AUTO-FALSE.EGU", "FIELDS.EGU", "FIELDS.PINI"} + autosave.Autosave._pvs = {} + builder.ClearRecords() + with autosave.Autosave(False): + builder.aOut("AUTO-DEFAULT") + builder.aOut("AUTO-TRUE", autosave=True) + assert set(autosave.Autosave._pvs) == {"AUTO-TRUE"} + autosave.Autosave._pvs = {} + builder.ClearRecords() + with autosave.Autosave("LOPR"): # single field + builder.aOut("FIELDS", autosave="HOPR") + assert set(autosave.Autosave._pvs) == {"FIELDS.HOPR", "FIELDS.LOPR"}