Merge branch 'pre-release' into stable

.github/workflows/flow.yml

@@ -1,4 +1,4 @@
-name: sr_insects Flow Tests.
+name: Flow Tests / AMQP
 
 on:
   pull_request:
@@ -36,7 +36,7 @@ jobs:
 
     runs-on: ${{ matrix.osver }}
 
-    name: ${{ matrix.which_test }} test on ${{ matrix.osver }}
+    name: ${{ matrix.which_test }} on ${{ matrix.osver }}
     timeout-minutes: 40
 
     steps:

.github/workflows/flow_amqp_consumer.yml

@@ -1,11 +1,11 @@
-name: sr_insects Flow Tests using AMQP Consumer.
+name: Flow Tests / AMQP Consumer
 
 on:
   pull_request:
     types: [opened, edited, reopened]
   push:
     branches:
-      - issue_457_amqp_consumer
+      - development
 
     paths-ignore:
       - '.github/**'
@@ -33,7 +33,7 @@ jobs:
 
     runs-on: ${{ matrix.osver }}
 
-    name: ${{ matrix.which_test }} test on ${{ matrix.osver }}
+    name: ${{ matrix.which_test }} on ${{ matrix.osver }}
     timeout-minutes: 40
 
     steps:
@@ -48,7 +48,7 @@ jobs:
           sudo sh -c 'echo "MaxStartups 750" >> /etc/ssh/sshd_config'
           sudo systemctl restart ssh
           echo "amqp_consumer True" >> ${HOME}/.config/sr3/default.conf
-          echo "set sarracenia.moth.amqpconsumer.AMQPConsumer.logLevel debug" >> ${HOME}/.config/sr3/default.conf
+          #echo "set sarracenia.moth.amqpconsumer.AMQPConsumer.logLevel debug" >> ${HOME}/.config/sr3/default.conf
       
       - name: Setup ${{ matrix.which_test }} test.
         run: |

.github/workflows/flow_mqtt.yml

@@ -1,4 +1,4 @@
-name: Sr3 sr_insects Flow Tests using MQTT.
+name: Flow Tests / MQTT
 
 on:
   pull_request:
@@ -33,7 +33,7 @@ jobs:
 
     runs-on: ${{ matrix.osver }}
 
-    name: ${{ matrix.which_test }} test on ${{ matrix.osver }}
+    name: ${{ matrix.which_test }} on ${{ matrix.osver }}
     timeout-minutes: 45
 
     steps:

.github/workflows/flow_redis.yml

@@ -1,11 +1,12 @@
-name: Flow Tests (with Redis).
+name: Flow Tests / Redis
 
 on:
   pull_request:
     types: [opened, edited, reopened]
   push:
     branches:
       - v03_disabled
+
     paths-ignore:
       - '.github/**'
       - 'debian/changelog'

.github/workflows/unit-test.yml

@@ -2,27 +2,30 @@ name: Unit Testing
 
 on:
   pull_request:
-    types: [opened, edited, reopened]
-  push:
-    paths-ignore:
-      - '.github/**'
+    types: [opened, edited, reopened, closed, synchronize]
+    paths:
+      - '**.py'
+    branches:
+      - 'development'
+  # push:
+  #   paths-ignore:
+  #     - '.github/**'
+  workflow_dispatch:
 
+permissions:  
+  contents: write
+  checks: write
+  pull-requests: write
 
 jobs:
   build:
+    runs-on: ubuntu-22.04
 
-    strategy:
-      fail-fast: false
-      matrix:
-        osver: [ "ubuntu-20.04", "ubuntu-22.04" ]
-
-    runs-on: ${{ matrix.osver }}
-
-    name: Unit test on ${{ matrix.osver }}
+    name: Unit test
     timeout-minutes: 10
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install dependencies
         run: |
@@ -38,42 +41,72 @@ jobs:
             --cov-config=tests/.coveragerc --cov=sarracenia --cov-report=html --cov-report=lcov --cov-report=xml \
             --html=tests/report.html --self-contained-html
 
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action/linux@v2
+        if: ${{ always() }}
+        with:
+          files: |
+            tests/junit/test-results.xml
+
+      # - name: Publish Coverage Results 
+      #   uses: orgoro/coverage@v3.1
+      #   with:
+      #     coverageFile: tests/coverage/coverage.xml
+
+      # - name: Pytest coverage comment
+      #   uses: MishaKav/pytest-coverage-comment@main
+      #   if: ${{ always() }}
+      #   with:
+      #     #pytest-coverage-path: tests/coverage/coverage.lcov
+      #     pytest-xml-coverage-path: tests/coverage/coverage.xml
+      #     #title: My Coverage Report Title
+      #     #badge-title: My Badge Coverage Title
+      #     #hide-badge: false
+      #     #hide-report: false
+      #     create-new-comment: false
+      #     hide-comment: false
+      #     report-only-changed-files: true
+      #     remove-link-from-badge: false
+      #     #unique-id-for-comment: python3.8
+      #     junitxml-path: tests/junit/test-results.xml
+      #     junitxml-title: Test Results
+
       - name: Upload pytest junit results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: results-junit-${{ matrix.osver }}
+          name: results-junit
           path: tests/junit/test-results.xml
         # Use always() to always run this step to publish test results when there are test failures
         if: ${{ always() }}
 
       - name: Upload pytest HTML report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: results-report-${{ matrix.osver }}
+          name: results-report
           path: tests/report.html
         # Use always() to always run this step to publish test results when there are test failures
         if: ${{ always() }}
 
       - name: Upload code coverage report (HTML)
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: coverage-report-${{ matrix.osver }}
+          name: coverage-report
           path: tests/coverage/html_report
         # Use always() to always run this step to publish test results when there are test failures
         if: ${{ always() }}
 
       - name: Upload code coverage report (LCOV)
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: coverage-lcov-${{ matrix.osver }}
+          name: coverage-lcov
           path: tests/coverage/coverage.lcov
         # Use always() to always run this step to publish test results when there are test failures
         if: ${{ always() }}
 
       - name: Upload code coverage report (XML)
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: coverage-xml-${{ matrix.osver }}
+          name: coverage-xml
           path: tests/coverage/coverage.xml
         # Use always() to always run this step to publish test results when there are test failures
         if: ${{ always() }}

debian/changelog

@@ -1,3 +1,49 @@
+metpx-sr3 (3.00.53) unstable; urgency=medium
+
+  * NEW! merge PR #1019 Azure Transfer protocol driver (experimental)
+  * NEW! merge PR #1030 WMO-00 Accumulated file support (experimental.)
+  * NEW! PR #1040 #1041 exchangeSplitOverride + documentation of overrides.
+  * PR #1038 prefer local plugins over app provided ones.
+  * PR #1037 clearer, more succinct error messages for sender.
+  * api improvement Message.updatePaths() with no paths has defaults.
+  * api improvement part of above, Message.getContent() now works for local files.
+  * api improvement Message.updatePaths() now makes reasonable guesses about missing arguments.
+  * improved support for WMO WIS2 messages.
+  * fix #980 additional header verification of receipt of AM messages.
+  * fix #908 AM server must reap dead children.
+  * fix #1027 Eumetsat Poll plugin improvements.
+  * fix #1023,#1025 topic processing improvements, topicCopy option added.
+  * fix #1022 various fixes to work better with DMS.
+  * fix #1020 Added sourceFromMessage option for v2 compatibility.
+  * fix #1012 *persist* is now a configurable option (was always on.)
+  * fix #947 sr3 status should ignore files if not .inc or .conf
+  * NEW! merge PR #1010 S3 Transfer protocol driver (experimental)
+  * fix #1004 make persistence an option (was always persistent until now.)
+  * fix #1002 sftp accellerator failure when colons in the destination name.
+  * fix #997 flowcb/gather/am refactor to fix many issues.
+  * fix #994 resolved issues with unit tests.
+  * fix #992 parsing of module declared options to ensure correct type.
+  * fix #988 flowcb/poll/NASA_CMR podack checksum calculation changed.
+  * fix #986 poll stop re-writing "directory" as path for poll.
+  * fix #984 refactored flow/run routine 288lines -> 159. more
+    logical/readable.
+  * fix #983 add S3 sender plugin.
+  * fix #982 no cleanup when statehost set.
+  * fix #979, #980 AM fixes.
+  * fix #978 housekeeping log messages reduuced 16 lines less than 5.
+  * fix #976 remove "downloaded" log message when nothing was downloaded.
+  * fix #971 sr3 overview crash.
+  * fix #967 define hostname as a string option (was defaulting to list.)
+  * fix #974 refactor poll to consume messages more frequently in wVip.
+  *   #974 poll implement 5 minute default interval
+  * fix #973 AM ... add missing tokIsYear better AM message filtering.
+  * fix #971 Add missing to list of possible process status
+  * docs: Add topicPrefix for #961
+  * docs: some revisions for clarity/syntax. 
+  * some fixes for misnaming of some (rare) files received by gather/am
+
+ -- Peter Silva <peter@bsqt.homeip.net>  Thu, 17 May 2024 09:31:04 -0400
+
 metpx-sr3 (3.00.52) unstable; urgency=medium
 
   * copernicus marine data store polling support ( #959 )

docs/source/Contribution/Development.rst

@@ -117,18 +117,22 @@ To run the sr_insects tests, the repository must be cloned with the development
 A gate for merging to development is for a second developer to run the flow_tests.
 **For v03, these tests must run:  static_flow, flakey_broker, dynamic_flow, transform_flow**
 
-Planned by 2022/04/11:
 
- * stable will be merged from (version 3) development, so the default version for new work is sr3.
 
- * launchpad has recipes to produce metpx-sr3 packages from the stable branch. 
+ * launchpad has recipes to produce metpx-sr3 packages from various branches.
 
  * The *MetPX Daily* repository is a snapshot of the development branch. 
 
  * The *MetPX Pre-Release* repository should receive versions ending in rcX (release candidate) 
+   The packages here from from pre-release branch which comes from snapshots of the development branch.
+   There is also a pre-release-py36 branch for building pre-release packages for older operating systems.
 
- * The *MetPX* repository should only contain stable releases that have graduated from the rcX series.
+ * stable comes from on snapshots of (version 3) pre-release branch.
 
+ * The *MetPX* repository should only contain stable releases that have graduated from the rcX series.
+   there is a stable_py36 branch to build packages for older operating systems that have
+   python 3.6 (redhat 8, ubuntu 18, ubuntu 20) or are too old to use hatchling installer.
+
 
 sr_insects
 ~~~~~~~~~~
@@ -342,17 +346,20 @@ The options are detailed below:
 Committing Code
 ~~~~~~~~~~~~~~~
 
-What should be done prior to committing to the stable branch?
+What should be done prior to committing to the development branch?
 Checklist:
 
 - do development on some other branch. Usually the branch will be named after the issue being
   addressed. Example: issue240, if we give up on an initial approach and start another one, 
   there may be issue240_2 for a second attempt. There may also be feature branches, such as v03.
-- **sr_insects tests works** (See Testing) The stable branch should always be functional, do not commit code if the sr_insects tests are not working.
+
+- **sr_insects tests works** (See Testing) The development branch should always be functional, do not commit code if the sr_insects tests are not working.
+
 - Natural consequence: if the code changes means tests need to change, include the test change in the commit.
+
 - **update doc/** manual pages should get their updates ideally at the same time as the code.
 
-Usually there will be many such cycles on a development branch before one is ready
+Usually there will be many such cycles on an issueXXX  branch before one is ready
 to issue a pull request. Eventually, we get to `Commits to the Development Branch`_
 
 
@@ -566,7 +573,7 @@ Running Flow Tests
 
 This section documents these steps in much more detail.  
 Before one can run the sr_insects tests, some pre-requisites must be taken care of.
-Note that there is Github Actions integration for at least the stable branch
+Note that there is Github Actions integration for at least the development branch
 to verify functionality on a variety of python version.  Consult::
 
    https://github.com/MetPX/sarracenia/actions
@@ -829,7 +836,7 @@ Then check show it went with flow_check.sh::
 
 If the flow_check.sh passes, then one has a reasonable confidence in the overall functionality of the 
 python application, but the test coverage is not exhaustive. This is the lowest gate for committing
-changes to thy python code into the stable branch. It is more qualitative sampling of the most
+changes to thy python code into the development branch. It is more qualitative sampling of the most
 common use cases rather than a thorough examination of all functionality. While not
 thorough, it is good to know the flows are working.
 
@@ -1215,6 +1222,8 @@ A Second developer will review the pull request and the reviewer will decide on
 merging is appropriate. The developer is expected to examine each commit, and 
 understand it to some degree.
 
+If the pull-request has one of the following (substantial changes, new functionality, modifications to critical code structure) , it is recommended to have a Third developer also review the pull request. The expectation from this developer are the same as from the previous.
+
 The github Actions looks at pull requests and will flow tests on them.
 If the tests pass, then that is good qualitative indicator, however the tests are a bit
 fragile at the moment, so if they fail, it would be ideal for the reviewer to run
@@ -1228,13 +1237,16 @@ Key Branches
 There is a long running discussion about `Which Version is stable <https://github.com/MetPX/sarracenia/issues/139>`_
 The current set up is that there are four principal branches:
 
-* stable branch is the release version of sr3, merging from development. used to build sr3 packages in the
+* stable branch is the release version of sr3, merging from pre-release. used to build sr3 packages in the
   `MetPX <https://launchpad.net/~ssc-hpc-chp-spc/+archive/ubuntu/metpx>`_ repository.
 
-* development ... The `version 3 <v03.rst>`_ work in progress branch is a next version of sarracenia in development.
+* development ... The `version 3 <v03.rst>`_ work in progress branch is a next version of sarracenia.
   the development branch is used to build sr3 packages for the `Daily <https://launchpad.net/~ssc-hpc-chp-spc/+archive/ubuntu/metpx-daily>`_
   and `Pre-Release <https://launchpad.net/~ssc-hpc-chp-spc/+archive/ubuntu/metpx-pre-release>`_ repositories on launchpad.net.
 
+* stable_py36 and pre-relrease-36 are branched from stable and pre_release respectively to adjust for building
+  packages on older operating systems that have older versions of python (and no support for hatchling.)
+
 * issue branches to be merged to development, it should be start with issueXXX or suggested branch names from github are ok also.
 
 * sometimes, multiple branches are needed for a single issue, say for variations of a fix, eg. issueXXX_2_do_it_this_way .