From 80302f0b4c5351a8d6dc849e2d2c57dbc0d4730d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 27 Aug 2024 13:28:04 -0600 Subject: [PATCH 01/97] Update old Slack Community links with Discord links --- README.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index dd32e11cc5fb..ff7026df37d9 100644 --- a/README.rst +++ b/README.rst @@ -11,8 +11,8 @@ :target: https://lgtm.com/projects/g/saltstack/salt/context:python .. image:: https://img.shields.io/badge/slack-SaltProject-blue.svg?logo=slack - :alt: Salt Project Slack Community - :target: https://via.vmw.com/salt-slack + :alt: Salt Project Discord Community + :target: https://discord.gg/hCysSRaV .. image:: https://img.shields.io/twitch/status/saltprojectoss :alt: Salt Project Twitch Channel @@ -103,7 +103,7 @@ Report bugs or problems using Salt by opening an issue: ``_ -and the `Salt Project Community Slack`_. +and the `Salt Project Community Discord`_. Responsibly reporting security vulnerabilities @@ -153,7 +153,7 @@ Please be sure to review our Also, check out some of our community resources including: * `Salt Project Community Wiki `_ -* `Salt Project Community Slack`_ +* `Salt Project Community Discord`_ * `Salt Project: IRC on LiberaChat `_ * `Salt Project YouTube channel `_ * `Salt Project Twitch channel `_ @@ -165,8 +165,7 @@ to the **Salt Project Community Events Calendar** on the main ``_ website. If you have additional questions, email us at saltproject@vmware.com or reach out -directly to the Community Manager, Jimmy Chunga via Slack. We'd be glad to -have you join our community! +directly to the Community Discord. We'd be glad to have you join our community! License @@ -180,7 +179,7 @@ used by external modules. A complete list of attributions and dependencies can be found here: `salt/DEPENDENCIES.md `_ -.. _Salt Project Community Slack: https://via.vmw.com/salt-slack +.. _Salt Project Community Discord: https://discord.gg/hCysSRaV .. _VMware Aria Automation Config: https://www.vmware.com/products/vrealize-automation/saltstack-config.html .. _Latest Salt Documentation: https://docs.saltproject.io/en/latest/ .. _Open an issue: https://github.com/saltstack/salt/issues/new/choose From 1eb4ec7f460aa52f71bc3ee00dec210e77aed0b2 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 27 Aug 2024 13:42:32 -0600 Subject: [PATCH 02/97] Updated with correct Discord invite link, and correct badge link --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index ff7026df37d9..32b20bdbbd6c 100644 --- a/README.rst +++ b/README.rst @@ -10,9 +10,9 @@ :alt: PyPi Package Downloads :target: https://lgtm.com/projects/g/saltstack/salt/context:python -.. image:: https://img.shields.io/badge/slack-SaltProject-blue.svg?logo=slack +.. image:: https://img.shields.io/badge/discord-SaltProject-blue.svg?logo=discord :alt: Salt Project Discord Community - :target: https://discord.gg/hCysSRaV + :target: https://discord.com/invite/J7b7EscrAs .. image:: https://img.shields.io/twitch/status/saltprojectoss :alt: Salt Project Twitch Channel @@ -179,7 +179,7 @@ used by external modules. A complete list of attributions and dependencies can be found here: `salt/DEPENDENCIES.md `_ -.. _Salt Project Community Discord: https://discord.gg/hCysSRaV +.. _Salt Project Community Discord: https://discord.com/invite/J7b7EscrAs .. _VMware Aria Automation Config: https://www.vmware.com/products/vrealize-automation/saltstack-config.html .. _Latest Salt Documentation: https://docs.saltproject.io/en/latest/ .. _Open an issue: https://github.com/saltstack/salt/issues/new/choose From ed7496ea470ba74d6715ecba3cdf20f049a38c52 Mon Sep 17 00:00:00 2001 From: Max Arnold Date: Wed, 31 Jul 2024 23:32:52 +0700 Subject: [PATCH 03/97] Replace Slack with Discord --- .github/ISSUE_TEMPLATE/config.yml | 4 ++-- .github/ISSUE_TEMPLATE/tech-debt.md | 2 +- .github/config.yml | 4 ++-- CONTRIBUTING.rst | 6 +++--- README.rst | 3 ++- SUPPORT.rst | 4 ++-- doc/_incl/requisite_incl.rst | 2 +- doc/conf.py | 2 +- doc/security/index.rst | 2 +- doc/topics/development/conventions/formulas.rst | 2 +- doc/topics/development/conventions/release.rst | 4 ++-- doc/topics/tutorials/states_pt4.rst | 2 +- 12 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index c465f8de0fb7..690b27ae6728 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,7 +1,7 @@ blank_issues_enabled: true contact_links: - - name: Salt Community Slack - url: https://saltstackcommunity.slack.com/ + - name: Salt Community Discord + url: https://discord.com/invite/J7b7EscrAs about: Please ask and answer questions here. - name: Salt-Users Forum url: https://groups.google.com/forum/#!forum/salt-users diff --git a/.github/ISSUE_TEMPLATE/tech-debt.md b/.github/ISSUE_TEMPLATE/tech-debt.md index a13303e3ee11..6761145a4429 100644 --- a/.github/ISSUE_TEMPLATE/tech-debt.md +++ b/.github/ISSUE_TEMPLATE/tech-debt.md @@ -8,7 +8,7 @@ assignees: '' --- ### Description of the tech debt to be addressed, include links and screenshots - + ### Versions Report (Provided by running `salt --versions-report`. Please also mention any differences in master/minion versions.) diff --git a/.github/config.yml b/.github/config.yml index 6bf3cadf069f..b89e3c822bd1 100644 --- a/.github/config.yml +++ b/.github/config.yml @@ -13,7 +13,7 @@ newIssueWelcomeComment: > - [Community Wiki](https://github.com/saltstack/community/wiki) - [Salt’s Contributor Guide](https://docs.saltproject.io/en/master/topics/development/contributing.html) - - [Join our Community Slack](https://via.vmw.com/salt-slack) + - [Join our Community Discord](https://discord.com/invite/J7b7EscrAs) - [IRC on LiberaChat](https://web.libera.chat/#salt) - [Salt Project YouTube channel](https://www.youtube.com/channel/UCpveTIucFx9ljGelW63-BWg) - [Salt Project Twitch channel](https://www.twitch.tv/saltprojectoss) @@ -39,7 +39,7 @@ newPRWelcomeComment: > - [Community Wiki](https://github.com/saltstack/community/wiki) - [Salt’s Contributor Guide](https://docs.saltproject.io/en/master/topics/development/contributing.html) - - [Join our Community Slack](https://via.vmw.com/salt-slack) + - [Join our Community Discord](https://discord.com/invite/J7b7EscrAs) - [IRC on LiberaChat](https://web.libera.chat/#salt) - [Salt Project YouTube channel](https://www.youtube.com/channel/UCpveTIucFx9ljGelW63-BWg) - [Salt Project Twitch channel](https://www.twitch.tv/saltprojectoss) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6a41d924ae30..5379f4485728 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -8,7 +8,7 @@ in a number of ways: - Use Salt and open well-written bug reports. - Join a `working group `__. - Answer questions on `irc `__, - the `community Slack `__, + the `community Discord `__, the `salt-users mailing list `__, `Server Fault `__, @@ -109,7 +109,7 @@ Then activate it: Sweet! Now you're ready to clone Salt so you can start hacking away! If you get stuck at any point, check out the resources at the beginning of -this guide. IRC and Slack are particularly helpful places to go. +this guide. IRC and Discord are particularly helpful places to go. Get the source! @@ -605,7 +605,7 @@ your PR is submitted during the week you should be able to expect some kind of communication within that business day. If your tests are passing and we're not in a code freeze, ideally your code will be merged that week or month. If you haven't heard from your assigned reviewer, ping them -on GitHub, `irc `__, or Community Slack. +on GitHub, `irc `__, or Community Discord. It's likely that your reviewer will leave some comments that need addressing - it may be a style change, or you forgot a changelog entry, diff --git a/README.rst b/README.rst index 32b20bdbbd6c..f3129299b114 100644 --- a/README.rst +++ b/README.rst @@ -103,7 +103,8 @@ Report bugs or problems using Salt by opening an issue: ``_ -**SaltStack Slack** - Alongside IRC is our SaltStack Community Slack for the +**SaltStack Slack** - Alongside IRC is our SaltStack Community Discord for the SaltStack Working groups. Use the following link to request an invitation. -``_ +``_ **Mailing List** - The SaltStack community users mailing list is hosted by Google groups. Anyone can post to ask questions about SaltStack products and diff --git a/doc/_incl/requisite_incl.rst b/doc/_incl/requisite_incl.rst index b478527d6b1b..f5723a952eaf 100644 --- a/doc/_incl/requisite_incl.rst +++ b/doc/_incl/requisite_incl.rst @@ -7,4 +7,4 @@ following the instructions in the The Salt Project community can help offer advice and help troubleshoot technical issues as you're learning about Salt. One of the best places to talk to the community is on the - `Salt Project Slack workspace `_. + `Salt Project Discord Community `_. diff --git a/doc/conf.py b/doc/conf.py index 33503d70e25b..78c3b29aebcb 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -180,7 +180,7 @@ .. _`salt-users`: https://groups.google.com/forum/#!forum/salt-users .. _`salt-announce`: https://groups.google.com/forum/#!forum/salt-announce .. _`salt-packagers`: https://groups.google.com/forum/#!forum/salt-packagers -.. _`salt-slack`: https://via.vmw.com/salt-slack +.. _`salt-discord`: https://discord.com/invite/J7b7EscrAs .. |windownload| raw:: html

Python3 x86: `__ landing page * `SaltStack Security RSS Feed `__ -* `SaltStack Community Slack Workspace `__ +* `Salt Project Discord Community `__ diff --git a/doc/topics/development/conventions/formulas.rst b/doc/topics/development/conventions/formulas.rst index ad9033b2d100..7e069d41ec96 100644 --- a/doc/topics/development/conventions/formulas.rst +++ b/doc/topics/development/conventions/formulas.rst @@ -222,7 +222,7 @@ repository in your own account on GitHub and notify a SaltStack employee when it is ready. We will add you to the Contributors team on the `saltstack-formulas`_ organization and help you transfer the repository over. Ping a SaltStack employee on IRC (`#salt`_ on LiberaChat), join the -``#formulas`` channel on the `salt-slack`_ (bridged to ``#saltstack-formulas`` +``#formulas`` channel on the `salt-discord`_ (bridged to ``#saltstack-formulas`` on LiberaChat) or send an email to the `salt-users`_ mailing list. Note that IRC logs are available at http://ngxbot.nginx.org/logs/%23salt/ and archives for FreeNode (up to mid-June 2021) https://logbot-archive.s3.amazonaws.com/freenode/salt.gz diff --git a/doc/topics/development/conventions/release.rst b/doc/topics/development/conventions/release.rst index c08447e20078..1cf05c8ff1f0 100644 --- a/doc/topics/development/conventions/release.rst +++ b/doc/topics/development/conventions/release.rst @@ -46,7 +46,7 @@ example): #. Publish the docs. #. Create release at `github`_ #. Update win-repo-ng with new salt versions. -#. Announce release is live to irc, salt-users, salt-announce and release slack +#. Announce release is live to irc, salt-users, salt-announce and release discord community channel. @@ -79,7 +79,7 @@ for a bugfix release. #. Publish the docs. #. Create release at `github`_ #. Update win-repo-ng with new salt versions. -#. Announce release is live to irc, salt-users, salt-announce and release slack channel. +#. Announce release is live to irc, salt-users, salt-announce and release discord channel. .. _`github`: https://github.com/saltstack/salt/releases .. _`repo.saltproject.io`: https://repo.saltproject.io diff --git a/doc/topics/tutorials/states_pt4.rst b/doc/topics/tutorials/states_pt4.rst index 96701b20af37..943f3079f114 100644 --- a/doc/topics/tutorials/states_pt4.rst +++ b/doc/topics/tutorials/states_pt4.rst @@ -211,7 +211,7 @@ can be found on GitHub in the `saltstack-formulas`_ collection of repositories. If you have any questions, suggestions, or just want to chat with other people who are using Salt, we have a very active community and we'd love to hear from you. One of the best places to talk to the community is on the -`Salt Project Slack workspace `_. +`Salt Project Discord Community `_. In addition, by continuing to the :ref:`Orchestrate Runner ` docs, you can learn about the powerful orchestration of which Salt is capable. From 4d2ad41860faf9731c02606fa92ccd276dccf3c9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 12 Aug 2024 14:18:18 -0600 Subject: [PATCH 04/97] Debugging --- salt/minion.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/salt/minion.py b/salt/minion.py index cb4ef919b305..8c83a2283c02 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -279,6 +279,12 @@ def get_proc_dir(cachedir, **kwargs): made. Same applies if the directory is already owned by this gid. Must be int. Works only on unix/unix like systems. """ + # pylint: disable=logging-fstring-interpolation + log.warning(f"DGM get_proc_dir entry, cachedir '{cachedir}', kwargs '{kwargs}'") + print( + f"DGM get_proc_dir entry, cachedir '{cachedir}', kwargs '{kwargs}'", flush=True + ) + fn_ = os.path.join(cachedir, "proc") mode = kwargs.pop("mode", None) @@ -305,11 +311,29 @@ def get_proc_dir(cachedir, **kwargs): uid = kwargs.pop("uid", -1) gid = kwargs.pop("gid", -1) + # pylint: disable=logging-fstring-interpolation + log.warning( + f"DGM get_proc_dir chown, d_stat '{d_stat}', uid '{uid}', gid '{gid}'" + ) + print( + f"DGM get_proc_dir chown, d_stat '{d_stat}', uid '{uid}', gid '{gid}'", + flush=True, + ) + # if uid and gid are both -1 then go ahead with # no changes at all if (d_stat.st_uid != uid or d_stat.st_gid != gid) and [ i for i in (uid, gid) if i != -1 ]: + # pylint: disable=logging-fstring-interpolation + log.warning( + f"DGM get_proc_dir chown file, file '{fn_}' , uid '{uid}', gid '{gid}'" + ) + print( + f"DGM get_proc_dir chown file, file '{fn_}' , uid '{uid}', gid '{gid}'", + flush=True, + ) + os.chown(fn_, uid, gid) return fn_ From d3ad2520d33edc56c26ed0d5c25db5f3012bbacc Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 14 Aug 2024 11:12:43 -0600 Subject: [PATCH 05/97] Updated salt-syndic to utilize user/group for Salt (master settings), and maintain on upgrade --- pkg/common/salt-api.service | 1 + pkg/debian/salt-api.preinst | 1 - pkg/debian/salt-syndic.postinst | 37 ++++++++++++++++++++++++++++++++ pkg/debian/salt-syndic.preinst | 27 +++++++++++++++++++++++ pkg/debian/salt-syndic.templates | 17 +++++++++++++++ pkg/rpm/salt.spec | 31 +++++++++++++++++++++++++- salt/minion.py | 36 +++++++++++++++---------------- 7 files changed, 130 insertions(+), 20 deletions(-) create mode 100644 pkg/debian/salt-syndic.postinst create mode 100644 pkg/debian/salt-syndic.preinst create mode 100644 pkg/debian/salt-syndic.templates diff --git a/pkg/common/salt-api.service b/pkg/common/salt-api.service index d0b6d741202b..3eefa7649009 100644 --- a/pkg/common/salt-api.service +++ b/pkg/common/salt-api.service @@ -2,6 +2,7 @@ Description=The Salt API Documentation=man:salt-api(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltproject.io/en/latest/contents.html After=network.target +PartOf=salt-master.service [Service] Type=notify diff --git a/pkg/debian/salt-api.preinst b/pkg/debian/salt-api.preinst index ddc7c9e0ec7b..c063108ea55e 100644 --- a/pkg/debian/salt-api.preinst +++ b/pkg/debian/salt-api.preinst @@ -22,7 +22,6 @@ case "$1" in else db_set salt-api/enabled enabled db_set salt-api/active active - fi ;; esac diff --git a/pkg/debian/salt-syndic.postinst b/pkg/debian/salt-syndic.postinst new file mode 100644 index 000000000000..071ba38e1859 --- /dev/null +++ b/pkg/debian/salt-syndic.postinst @@ -0,0 +1,37 @@ +#!/bin/sh + +. /usr/share/debconf/confmodule + +case "$1" in + configure) + db_get salt-syndic/user + if [ "$RET" != "root" ]; then + if [ ! -e "/var/log/salt/syndic" ]; then + touch /var/log/salt/syndic + chmod 640 /var/log/salt/syndic + fi + chown $RET:$RET /var/log/salt/syndic + fi + if command -v systemctl; then + db_get salt-syndic/active + RESLT=$(echo "$RET" | cut -d ' ' -f 1) + if [ "$RESLT" != 10 ]; then + systemctl daemon-reload + if [ "$RESLT" = "active" ]; then + systemctl restart salt-syndic + fi + db_get salt-syndic/enabled + RESLT=$(echo "$RET" | cut -d ' ' -f 1) + if [ "$RESLT" = "disabled" ]; then + systemctl disable salt-syndic + else + systemctl enable salt-syndic + fi + else + systemctl daemon-reload + systemctl restart salt-syndic + systemctl enable salt-syndic + fi + fi + ;; +esac diff --git a/pkg/debian/salt-syndic.preinst b/pkg/debian/salt-syndic.preinst new file mode 100644 index 000000000000..da43d779163c --- /dev/null +++ b/pkg/debian/salt-syndic.preinst @@ -0,0 +1,27 @@ +#!/bin/sh + +. /usr/share/debconf/confmodule + +case "$1" in + upgrade) + [ -z "$SALT_HOME" ] && SALT_HOME=/opt/saltstack/salt + [ -z "$SALT_USER" ] && SALT_USER=salt + [ -z "$SALT_NAME" ] && SALT_NAME="Salt" + [ -z "$SALT_GROUP" ] && SALT_GROUP=salt + + # Reset permissions to fix previous installs + CUR_USER=$(ls -dl /run/salt-syndic.pid | cut -d ' ' -f 3) + CUR_GROUP=$(ls -dl /run/salt-syndic.pid | cut -d ' ' -f 4) + db_set salt-syndic/user $CUR_USER + chown -R $CUR_USER:$CUR_GROUP /var/log/salt/syndic + if command -v systemctl; then + SM_ENABLED=$(systemctl show -p UnitFileState salt-syndic | cut -d '=' -f 2) + db_set salt-syndic/enabled $SM_ENABLED + SM_ACTIVE=$(systemctl is-active salt-syndic) + db_set salt-syndic/active $SM_ACTIVE + else + db_set salt-syndic/enabled enabled + db_set salt-syndic/active active + fi + ;; +esac diff --git a/pkg/debian/salt-syndic.templates b/pkg/debian/salt-syndic.templates new file mode 100644 index 000000000000..c27859e0a24f --- /dev/null +++ b/pkg/debian/salt-syndic.templates @@ -0,0 +1,17 @@ +Template: salt-syndic/user +Type: string +Default: salt +Description: User for salt-syndic + User to run the salt-syndic process as + +Template: salt-syndic/enabled +Type: string +Default: enabled +Description: Systemd enable state for salt-syndic + default enable state for salt-syndic systemd state + +Template: salt-syndic/active +Type: string +Default: active +Description: Systemd active state for salt-syndic + default active state for salt-syndic systemd state diff --git a/pkg/rpm/salt.spec b/pkg/rpm/salt.spec index 180df99f34b1..f1707cb5dd07 100644 --- a/pkg/rpm/salt.spec +++ b/pkg/rpm/salt.spec @@ -441,6 +441,15 @@ if [ $1 -gt 1 ] ; then %global _MS_CUR_GROUP %{_MS_LCUR_GROUP} fi +%pre syndic +if [ $1 -gt 1 ] ; then + # Reset permissions to match previous installs - performing upgrade + _MS_LCUR_USER=$(ls -dl /run/salt/master | cut -d ' ' -f 3) + _MS_LCUR_GROUP=$(ls -dl /run/salt/master | cut -d ' ' -f 4) + %global _MS_CUR_USER %{_MS_LCUR_USER} + %global _MS_CUR_GROUP %{_MS_LCUR_GROUP} +fi + %pre minion if [ $1 -gt 1 ] ; then # Reset permissions to match previous installs - performing upgrade @@ -463,6 +472,14 @@ if [ $1 -eq 0 ] ; then /bin/systemctl stop salt-syndic.service > /dev/null 2>&1 || : fi +%preun syndic +# %%systemd_preun salt-syndic.service +if [ $1 -eq 0 ] ; then + # Package removal, not upgrade + /bin/systemctl --no-reload disable salt-syndic.service > /dev/null 2>&1 || : + /bin/systemctl stop salt-syndic.service > /dev/null 2>&1 || : +fi + %preun minion # %%systemd_preun salt-minion.service if [ $1 -eq 0 ] ; then @@ -471,7 +488,6 @@ if [ $1 -eq 0 ] ; then /bin/systemctl stop salt-minion.service > /dev/null 2>&1 || : fi - %preun api # %%systemd_preun salt-api.service if [ $1 -eq 0 ] ; then @@ -602,6 +618,19 @@ else fi +%posttrans syndic +if [ ! -e "/var/log/salt/syndic" ]; then + touch /var/log/salt/syndic + chmod 640 /var/log/salt/syndic +fi +if [ $1 -gt 1 ] ; then + # Reset permissions to match previous installs - performing upgrade + chown -R %{_MS_CUR_USER}:%{_MS_CUR_GROUP} /var/log/salt/syndic +else + chown -R %{_SALT_USER}:%{_SALT_GROUP} /var/log/salt/syndic +fi + + %posttrans api if [ ! -e "/var/log/salt/api" ]; then touch /var/log/salt/api diff --git a/salt/minion.py b/salt/minion.py index 8c83a2283c02..64057e81fccc 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -280,10 +280,10 @@ def get_proc_dir(cachedir, **kwargs): gid. Must be int. Works only on unix/unix like systems. """ # pylint: disable=logging-fstring-interpolation - log.warning(f"DGM get_proc_dir entry, cachedir '{cachedir}', kwargs '{kwargs}'") - print( - f"DGM get_proc_dir entry, cachedir '{cachedir}', kwargs '{kwargs}'", flush=True - ) + ## DGM log.warning(f"DGM get_proc_dir entry, cachedir '{cachedir}', kwargs '{kwargs}'") + ## DGM print( + ## DGM f"DGM get_proc_dir entry, cachedir '{cachedir}', kwargs '{kwargs}'", flush=True + ## DGM ) fn_ = os.path.join(cachedir, "proc") mode = kwargs.pop("mode", None) @@ -312,13 +312,13 @@ def get_proc_dir(cachedir, **kwargs): gid = kwargs.pop("gid", -1) # pylint: disable=logging-fstring-interpolation - log.warning( - f"DGM get_proc_dir chown, d_stat '{d_stat}', uid '{uid}', gid '{gid}'" - ) - print( - f"DGM get_proc_dir chown, d_stat '{d_stat}', uid '{uid}', gid '{gid}'", - flush=True, - ) + ## DGM log.warning( + ## DGM f"DGM get_proc_dir chown, d_stat '{d_stat}', uid '{uid}', gid '{gid}'" + ## DGM ) + ## DGM print( + ## DGM f"DGM get_proc_dir chown, d_stat '{d_stat}', uid '{uid}', gid '{gid}'", + ## DGM flush=True, + ## DGM ) # if uid and gid are both -1 then go ahead with # no changes at all @@ -326,13 +326,13 @@ def get_proc_dir(cachedir, **kwargs): i for i in (uid, gid) if i != -1 ]: # pylint: disable=logging-fstring-interpolation - log.warning( - f"DGM get_proc_dir chown file, file '{fn_}' , uid '{uid}', gid '{gid}'" - ) - print( - f"DGM get_proc_dir chown file, file '{fn_}' , uid '{uid}', gid '{gid}'", - flush=True, - ) + ## DGM log.warning( + ## DGM f"DGM get_proc_dir chown file, file '{fn_}' , uid '{uid}', gid '{gid}'" + ## DGM ) + ## DGM print( + ## DGM f"DGM get_proc_dir chown file, file '{fn_}' , uid '{uid}', gid '{gid}'", + ## DGM flush=True, + ## DGM ) os.chown(fn_, uid, gid) From 5298d333fa8585d8fed6495281cd0ce24ff0d15d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 14 Aug 2024 13:20:50 -0600 Subject: [PATCH 06/97] Update tests --- tests/pytests/pkg/integration/test_salt_user.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index fb42ae3c9f6b..aad608275181 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -67,8 +67,12 @@ def pkg_paths_salt_user(): "/var/log/salt/master", "/var/log/salt/api", "/var/log/salt/key", + "/var/log/salt/syndic", "/var/cache/salt/master", "/var/run/salt/master", + "/run/salt-master.pid", + "/run/salt-syndic.pid", + "/run/salt-api.pid", ] From e097b541c916de4587b8f95502a39b30f9eed454 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 19 Aug 2024 16:39:15 -0600 Subject: [PATCH 07/97] Testing with master_opts user for syndic_user --- salt/config/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 6d0bc947fc22..cffce8929f6a 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -2502,7 +2502,11 @@ def syndic_config( ), ) ), - "user": opts.get("syndic_user", opts["user"]), + # DGM conf/suse/master has syndic_user for running syndic as a different user + # DGM use urrently set it to 'salt' + # DGM testing getting the master_opts user, and run syndic as that + # DGM "user": opts.get("syndic_user", opts["user"]), + "user": opts.get("syndic_user", master_opts["user"]), "sock_dir": os.path.join( opts["cachedir"], opts.get("syndic_sock_dir", opts["sock_dir"]) ), @@ -2510,6 +2514,10 @@ def syndic_config( "cachedir": master_opts["cachedir"], } opts.update(syndic_opts) + log.warning("DGM syndic_config, user is set to '%s'", opts["user"]) + dgm_user = opts["user"] + print(f"DGM syndic_config, user is set to '{dgm_user}'", flush=True) + # Prepend root_dir to other paths prepend_root_dirs = [ "pki_dir", From 853d3dafe3b9228cbecdca317268d7d9dfc2026d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 20 Aug 2024 11:38:59 -0600 Subject: [PATCH 08/97] Added delay for salt_master running, and updated some package tests --- .../pytests/pkg/integration/test_salt_api.py | 5 ---- .../pytests/pkg/integration/test_salt_user.py | 30 ++++++++++++------- tests/pytests/pkg/integration/test_version.py | 8 +++-- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/tests/pytests/pkg/integration/test_salt_api.py b/tests/pytests/pkg/integration/test_salt_api.py index e962fbe32213..a4f11973748f 100644 --- a/tests/pytests/pkg/integration/test_salt_api.py +++ b/tests/pytests/pkg/integration/test_salt_api.py @@ -9,11 +9,6 @@ def test_salt_api(api_request, salt_master, install_salt): """ Test running a command against the salt api """ - if install_salt.distro_id in ("ubuntu", "debian"): - pytest.skip( - "Package test are getting reworked in https://github.com/saltstack/salt/issues/66672" - ) - assert salt_master.is_running() ret = api_request.post( diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index aad608275181..96180a972fe9 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -2,6 +2,7 @@ import pathlib import subprocess import sys +import time import packaging.version import psutil @@ -91,6 +92,13 @@ def test_salt_user_master(salt_master, install_salt): """ Test the correct user is running the Salt Master """ + # DGM assert salt_master.is_running() + + for count in range(0, 10): + if salt_master.is_running(): + break + else: + time.sleep(1) assert salt_master.is_running() match = False @@ -227,17 +235,17 @@ def test_paths_log_rotation( ): pytest.skip("Package path ownership was changed in salt 3006.4") - if install_salt.distro_id not in ( - "almalinux", - "rocky", - "centos", - "redhat", - "amzn", - "fedora", - ): - pytest.skip( - "Only tests RedHat family packages till logrotation paths are resolved on Ubuntu/Debian, see issue 65231" - ) + # DGM if install_salt.distro_id not in ( + # DGM "almalinux", + # DGM "rocky", + # DGM "centos", + # DGM "redhat", + # DGM "amzn", + # DGM "fedora", + # DGM ): + # DGM pytest.skip( + # DGM "Only tests RedHat family packages till logrotation paths are resolved on Ubuntu/Debian, see issue 65231" + # DGM ) match = False for proc in psutil.Process(salt_master.pid).children(): diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index b1bee9d60afa..dc710f1b7ccc 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -6,7 +6,7 @@ from pytestskipmarkers.utils import platform -@pytest.mark.skip_on_windows +# DGM @pytest.mark.skip_on_windows def test_salt_version(version, install_salt): """ Test version output from salt --version @@ -35,6 +35,7 @@ def test_salt_version(version, install_salt): @pytest.mark.skip_on_windows +@pytest.mark.skip_on_darwin def test_salt_versions_report_master(install_salt): """ Test running --versions-report on master @@ -52,7 +53,7 @@ def test_salt_versions_report_master(install_salt): ret.stdout.matcher.fnmatch_lines([f"*{py_version}*"]) -@pytest.mark.skip_on_windows +# DGM @pytest.mark.skip_on_windows def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion): """ Test running test.versions_report on minion @@ -104,7 +105,8 @@ def test_compare_versions(binary, install_salt): ) -@pytest.mark.skip_unless_on_darwin() +# DGM +@pytest.mark.skip_on_windows @pytest.mark.parametrize( "symlink", [ From 6874ff572e2ba832c827df6ebfb63b2f11d24035 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 21 Aug 2024 10:53:55 -0600 Subject: [PATCH 09/97] Revert syndic from master opts to check test failures --- salt/config/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index cffce8929f6a..87649b5e2ad5 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -2505,8 +2505,8 @@ def syndic_config( # DGM conf/suse/master has syndic_user for running syndic as a different user # DGM use urrently set it to 'salt' # DGM testing getting the master_opts user, and run syndic as that - # DGM "user": opts.get("syndic_user", opts["user"]), - "user": opts.get("syndic_user", master_opts["user"]), + "user": opts.get("syndic_user", opts["user"]), + # DGM test revert "user": opts.get("syndic_user", master_opts["user"]), "sock_dir": os.path.join( opts["cachedir"], opts.get("syndic_sock_dir", opts["sock_dir"]) ), From b0e1dfd7b32c9b155b9190868ea17ec6823fef46 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 21 Aug 2024 14:04:53 -0600 Subject: [PATCH 10/97] Revert logrotation test to RedHat family only --- .../pytests/pkg/integration/test_salt_user.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 96180a972fe9..864528b01cb2 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -235,17 +235,17 @@ def test_paths_log_rotation( ): pytest.skip("Package path ownership was changed in salt 3006.4") - # DGM if install_salt.distro_id not in ( - # DGM "almalinux", - # DGM "rocky", - # DGM "centos", - # DGM "redhat", - # DGM "amzn", - # DGM "fedora", - # DGM ): - # DGM pytest.skip( - # DGM "Only tests RedHat family packages till logrotation paths are resolved on Ubuntu/Debian, see issue 65231" - # DGM ) + if install_salt.distro_id not in ( + "almalinux", + "rocky", + "centos", + "redhat", + "amzn", + "fedora", + ): + pytest.skip( + "Only tests RedHat family packages till logrotation paths are resolved on Ubuntu/Debian, see issue 65231" + ) match = False for proc in psutil.Process(salt_master.pid).children(): From 2c8becaf8115ba272668c730bf3b04e51f121a06 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 23 Aug 2024 13:18:17 -0600 Subject: [PATCH 11/97] Adjust tests for syndic --- salt/config/__init__.py | 4 ++-- .../pytests/pkg/integration/test_salt_api.py | 6 ++++++ .../pytests/pkg/integration/test_salt_user.py | 8 ++++---- tests/pytests/pkg/integration/test_version.py | 19 ++++++++++++++----- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 87649b5e2ad5..cffce8929f6a 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -2505,8 +2505,8 @@ def syndic_config( # DGM conf/suse/master has syndic_user for running syndic as a different user # DGM use urrently set it to 'salt' # DGM testing getting the master_opts user, and run syndic as that - "user": opts.get("syndic_user", opts["user"]), - # DGM test revert "user": opts.get("syndic_user", master_opts["user"]), + # DGM "user": opts.get("syndic_user", opts["user"]), + "user": opts.get("syndic_user", master_opts["user"]), "sock_dir": os.path.join( opts["cachedir"], opts.get("syndic_sock_dir", opts["sock_dir"]) ), diff --git a/tests/pytests/pkg/integration/test_salt_api.py b/tests/pytests/pkg/integration/test_salt_api.py index a4f11973748f..f01e7c5fd7c2 100644 --- a/tests/pytests/pkg/integration/test_salt_api.py +++ b/tests/pytests/pkg/integration/test_salt_api.py @@ -9,6 +9,12 @@ def test_salt_api(api_request, salt_master, install_salt): """ Test running a command against the salt api """ + # DGM TBD 66672 is merged, need to remove this check and see if test passes + if install_salt.distro_id in ("ubuntu", "debian"): + pytest.skip( + "Package test are getting reworked in https://github.com/saltstack/salt/issues/66672" + ) + assert salt_master.is_running() ret = api_request.post( diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 864528b01cb2..941f8c758d9f 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -92,13 +92,13 @@ def test_salt_user_master(salt_master, install_salt): """ Test the correct user is running the Salt Master """ - # DGM assert salt_master.is_running() - - for count in range(0, 10): + for count in range(0, 30): if salt_master.is_running(): break else: - time.sleep(1) + time.sleep(2) + + print(f"DGM test_salt_user_master, salt_master '{salt_master}'", flush=True) assert salt_master.is_running() match = False diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index dc710f1b7ccc..815df570d936 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -1,12 +1,13 @@ import os.path import pathlib import subprocess +import time import pytest from pytestskipmarkers.utils import platform -# DGM @pytest.mark.skip_on_windows +@pytest.mark.skip_on_windows def test_salt_version(version, install_salt): """ Test version output from salt --version @@ -53,12 +54,19 @@ def test_salt_versions_report_master(install_salt): ret.stdout.matcher.fnmatch_lines([f"*{py_version}*"]) -# DGM @pytest.mark.skip_on_windows +@pytest.mark.skip_on_windows def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion): """ Test running test.versions_report on minion """ # Make sure the minion is running + for count in range(0, 30): + if salt_minion.is_running(): + break + else: + time.sleep(2) + + print(f"DGM test_salt_user_mnion, salt_minion '{salt_minion}'", flush=True) assert salt_minion.is_running() # Make sure we can ping the minion ... @@ -78,6 +86,8 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion): ret.stdout.matcher.fnmatch_lines(["*Salt Version:*"]) +@pytest.mark.skip_on_windows +@pytest.mark.skip_on_darwin @pytest.mark.parametrize( "binary", ["master", "cloud", "syndic", "minion", "call", "api"] ) @@ -105,8 +115,8 @@ def test_compare_versions(binary, install_salt): ) -# DGM @pytest.mark.skip_on_windows +@pytest.mark.skip_on_darwin @pytest.mark.parametrize( "symlink", [ @@ -134,8 +144,7 @@ def test_symlinks_created(version, symlink, install_salt): ret.stdout.matcher.fnmatch_lines([f"*{version}*"]) -@pytest.mark.skip_on_windows -@pytest.mark.skip_on_darwin +@pytest.mark.skip_unless_on_linux def test_compare_pkg_versions_redhat_rc(version, install_salt): """ Test compare pkg versions for redhat RC packages. A tilde should be included From fbd6b7490022eaed1f0eb7845105c1a387bbcd4a Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 23 Aug 2024 15:20:25 -0600 Subject: [PATCH 12/97] Further debug tests --- pkg/common/salt-api.service | 2 +- .../pytests/pkg/integration/test_salt_user.py | 5 +++- tests/pytests/pkg/integration/test_version.py | 24 +++++++++++++++++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/pkg/common/salt-api.service b/pkg/common/salt-api.service index 3eefa7649009..3e47a2e6366a 100644 --- a/pkg/common/salt-api.service +++ b/pkg/common/salt-api.service @@ -2,7 +2,7 @@ Description=The Salt API Documentation=man:salt-api(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltproject.io/en/latest/contents.html After=network.target -PartOf=salt-master.service +# DGM PartOf=salt-master.service [Service] Type=notify diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 941f8c758d9f..07c29837e901 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -98,7 +98,10 @@ def test_salt_user_master(salt_master, install_salt): else: time.sleep(2) - print(f"DGM test_salt_user_master, salt_master '{salt_master}'", flush=True) + print( + f"DGM test_salt_user_master, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", + flush=True, + ) assert salt_master.is_running() match = False diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index 815df570d936..eb4278732570 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -55,7 +55,7 @@ def test_salt_versions_report_master(install_salt): @pytest.mark.skip_on_windows -def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion): +def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion, salt_master): """ Test running test.versions_report on minion """ @@ -66,13 +66,33 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion): else: time.sleep(2) - print(f"DGM test_salt_user_mnion, salt_minion '{salt_minion}'", flush=True) + print( + f"DGM test_salt_user_minion, salt_minion '{salt_minion}' and is_running '{salt_minion.is_running()}'", + flush=True, + ) assert salt_minion.is_running() + # DGM + # Make sure the master is running + for count in range(0, 30): + if salt_master.is_running(): + break + else: + time.sleep(2) + + print( + f"DGM test_salt_user_minion, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", + flush=True, + ) + assert salt_master.is_running() + # DGM + # Make sure we can ping the minion ... ret = salt_cli.run( "--timeout=300", "test.ping", minion_tgt=salt_minion.id, _timeout=300 ) + print(f"DGM test_salt_user_minion, test.ping ret '{ret}'", flush=True) + assert ret.returncode == 0 assert ret.data is True ret = salt_cli.run( From 7be41f56db50bac4fcfb7d32dda160ed3a0e2953 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 26 Aug 2024 17:28:53 -0600 Subject: [PATCH 13/97] Updated tests, and debugging --- tests/pytests/pkg/integration/test_version.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index eb4278732570..1402a4ef79fd 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -62,6 +62,10 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion, salt_ # Make sure the minion is running for count in range(0, 30): if salt_minion.is_running(): + print( + f"DGM test_salt_user_minion, salt_minion is running, final count '{count}'", + flush=True, + ) break else: time.sleep(2) @@ -76,6 +80,10 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion, salt_ # Make sure the master is running for count in range(0, 30): if salt_master.is_running(): + print( + f"DGM test_salt_user_minion, salt_master is running, final count '{count}'", + flush=True, + ) break else: time.sleep(2) @@ -85,11 +93,16 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion, salt_ flush=True, ) assert salt_master.is_running() + + ret = salt_call_cli.run("--local", "test.ping") + print( + f"DGM test_salt_user_minion, salt_call local test.ping ret '{ret}'", flush=True + ) # DGM # Make sure we can ping the minion ... ret = salt_cli.run( - "--timeout=300", "test.ping", minion_tgt=salt_minion.id, _timeout=300 + "--timeout=600", "test.ping", minion_tgt=salt_minion.id, _timeout=600 ) print(f"DGM test_salt_user_minion, test.ping ret '{ret}'", flush=True) @@ -135,8 +148,7 @@ def test_compare_versions(binary, install_salt): ) -@pytest.mark.skip_on_windows -@pytest.mark.skip_on_darwin +@pytest.mark.skip_unless_on_darwin @pytest.mark.parametrize( "symlink", [ From 4ee0f9410b23d787d75e9ab1b39f393c349d89d9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 27 Aug 2024 09:45:46 -0600 Subject: [PATCH 14/97] Rearrangment of test parameters --- tests/pytests/pkg/integration/test_salt_api.py | 2 +- tests/pytests/pkg/integration/test_salt_ufw.py | 4 ++-- tests/pytests/pkg/integration/test_salt_user.py | 10 +++++++--- tests/pytests/pkg/integration/test_version.py | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/pytests/pkg/integration/test_salt_api.py b/tests/pytests/pkg/integration/test_salt_api.py index f01e7c5fd7c2..ce9ffce8656e 100644 --- a/tests/pytests/pkg/integration/test_salt_api.py +++ b/tests/pytests/pkg/integration/test_salt_api.py @@ -5,7 +5,7 @@ ] -def test_salt_api(api_request, salt_master, install_salt): +def test_salt_api(api_request, install_salt, salt_master): """ Test running a command against the salt api """ diff --git a/tests/pytests/pkg/integration/test_salt_ufw.py b/tests/pytests/pkg/integration/test_salt_ufw.py index 0e0471aebf24..6c86e0a3339f 100644 --- a/tests/pytests/pkg/integration/test_salt_ufw.py +++ b/tests/pytests/pkg/integration/test_salt_ufw.py @@ -9,8 +9,8 @@ @pytest.fixture def salt_systemd_setup( - salt_call_cli, install_salt, + salt_call_cli, ): """ Fixture to set systemd for salt packages to enabled and active @@ -31,7 +31,7 @@ def salt_systemd_setup( @pytest.mark.skip_if_binaries_missing("ufw") -def test_salt_ufw(salt_systemd_setup, salt_call_cli, install_salt): +def test_salt_ufw(salt_systemd_setup, install_salt, salt_call_cli): """ Test salt.ufw for Debian/Ubuntu salt-master """ diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 07c29837e901..70a08742970a 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -16,8 +16,8 @@ @pytest.fixture def salt_systemd_setup( - salt_call_cli, install_salt, + salt_call_cli, ): """ Fixture to set systemd for salt packages to enabled and active @@ -88,12 +88,16 @@ def pkg_paths_salt_user_exclusions(): return paths -def test_salt_user_master(salt_master, install_salt): +def test_salt_user_master(install_salt, salt_master): """ Test the correct user is running the Salt Master """ for count in range(0, 30): if salt_master.is_running(): + print( + f"DGM test_salt_user_master, salt_master is_running, count '{count}'", + flush=True, + ) break else: time.sleep(2) @@ -221,10 +225,10 @@ def test_pkg_paths( @pytest.mark.skip_if_binaries_missing("logrotate") def test_paths_log_rotation( + install_salt, salt_master, salt_minion, salt_call_cli, - install_salt, pkg_tests_account, ): """ diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index 1402a4ef79fd..01225dd2e6be 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -55,7 +55,7 @@ def test_salt_versions_report_master(install_salt): @pytest.mark.skip_on_windows -def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion, salt_master): +def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_master, salt_minion): """ Test running test.versions_report on minion """ From 770def921331000b5aaacbe8b821ad6311f27ca9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 28 Aug 2024 11:34:13 -0600 Subject: [PATCH 15/97] Ensure salt_master is left running after test stopped it --- tests/pytests/pkg/integration/test_salt_user.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 70a08742970a..bb79e9f5293f 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -420,3 +420,7 @@ def test_paths_log_rotation( bkup_count += 1 assert ret.returncode == 0 + + # ensure leave salt_master running + salt_master.start() + assert salt_master.is_running() is True From ba558deaa869471ceaee65aee6eb1730cbc9b92e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 29 Aug 2024 14:08:56 -0600 Subject: [PATCH 16/97] Altered debugging --- .../pytests/pkg/integration/test_salt_user.py | 46 ++++++++++----- tests/pytests/pkg/integration/test_version.py | 59 ++++++++++--------- 2 files changed, 61 insertions(+), 44 deletions(-) diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index bb79e9f5293f..99228d743049 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -2,13 +2,15 @@ import pathlib import subprocess import sys -import time import packaging.version import psutil import pytest from saltfactories.utils.tempfiles import temp_directory +## DGM import time + + pytestmark = [ pytest.mark.skip_unless_on_linux, ] @@ -92,20 +94,20 @@ def test_salt_user_master(install_salt, salt_master): """ Test the correct user is running the Salt Master """ - for count in range(0, 30): - if salt_master.is_running(): - print( - f"DGM test_salt_user_master, salt_master is_running, count '{count}'", - flush=True, - ) - break - else: - time.sleep(2) - - print( - f"DGM test_salt_user_master, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", - flush=True, - ) + ## DGM for count in range(0, 30): + ## DGM if salt_master.is_running(): + ## DGM print( + ## DGM f"DGM test_salt_user_master, salt_master is_running, count '{count}'", + ## DGM flush=True, + ## DGM ) + ## DGM break + ## DGM else: + ## DGM time.sleep(2) + + ## DGM print( + ## DGM f"DGM test_salt_user_master, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", + ## DGM flush=True, + ## DGM ) assert salt_master.is_running() match = False @@ -193,6 +195,12 @@ def test_pkg_paths( assert pkg_path.exists() for dirpath, sub_dirs, files in os.walk(pkg_path): path = pathlib.Path(dirpath) + + print( + f"DGM test_pkg_paths, dirpath '{str(dirpath)}', path '{str(path)}', path owner '{path.owner()}', path group '{path.group()}'", + flush=True, + ) + # Directories owned by salt:salt or their subdirs/files if ( str(path) in pkg_paths_salt_user or str(path) in salt_user_subdirs @@ -206,6 +214,10 @@ def test_pkg_paths( for file in files: file_path = path.joinpath(file) if str(file_path) not in pkg_paths_salt_user_exclusions: + print( + f"DGM test_pkg_paths, salt:salt file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", + flush=True, + ) assert file_path.owner() == "salt" # Directories owned by root:root else: @@ -216,6 +228,10 @@ def test_pkg_paths( continue file_path = path.joinpath(file) # Individual files owned by salt user + print( + f"DGM test_pkg_paths, root:root file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", + flush=True, + ) if str(file_path) in pkg_paths_salt_user: assert file_path.owner() == "salt" else: diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index 01225dd2e6be..ff78ee080b63 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -1,11 +1,12 @@ import os.path import pathlib import subprocess -import time import pytest from pytestskipmarkers.utils import platform +## DGM import time + @pytest.mark.skip_on_windows def test_salt_version(version, install_salt): @@ -60,38 +61,38 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_master, salt_ Test running test.versions_report on minion """ # Make sure the minion is running - for count in range(0, 30): - if salt_minion.is_running(): - print( - f"DGM test_salt_user_minion, salt_minion is running, final count '{count}'", - flush=True, - ) - break - else: - time.sleep(2) - - print( - f"DGM test_salt_user_minion, salt_minion '{salt_minion}' and is_running '{salt_minion.is_running()}'", - flush=True, - ) + ## DGM for count in range(0, 30): + ## DGM if salt_minion.is_running(): + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_minion is running, final count '{count}'", + ## DGM flush=True, + ## DGM ) + ## DGM break + ## DGM else: + ## DGM time.sleep(2) + + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_minion '{salt_minion}' and is_running '{salt_minion.is_running()}'", + ## DGM flush=True, + ## DGM ) assert salt_minion.is_running() # DGM # Make sure the master is running - for count in range(0, 30): - if salt_master.is_running(): - print( - f"DGM test_salt_user_minion, salt_master is running, final count '{count}'", - flush=True, - ) - break - else: - time.sleep(2) - - print( - f"DGM test_salt_user_minion, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", - flush=True, - ) + ## DGM for count in range(0, 30): + ## DGM if salt_master.is_running(): + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_master is running, final count '{count}'", + ## DGM flush=True, + ## DGM ) + ## DGM break + ## DGM else: + ## DGM time.sleep(2) + + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", + ## DGM flush=True, + ## DGM ) assert salt_master.is_running() ret = salt_call_cli.run("--local", "test.ping") From 067ee3ca04c2b9f8f9c50e29e970537b78e70135 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 3 Sep 2024 16:30:53 -0600 Subject: [PATCH 17/97] Further debugging --- .../pytests/pkg/integration/test_salt_user.py | 69 +++++++++++++------ tests/pytests/pkg/integration/test_version.py | 59 ++++++++-------- 2 files changed, 76 insertions(+), 52 deletions(-) diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 99228d743049..023da47d24ac 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -2,15 +2,13 @@ import pathlib import subprocess import sys +import time import packaging.version import psutil import pytest from saltfactories.utils.tempfiles import temp_directory -## DGM import time - - pytestmark = [ pytest.mark.skip_unless_on_linux, ] @@ -94,20 +92,20 @@ def test_salt_user_master(install_salt, salt_master): """ Test the correct user is running the Salt Master """ - ## DGM for count in range(0, 30): - ## DGM if salt_master.is_running(): - ## DGM print( - ## DGM f"DGM test_salt_user_master, salt_master is_running, count '{count}'", - ## DGM flush=True, - ## DGM ) - ## DGM break - ## DGM else: - ## DGM time.sleep(2) - - ## DGM print( - ## DGM f"DGM test_salt_user_master, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", - ## DGM flush=True, - ## DGM ) + for count in range(0, 30): + if salt_master.is_running(): + print( + f"DGM test_salt_user_master, salt_master is_running, count '{count}'", + flush=True, + ) + break + else: + time.sleep(2) + + print( + f"DGM test_salt_user_master, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", + flush=True, + ) assert salt_master.is_running() match = False @@ -179,6 +177,7 @@ def test_pkg_paths( pkg_paths, pkg_paths_salt_user, pkg_paths_salt_user_exclusions, + salt_call_cli, ): """ Test package paths ownership @@ -190,6 +189,32 @@ def test_pkg_paths( salt_user_subdirs = [] + print( + f"DGM test_pkg_paths, pkg_paths '{pkg_paths}', pkg_paths_salt_user '{pkg_paths_salt_user}', exclusions '{pkg_paths_salt_user_exclusions}'", + flush=True, + ) + + dgm_cmd = "ps -ef" + ret = salt_call_cli("--local", "cmd.run", dgm_cmd) + print( + f"DGM test_pkg_paths, test ps -ef, ret '{ret}'", + flush=True, + ) + + dgm_cmd = "ls -al /var/log/" + ret = salt_call_cli("--local", "cmd.run", dgm_cmd) + print( + f"DGM test_pkg_paths, test ls -al /var/log/, ret '{ret}'", + flush=True, + ) + + dgm_cmd = "ls -al /var/log/salt" + ret = salt_call_cli("--local", "cmd.run", dgm_cmd) + print( + f"DGM test_pkg_paths, test ls -al /var/log/salt, ret '{ret}'", + flush=True, + ) + for _path in pkg_paths: pkg_path = pathlib.Path(_path) assert pkg_path.exists() @@ -197,7 +222,7 @@ def test_pkg_paths( path = pathlib.Path(dirpath) print( - f"DGM test_pkg_paths, dirpath '{str(dirpath)}', path '{str(path)}', path owner '{path.owner()}', path group '{path.group()}'", + f"DGM test_pkg_paths, dirpath '{str(dirpath)}', path '{str(path)}', path owner '{path.owner()}', path group '{path.group()}', salt_user_subdirs '{salt_user_subdirs}', sub_dirs '{sub_dirs}', files '{files}'", flush=True, ) @@ -213,11 +238,11 @@ def test_pkg_paths( # Individual files owned by salt user for file in files: file_path = path.joinpath(file) + print( + f"DGM test_pkg_paths, salt:salt file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", + flush=True, + ) if str(file_path) not in pkg_paths_salt_user_exclusions: - print( - f"DGM test_pkg_paths, salt:salt file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", - flush=True, - ) assert file_path.owner() == "salt" # Directories owned by root:root else: diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index ff78ee080b63..01225dd2e6be 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -1,12 +1,11 @@ import os.path import pathlib import subprocess +import time import pytest from pytestskipmarkers.utils import platform -## DGM import time - @pytest.mark.skip_on_windows def test_salt_version(version, install_salt): @@ -61,38 +60,38 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_master, salt_ Test running test.versions_report on minion """ # Make sure the minion is running - ## DGM for count in range(0, 30): - ## DGM if salt_minion.is_running(): - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_minion is running, final count '{count}'", - ## DGM flush=True, - ## DGM ) - ## DGM break - ## DGM else: - ## DGM time.sleep(2) - - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_minion '{salt_minion}' and is_running '{salt_minion.is_running()}'", - ## DGM flush=True, - ## DGM ) + for count in range(0, 30): + if salt_minion.is_running(): + print( + f"DGM test_salt_user_minion, salt_minion is running, final count '{count}'", + flush=True, + ) + break + else: + time.sleep(2) + + print( + f"DGM test_salt_user_minion, salt_minion '{salt_minion}' and is_running '{salt_minion.is_running()}'", + flush=True, + ) assert salt_minion.is_running() # DGM # Make sure the master is running - ## DGM for count in range(0, 30): - ## DGM if salt_master.is_running(): - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_master is running, final count '{count}'", - ## DGM flush=True, - ## DGM ) - ## DGM break - ## DGM else: - ## DGM time.sleep(2) - - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", - ## DGM flush=True, - ## DGM ) + for count in range(0, 30): + if salt_master.is_running(): + print( + f"DGM test_salt_user_minion, salt_master is running, final count '{count}'", + flush=True, + ) + break + else: + time.sleep(2) + + print( + f"DGM test_salt_user_minion, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", + flush=True, + ) assert salt_master.is_running() ret = salt_call_cli.run("--local", "test.ping") From 2bd4e000a8b3632ee7a05aa38527e1e5557aa996 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 4 Sep 2024 10:44:19 -0600 Subject: [PATCH 18/97] Fix typo --- tests/pytests/pkg/integration/test_salt_user.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 023da47d24ac..7f9a21e90e33 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -195,21 +195,21 @@ def test_pkg_paths( ) dgm_cmd = "ps -ef" - ret = salt_call_cli("--local", "cmd.run", dgm_cmd) + ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) print( f"DGM test_pkg_paths, test ps -ef, ret '{ret}'", flush=True, ) dgm_cmd = "ls -al /var/log/" - ret = salt_call_cli("--local", "cmd.run", dgm_cmd) + ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) print( f"DGM test_pkg_paths, test ls -al /var/log/, ret '{ret}'", flush=True, ) dgm_cmd = "ls -al /var/log/salt" - ret = salt_call_cli("--local", "cmd.run", dgm_cmd) + ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) print( f"DGM test_pkg_paths, test ls -al /var/log/salt, ret '{ret}'", flush=True, From eada163968045edece507e3cca1ac528a38cc154 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 4 Sep 2024 14:52:35 -0600 Subject: [PATCH 19/97] Commented out debugging print statements --- salt/config/__init__.py | 6 +- .../pytests/pkg/integration/test_salt_user.py | 90 +++++++++---------- tests/pytests/pkg/integration/test_version.py | 44 ++++----- 3 files changed, 70 insertions(+), 70 deletions(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index cffce8929f6a..e9e033d656ce 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -2514,9 +2514,9 @@ def syndic_config( "cachedir": master_opts["cachedir"], } opts.update(syndic_opts) - log.warning("DGM syndic_config, user is set to '%s'", opts["user"]) - dgm_user = opts["user"] - print(f"DGM syndic_config, user is set to '{dgm_user}'", flush=True) + ## DGM log.warning("DGM syndic_config, user is set to '%s'", opts["user"]) + ## DGM dgm_user = opts["user"] + ## DGM print(f"DGM syndic_config, user is set to '{dgm_user}'", flush=True) # Prepend root_dir to other paths prepend_root_dirs = [ diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 7f9a21e90e33..a8bb166f068e 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -94,18 +94,18 @@ def test_salt_user_master(install_salt, salt_master): """ for count in range(0, 30): if salt_master.is_running(): - print( - f"DGM test_salt_user_master, salt_master is_running, count '{count}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_salt_user_master, salt_master is_running, count '{count}'", + ## DGM flush=True, + ## DGM ) break else: time.sleep(2) - print( - f"DGM test_salt_user_master, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_salt_user_master, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", + ## DGM flush=True, + ## DGM ) assert salt_master.is_running() match = False @@ -189,31 +189,31 @@ def test_pkg_paths( salt_user_subdirs = [] - print( - f"DGM test_pkg_paths, pkg_paths '{pkg_paths}', pkg_paths_salt_user '{pkg_paths_salt_user}', exclusions '{pkg_paths_salt_user_exclusions}'", - flush=True, - ) - - dgm_cmd = "ps -ef" - ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) - print( - f"DGM test_pkg_paths, test ps -ef, ret '{ret}'", - flush=True, - ) - - dgm_cmd = "ls -al /var/log/" - ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) - print( - f"DGM test_pkg_paths, test ls -al /var/log/, ret '{ret}'", - flush=True, - ) - - dgm_cmd = "ls -al /var/log/salt" - ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) - print( - f"DGM test_pkg_paths, test ls -al /var/log/salt, ret '{ret}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_pkg_paths, pkg_paths '{pkg_paths}', pkg_paths_salt_user '{pkg_paths_salt_user}', exclusions '{pkg_paths_salt_user_exclusions}'", + ## DGM flush=True, + ## DGM ) + + ## DGM dgm_cmd = "ps -ef" + ## DGM ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) + ## DGM print( + ## DGM f"DGM test_pkg_paths, test ps -ef, ret '{ret}'", + ## DGM flush=True, + ## DGM ) + + ## DGM dgm_cmd = "ls -al /var/log/" + ## DGM ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) + ## DGM print( + ## DGM f"DGM test_pkg_paths, test ls -al /var/log/, ret '{ret}'", + ## DGM flush=True, + ## DGM ) + + ## DGM dgm_cmd = "ls -al /var/log/salt" + ## DGM ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) + ## DGM print( + ## DGM f"DGM test_pkg_paths, test ls -al /var/log/salt, ret '{ret}'", + ## DGM flush=True, + ## DGM ) for _path in pkg_paths: pkg_path = pathlib.Path(_path) @@ -221,10 +221,10 @@ def test_pkg_paths( for dirpath, sub_dirs, files in os.walk(pkg_path): path = pathlib.Path(dirpath) - print( - f"DGM test_pkg_paths, dirpath '{str(dirpath)}', path '{str(path)}', path owner '{path.owner()}', path group '{path.group()}', salt_user_subdirs '{salt_user_subdirs}', sub_dirs '{sub_dirs}', files '{files}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_pkg_paths, dirpath '{str(dirpath)}', path '{str(path)}', path owner '{path.owner()}', path group '{path.group()}', salt_user_subdirs '{salt_user_subdirs}', sub_dirs '{sub_dirs}', files '{files}'", + ## DGM flush=True, + ## DGM ) # Directories owned by salt:salt or their subdirs/files if ( @@ -238,10 +238,10 @@ def test_pkg_paths( # Individual files owned by salt user for file in files: file_path = path.joinpath(file) - print( - f"DGM test_pkg_paths, salt:salt file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_pkg_paths, salt:salt file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", + ## DGM flush=True, + ## DGM ) if str(file_path) not in pkg_paths_salt_user_exclusions: assert file_path.owner() == "salt" # Directories owned by root:root @@ -253,10 +253,10 @@ def test_pkg_paths( continue file_path = path.joinpath(file) # Individual files owned by salt user - print( - f"DGM test_pkg_paths, root:root file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_pkg_paths, root:root file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", + ## DGM flush=True, + ## DGM ) if str(file_path) in pkg_paths_salt_user: assert file_path.owner() == "salt" else: diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index 01225dd2e6be..1c29d033c77c 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -62,49 +62,49 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_master, salt_ # Make sure the minion is running for count in range(0, 30): if salt_minion.is_running(): - print( - f"DGM test_salt_user_minion, salt_minion is running, final count '{count}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_minion is running, final count '{count}'", + ## DGM flush=True, + ## DGM ) break else: time.sleep(2) - print( - f"DGM test_salt_user_minion, salt_minion '{salt_minion}' and is_running '{salt_minion.is_running()}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_minion '{salt_minion}' and is_running '{salt_minion.is_running()}'", + ## DGM flush=True, + ## DGM ) assert salt_minion.is_running() # DGM # Make sure the master is running for count in range(0, 30): if salt_master.is_running(): - print( - f"DGM test_salt_user_minion, salt_master is running, final count '{count}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_master is running, final count '{count}'", + ## DGM flush=True, + ## DGM ) break else: time.sleep(2) - print( - f"DGM test_salt_user_minion, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", + ## DGM flush=True, + ## DGM ) assert salt_master.is_running() - ret = salt_call_cli.run("--local", "test.ping") - print( - f"DGM test_salt_user_minion, salt_call local test.ping ret '{ret}'", flush=True - ) - # DGM + ## DGM ret = salt_call_cli.run("--local", "test.ping") + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_call local test.ping ret '{ret}'", flush=True + ## DGM ) + ## DGM # DGM # Make sure we can ping the minion ... ret = salt_cli.run( "--timeout=600", "test.ping", minion_tgt=salt_minion.id, _timeout=600 ) - print(f"DGM test_salt_user_minion, test.ping ret '{ret}'", flush=True) + ## DGM print(f"DGM test_salt_user_minion, test.ping ret '{ret}'", flush=True) assert ret.returncode == 0 assert ret.data is True From 1ebd3c6b8ebfcd5a3f8e8bdb49f34bc6b0d4fb2a Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 6 Sep 2024 09:54:10 -0600 Subject: [PATCH 20/97] Forced rebuild --- salt/config/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index e9e033d656ce..cd03a3ab745b 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -2502,6 +2502,7 @@ def syndic_config( ), ) ), + ## DGM line to force rebuild # DGM conf/suse/master has syndic_user for running syndic as a different user # DGM use urrently set it to 'salt' # DGM testing getting the master_opts user, and run syndic as that From a7da90d8f675925f87e2fab26b034bfc795e5110 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 6 Sep 2024 15:59:14 -0600 Subject: [PATCH 21/97] Removed commented out debug print statements --- pkg/common/salt-api.service | 1 - salt/config/__init__.py | 8 ---- salt/minion.py | 21 --------- .../pytests/pkg/integration/test_salt_api.py | 1 - .../pytests/pkg/integration/test_salt_user.py | 47 ------------------- tests/pytests/pkg/integration/test_version.py | 24 ---------- 6 files changed, 102 deletions(-) diff --git a/pkg/common/salt-api.service b/pkg/common/salt-api.service index 3e47a2e6366a..d0b6d741202b 100644 --- a/pkg/common/salt-api.service +++ b/pkg/common/salt-api.service @@ -2,7 +2,6 @@ Description=The Salt API Documentation=man:salt-api(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltproject.io/en/latest/contents.html After=network.target -# DGM PartOf=salt-master.service [Service] Type=notify diff --git a/salt/config/__init__.py b/salt/config/__init__.py index cd03a3ab745b..8226ba452b80 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -2502,11 +2502,6 @@ def syndic_config( ), ) ), - ## DGM line to force rebuild - # DGM conf/suse/master has syndic_user for running syndic as a different user - # DGM use urrently set it to 'salt' - # DGM testing getting the master_opts user, and run syndic as that - # DGM "user": opts.get("syndic_user", opts["user"]), "user": opts.get("syndic_user", master_opts["user"]), "sock_dir": os.path.join( opts["cachedir"], opts.get("syndic_sock_dir", opts["sock_dir"]) @@ -2515,9 +2510,6 @@ def syndic_config( "cachedir": master_opts["cachedir"], } opts.update(syndic_opts) - ## DGM log.warning("DGM syndic_config, user is set to '%s'", opts["user"]) - ## DGM dgm_user = opts["user"] - ## DGM print(f"DGM syndic_config, user is set to '{dgm_user}'", flush=True) # Prepend root_dir to other paths prepend_root_dirs = [ diff --git a/salt/minion.py b/salt/minion.py index 64057e81fccc..2c2585637420 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -280,11 +280,6 @@ def get_proc_dir(cachedir, **kwargs): gid. Must be int. Works only on unix/unix like systems. """ # pylint: disable=logging-fstring-interpolation - ## DGM log.warning(f"DGM get_proc_dir entry, cachedir '{cachedir}', kwargs '{kwargs}'") - ## DGM print( - ## DGM f"DGM get_proc_dir entry, cachedir '{cachedir}', kwargs '{kwargs}'", flush=True - ## DGM ) - fn_ = os.path.join(cachedir, "proc") mode = kwargs.pop("mode", None) @@ -312,28 +307,12 @@ def get_proc_dir(cachedir, **kwargs): gid = kwargs.pop("gid", -1) # pylint: disable=logging-fstring-interpolation - ## DGM log.warning( - ## DGM f"DGM get_proc_dir chown, d_stat '{d_stat}', uid '{uid}', gid '{gid}'" - ## DGM ) - ## DGM print( - ## DGM f"DGM get_proc_dir chown, d_stat '{d_stat}', uid '{uid}', gid '{gid}'", - ## DGM flush=True, - ## DGM ) - # if uid and gid are both -1 then go ahead with # no changes at all if (d_stat.st_uid != uid or d_stat.st_gid != gid) and [ i for i in (uid, gid) if i != -1 ]: # pylint: disable=logging-fstring-interpolation - ## DGM log.warning( - ## DGM f"DGM get_proc_dir chown file, file '{fn_}' , uid '{uid}', gid '{gid}'" - ## DGM ) - ## DGM print( - ## DGM f"DGM get_proc_dir chown file, file '{fn_}' , uid '{uid}', gid '{gid}'", - ## DGM flush=True, - ## DGM ) - os.chown(fn_, uid, gid) return fn_ diff --git a/tests/pytests/pkg/integration/test_salt_api.py b/tests/pytests/pkg/integration/test_salt_api.py index ce9ffce8656e..b13775bd7942 100644 --- a/tests/pytests/pkg/integration/test_salt_api.py +++ b/tests/pytests/pkg/integration/test_salt_api.py @@ -9,7 +9,6 @@ def test_salt_api(api_request, install_salt, salt_master): """ Test running a command against the salt api """ - # DGM TBD 66672 is merged, need to remove this check and see if test passes if install_salt.distro_id in ("ubuntu", "debian"): pytest.skip( "Package test are getting reworked in https://github.com/saltstack/salt/issues/66672" diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index a8bb166f068e..3978bfe9ca75 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -94,18 +94,10 @@ def test_salt_user_master(install_salt, salt_master): """ for count in range(0, 30): if salt_master.is_running(): - ## DGM print( - ## DGM f"DGM test_salt_user_master, salt_master is_running, count '{count}'", - ## DGM flush=True, - ## DGM ) break else: time.sleep(2) - ## DGM print( - ## DGM f"DGM test_salt_user_master, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", - ## DGM flush=True, - ## DGM ) assert salt_master.is_running() match = False @@ -189,43 +181,12 @@ def test_pkg_paths( salt_user_subdirs = [] - ## DGM print( - ## DGM f"DGM test_pkg_paths, pkg_paths '{pkg_paths}', pkg_paths_salt_user '{pkg_paths_salt_user}', exclusions '{pkg_paths_salt_user_exclusions}'", - ## DGM flush=True, - ## DGM ) - - ## DGM dgm_cmd = "ps -ef" - ## DGM ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) - ## DGM print( - ## DGM f"DGM test_pkg_paths, test ps -ef, ret '{ret}'", - ## DGM flush=True, - ## DGM ) - - ## DGM dgm_cmd = "ls -al /var/log/" - ## DGM ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) - ## DGM print( - ## DGM f"DGM test_pkg_paths, test ls -al /var/log/, ret '{ret}'", - ## DGM flush=True, - ## DGM ) - - ## DGM dgm_cmd = "ls -al /var/log/salt" - ## DGM ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) - ## DGM print( - ## DGM f"DGM test_pkg_paths, test ls -al /var/log/salt, ret '{ret}'", - ## DGM flush=True, - ## DGM ) - for _path in pkg_paths: pkg_path = pathlib.Path(_path) assert pkg_path.exists() for dirpath, sub_dirs, files in os.walk(pkg_path): path = pathlib.Path(dirpath) - ## DGM print( - ## DGM f"DGM test_pkg_paths, dirpath '{str(dirpath)}', path '{str(path)}', path owner '{path.owner()}', path group '{path.group()}', salt_user_subdirs '{salt_user_subdirs}', sub_dirs '{sub_dirs}', files '{files}'", - ## DGM flush=True, - ## DGM ) - # Directories owned by salt:salt or their subdirs/files if ( str(path) in pkg_paths_salt_user or str(path) in salt_user_subdirs @@ -238,10 +199,6 @@ def test_pkg_paths( # Individual files owned by salt user for file in files: file_path = path.joinpath(file) - ## DGM print( - ## DGM f"DGM test_pkg_paths, salt:salt file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", - ## DGM flush=True, - ## DGM ) if str(file_path) not in pkg_paths_salt_user_exclusions: assert file_path.owner() == "salt" # Directories owned by root:root @@ -253,10 +210,6 @@ def test_pkg_paths( continue file_path = path.joinpath(file) # Individual files owned by salt user - ## DGM print( - ## DGM f"DGM test_pkg_paths, root:root file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", - ## DGM flush=True, - ## DGM ) if str(file_path) in pkg_paths_salt_user: assert file_path.owner() == "salt" else: diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index 1c29d033c77c..b7fa262fd53b 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -62,49 +62,25 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_master, salt_ # Make sure the minion is running for count in range(0, 30): if salt_minion.is_running(): - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_minion is running, final count '{count}'", - ## DGM flush=True, - ## DGM ) break else: time.sleep(2) - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_minion '{salt_minion}' and is_running '{salt_minion.is_running()}'", - ## DGM flush=True, - ## DGM ) assert salt_minion.is_running() - # DGM # Make sure the master is running for count in range(0, 30): if salt_master.is_running(): - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_master is running, final count '{count}'", - ## DGM flush=True, - ## DGM ) break else: time.sleep(2) - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", - ## DGM flush=True, - ## DGM ) assert salt_master.is_running() - ## DGM ret = salt_call_cli.run("--local", "test.ping") - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_call local test.ping ret '{ret}'", flush=True - ## DGM ) - ## DGM # DGM - # Make sure we can ping the minion ... ret = salt_cli.run( "--timeout=600", "test.ping", minion_tgt=salt_minion.id, _timeout=600 ) - ## DGM print(f"DGM test_salt_user_minion, test.ping ret '{ret}'", flush=True) assert ret.returncode == 0 assert ret.data is True From c738a2f2ed2e8821e0e16a22af69cdd257fcd61d Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 4 Sep 2024 10:22:25 -0600 Subject: [PATCH 22/97] Move installer tests to workflow --- .github/workflows/templates/ci.yml.jinja | 13 +++++++++++++ .github/workflows/test-installer-action-windows.yml | 12 ++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index eef2e77ba846..01b2fadc2100 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -39,6 +39,19 @@ <%- endif %> + <%- set job_name = "test-windows-installer" %> + <%- if includes.get(job_name, True) %> + <{ job_name }>: + <%- do conclusion_needs.append(job_name) %> + name: Installer + if: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + uses: ./.github/workflows/test-installer-action-windows.yml + needs: + - prepare-workflow + with: + changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} + + <%- endif %> <%- set job_name = "prepare-release" %> <%- if includes.get(job_name, True) %> diff --git a/.github/workflows/test-installer-action-windows.yml b/.github/workflows/test-installer-action-windows.yml index cf0b48556bb9..e3f91661f05a 100644 --- a/.github/workflows/test-installer-action-windows.yml +++ b/.github/workflows/test-installer-action-windows.yml @@ -1,13 +1,17 @@ --- name: Test Windows Installer -on: pull_request - -permissions: - contents: read +on: + workflow_call: + inputs: + changed-files: + required: true + type: string + description: JSON string containing information about changed files jobs: Test-Windows-Installer: + name: Test Windows Installer logic runs-on: - windows-latest From fb2297e11cd714fe5911e5d00916a00be56d566b Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 4 Sep 2024 10:45:19 -0600 Subject: [PATCH 23/97] Update workflows --- .github/workflows/ci.yml | 12 ++++ .github/workflows/nightly.yml | 12 ++++ .github/workflows/nsis-tests.yml | 66 +++++++++++++++++++ .github/workflows/scheduled.yml | 12 ++++ .github/workflows/staging.yml | 12 ++++ .github/workflows/templates/ci.yml.jinja | 6 +- .github/workflows/templates/layout.yml.jinja | 3 + .../test-installer-action-windows.yml | 42 ------------ 8 files changed, 120 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/nsis-tests.yml delete mode 100644 .github/workflows/test-installer-action-windows.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 207699290786..0b6de9cff239 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,6 +127,9 @@ jobs: - pkg/** - *pkg_requirements - *salt_added_modified + nsis_tests: + - added|modified: &nsis_tests + - pkg/windows/nsis/** testrun: - added|modified: - *pkg_requirements @@ -254,6 +257,14 @@ jobs: - prepare-workflow with: changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} + nsis-tests: + name: NSIS Tests + if: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + uses: ./.github/workflows/nsis-tests.yml + needs: + - prepare-workflow + with: + changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} prepare-release: name: "Prepare Release: ${{ needs.prepare-workflow.outputs.salt-version }}" @@ -2131,6 +2142,7 @@ jobs: - prepare-workflow - pre-commit - lint + - nsis-tests - build-docs - build-deps-onedir - build-salt-onedir diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 93c4c09f55e8..18e390d8ab7f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -176,6 +176,9 @@ jobs: - pkg/** - *pkg_requirements - *salt_added_modified + nsis_tests: + - added|modified: &nsis_tests + - pkg/windows/nsis/** testrun: - added|modified: - *pkg_requirements @@ -303,6 +306,14 @@ jobs: - prepare-workflow with: changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} + nsis-tests: + name: NSIS Tests + if: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + uses: ./.github/workflows/nsis-tests.yml + needs: + - prepare-workflow + with: + changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} prepare-release: name: "Prepare Release: ${{ needs.prepare-workflow.outputs.salt-version }}" @@ -3026,6 +3037,7 @@ jobs: - prepare-workflow - pre-commit - lint + - nsis-tests - build-docs - build-deps-onedir - build-salt-onedir diff --git a/.github/workflows/nsis-tests.yml b/.github/workflows/nsis-tests.yml new file mode 100644 index 000000000000..958f3a253038 --- /dev/null +++ b/.github/workflows/nsis-tests.yml @@ -0,0 +1,66 @@ +--- +name: Test NSIS Installer + +on: + workflow_call: + inputs: + changed-files: + required: true + type: string + description: JSON string containing information about changed files + +jobs: + Test-NSIS-Logic: + name: Logic Tests + runs-on: + - windows-latest + + steps: + + - name: Checkout Salt + uses: actions/checkout@v4 + + - name: Set Up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install NSIS + run: .\pkg\windows\install_nsis.cmd -CICD + shell: cmd + + - name: Build Test Installer + run: .\pkg\windows\nsis\tests\setup.cmd -CICD + shell: cmd + + - name: Run Config Tests + run: .\pkg\windows\nsis\tests\test.cmd -CICD .\config_tests + shell: cmd + + Test-NSIS-Stress: + name: Stress Tests + runs-on: + - windows-latest + if: ${{ contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) || fromJSON(inputs.changed-files)['nsis_tests'] }} + + steps: + + - name: Checkout Salt + uses: actions/checkout@v4 + + - name: Set Up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install NSIS + run: .\pkg\windows\install_nsis.cmd -CICD + shell: cmd + + - name: Build Test Installer + run: .\pkg\windows\nsis\tests\setup.cmd -CICD + shell: cmd + + - name: Run Stress Test + run: .\pkg\windows\nsis\tests\test.cmd -CICD .\stress_tests + shell: cmd diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 2ab7dc11b6d1..767b8c17f46d 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -166,6 +166,9 @@ jobs: - pkg/** - *pkg_requirements - *salt_added_modified + nsis_tests: + - added|modified: &nsis_tests + - pkg/windows/nsis/** testrun: - added|modified: - *pkg_requirements @@ -293,6 +296,14 @@ jobs: - prepare-workflow with: changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} + nsis-tests: + name: NSIS Tests + if: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + uses: ./.github/workflows/nsis-tests.yml + needs: + - prepare-workflow + with: + changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} prepare-release: name: "Prepare Release: ${{ needs.prepare-workflow.outputs.salt-version }}" @@ -2172,6 +2183,7 @@ jobs: - prepare-workflow - pre-commit - lint + - nsis-tests - build-docs - build-deps-onedir - build-salt-onedir diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 616582c55eca..2cf7786fe2c8 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -157,6 +157,9 @@ jobs: - pkg/** - *pkg_requirements - *salt_added_modified + nsis_tests: + - added|modified: &nsis_tests + - pkg/windows/nsis/** testrun: - added|modified: - *pkg_requirements @@ -293,6 +296,14 @@ jobs: - prepare-workflow with: changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} + nsis-tests: + name: NSIS Tests + if: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + uses: ./.github/workflows/nsis-tests.yml + needs: + - prepare-workflow + with: + changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} prepare-release: name: "Prepare Release: ${{ needs.prepare-workflow.outputs.salt-version }}" @@ -3023,6 +3034,7 @@ jobs: - prepare-workflow - pre-commit - lint + - nsis-tests - build-docs - build-deps-onedir - build-salt-onedir diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index 01b2fadc2100..91713863f187 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -39,13 +39,13 @@ <%- endif %> - <%- set job_name = "test-windows-installer" %> + <%- set job_name = "nsis-tests" %> <%- if includes.get(job_name, True) %> <{ job_name }>: <%- do conclusion_needs.append(job_name) %> - name: Installer + name: NSIS Tests if: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} - uses: ./.github/workflows/test-installer-action-windows.yml + uses: ./.github/workflows/nsis-tests.yml needs: - prepare-workflow with: diff --git a/.github/workflows/templates/layout.yml.jinja b/.github/workflows/templates/layout.yml.jinja index 0f0e795c0c63..2259e26bff44 100644 --- a/.github/workflows/templates/layout.yml.jinja +++ b/.github/workflows/templates/layout.yml.jinja @@ -175,6 +175,9 @@ jobs: - pkg/** - *pkg_requirements - *salt_added_modified + nsis_tests: + - added|modified: &nsis_tests + - pkg/windows/nsis/** testrun: - added|modified: - *pkg_requirements diff --git a/.github/workflows/test-installer-action-windows.yml b/.github/workflows/test-installer-action-windows.yml deleted file mode 100644 index e3f91661f05a..000000000000 --- a/.github/workflows/test-installer-action-windows.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -name: Test Windows Installer - -on: - workflow_call: - inputs: - changed-files: - required: true - type: string - description: JSON string containing information about changed files - -jobs: - Test-Windows-Installer: - name: Test Windows Installer logic - runs-on: - - windows-latest - - steps: - - - name: Checkout Salt - uses: actions/checkout@v4 - - - name: Set Up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install NSIS - run: .\pkg\windows\install_nsis.cmd -CICD - shell: cmd - - - name: Build Test Installer - run: .\pkg\windows\nsis\tests\setup.cmd -CICD - shell: cmd - - - name: Run Stress Test - run: .\pkg\windows\nsis\tests\test.cmd -CICD .\stress_tests - shell: cmd - - - name: Run Config Tests - run: .\pkg\windows\nsis\tests\test.cmd -CICD .\config_tests - shell: cmd From 676945eec7eaca3e22a960ddd7477d43b2b91994 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 4 Sep 2024 11:50:44 -0600 Subject: [PATCH 24/97] Only run tests when nsis files are modified --- .github/workflows/nsis-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nsis-tests.yml b/.github/workflows/nsis-tests.yml index 958f3a253038..e80ed43f4099 100644 --- a/.github/workflows/nsis-tests.yml +++ b/.github/workflows/nsis-tests.yml @@ -14,6 +14,7 @@ jobs: name: Logic Tests runs-on: - windows-latest + if: ${{ contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) || fromJSON(inputs.changed-files)['nsis_tests'] }} steps: From 8b16003cabe02ed137879f902b0a9284028f19a9 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 26 Aug 2024 15:05:12 -0600 Subject: [PATCH 25/97] Fix parsing of IPv4 mapped IPv6 addresses --- changelog/66837.fixed.md | 3 +++ salt/utils/network.py | 8 +++++++- tests/pytests/unit/utils/test_network.py | 12 +++++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 changelog/66837.fixed.md diff --git a/changelog/66837.fixed.md b/changelog/66837.fixed.md new file mode 100644 index 000000000000..ccbe4a1155fd --- /dev/null +++ b/changelog/66837.fixed.md @@ -0,0 +1,3 @@ +Issue 66837: Fixes an issue with the `network.local_port_tcp` function +where it was not parsing the IPv4 mapped IPv6 address correctly. The +``::ffff:`` is now removed and only the IP address is returned. diff --git a/salt/utils/network.py b/salt/utils/network.py index f964e25fa54b..bfa38e4fe9ff 100644 --- a/salt/utils/network.py +++ b/salt/utils/network.py @@ -1740,7 +1740,13 @@ def _netlink_tool_remote_on(port, which_end): continue if which_end == "local_port" and int(local_port) != int(port): continue - remotes.add(remote_host.strip("[]")) + + # Interpret IPv4-mapped IPv6 addresses as IPv4 (strip prefix) + remote_host = remote_host.strip("[]").lower() + if remote_host.startswith("::ffff:"): + remote_host = remote_host[7:] + + remotes.add(remote_host) if valid is False: remotes = None diff --git a/tests/pytests/unit/utils/test_network.py b/tests/pytests/unit/utils/test_network.py index 12d545b01545..cbc7fd3f14ae 100644 --- a/tests/pytests/unit/utils/test_network.py +++ b/tests/pytests/unit/utils/test_network.py @@ -11,7 +11,7 @@ from tests.support.mock import MagicMock, create_autospec, mock_open, patch pytestmark = [ - pytest.mark.skip_on_windows, + pytest.mark.windows_whitelisted, ] @@ -722,13 +722,13 @@ def test_netlink_tool_remote_on_a(): with patch("salt.utils.platform.is_linux", return_value=True): with patch("subprocess.check_output", return_value=LINUX_NETLINK_SS_OUTPUT): remotes = network._netlink_tool_remote_on("4506", "local_port") - assert remotes == {"192.168.122.177", "::ffff:127.0.0.1"} + assert remotes == {"192.168.122.177", "127.0.0.1"} def test_netlink_tool_remote_on_b(): with patch("subprocess.check_output", return_value=LINUX_NETLINK_SS_OUTPUT): remotes = network._netlink_tool_remote_on("4505", "remote_port") - assert remotes == {"127.0.0.1", "::ffff:1.2.3.4"} + assert remotes == {"127.0.0.1", "1.2.3.4"} def test_openbsd_remotes_on(): @@ -1430,6 +1430,7 @@ def test_isportopen_false(): assert ret is False +@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_isportopen(): ret = network.isportopen("127.0.0.1", "22") assert ret == 0 @@ -1445,6 +1446,7 @@ def test_get_socket(): assert ret.type == socket.SOCK_STREAM +@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_ip_to_host(grains): ret = network.ip_to_host("127.0.0.1") if grains["oscodename"] == "Photon": @@ -1506,6 +1508,7 @@ def test_rpad_ipv4_network(addr, expected): assert network.rpad_ipv4_network(addr) == expected +@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_hw_addr(linux_interfaces_dict, freebsd_interfaces_dict): with patch( @@ -1531,6 +1534,7 @@ def test_hw_addr(linux_interfaces_dict, freebsd_interfaces_dict): ) +@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_interface_and_ip(linux_interfaces_dict): with patch( @@ -1557,6 +1561,7 @@ def test_interface_and_ip(linux_interfaces_dict): assert ret == 'Interface "dog" not in available interfaces: "eth0", "lo"' +@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_subnets(linux_interfaces_dict): with patch( @@ -1581,6 +1586,7 @@ def test_in_subnet(caplog): assert not ret +@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_ip_addrs(linux_interfaces_dict): with patch( "salt.utils.network.linux_interfaces", From cae94bca24479fbf0e50fcc1789725ba63613a14 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 28 Aug 2024 14:23:13 -0600 Subject: [PATCH 26/97] Make all network unit tests run on Windows --- tests/pytests/unit/utils/test_network.py | 41 +++++++++++++----------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/tests/pytests/unit/utils/test_network.py b/tests/pytests/unit/utils/test_network.py index cbc7fd3f14ae..557f315a3372 100644 --- a/tests/pytests/unit/utils/test_network.py +++ b/tests/pytests/unit/utils/test_network.py @@ -7,6 +7,7 @@ import salt.exceptions import salt.utils.network import salt.utils.network as network +import salt.utils.platform from salt._compat import ipaddress from tests.support.mock import MagicMock, create_autospec, mock_open, patch @@ -1430,9 +1431,12 @@ def test_isportopen_false(): assert ret is False -@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_isportopen(): - ret = network.isportopen("127.0.0.1", "22") + if salt.utils.platform.is_windows(): + port = "135" + else: + port = "22" + ret = network.isportopen("127.0.0.1", port) assert ret == 0 @@ -1446,14 +1450,19 @@ def test_get_socket(): assert ret.type == socket.SOCK_STREAM -@pytest.mark.skip_on_windows(reason="Do not run on Windows") +# @pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_ip_to_host(grains): + if salt.utils.platform.is_windows(): + hostname = socket.gethostname() + else: + hostname = "localhost" + ret = network.ip_to_host("127.0.0.1") - if grains["oscodename"] == "Photon": + if grains.get("oscodename") == "Photon": # Photon returns this for IPv4 assert ret == "ipv6-localhost" else: - assert ret == "localhost" + assert ret == hostname ret = network.ip_to_host("2001:a71::1") assert ret is None @@ -1463,22 +1472,22 @@ def test_ip_to_host(grains): assert ret == "localhost6" elif grains["os_family"] == "Debian": if grains["osmajorrelease"] == 12: - assert ret == "localhost" + assert ret == hostname else: assert ret == "ip6-localhost" elif grains["os_family"] == "RedHat": if grains["oscodename"] == "Photon": assert ret == "ipv6-localhost" else: - assert ret == "localhost" + assert ret == hostname elif grains["os_family"] == "Arch": if grains.get("osmajorrelease", None) is None: # running doesn't have osmajorrelease grains - assert ret == "localhost" + assert ret == hostname else: assert ret == "ip6-localhost" else: - assert ret == "localhost" + assert ret == hostname @pytest.mark.parametrize( @@ -1508,11 +1517,10 @@ def test_rpad_ipv4_network(addr, expected): assert network.rpad_ipv4_network(addr) == expected -@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_hw_addr(linux_interfaces_dict, freebsd_interfaces_dict): with patch( - "salt.utils.network.linux_interfaces", + "salt.utils.network.interfaces", MagicMock(return_value=linux_interfaces_dict), ): hw_addrs = network.hw_addr("eth0") @@ -1534,11 +1542,10 @@ def test_hw_addr(linux_interfaces_dict, freebsd_interfaces_dict): ) -@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_interface_and_ip(linux_interfaces_dict): with patch( - "salt.utils.network.linux_interfaces", + "salt.utils.network.interfaces", MagicMock(return_value=linux_interfaces_dict), ): expected = [ @@ -1561,11 +1568,10 @@ def test_interface_and_ip(linux_interfaces_dict): assert ret == 'Interface "dog" not in available interfaces: "eth0", "lo"' -@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_subnets(linux_interfaces_dict): with patch( - "salt.utils.network.linux_interfaces", + "salt.utils.network.interfaces", MagicMock(return_value=linux_interfaces_dict), ): ret = network.subnets() @@ -1586,17 +1592,16 @@ def test_in_subnet(caplog): assert not ret -@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_ip_addrs(linux_interfaces_dict): with patch( - "salt.utils.network.linux_interfaces", + "salt.utils.network.interfaces", MagicMock(return_value=linux_interfaces_dict), ): ret = network.ip_addrs("eth0") assert ret == ["10.10.10.56"] with patch( - "salt.utils.network.linux_interfaces", + "salt.utils.network.interfaces", MagicMock(return_value=linux_interfaces_dict), ): ret = network.ip_addrs6("eth0") From 63b9da8bde36e79be0fc5cbcfc0545deaa71dcd1 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 15 Aug 2024 08:32:22 -0600 Subject: [PATCH 27/97] Fix pkg.removed with multiple versions on Windows --- changelog/61001.fixed.md | 2 + salt/states/pkg.py | 4 +- tests/pytests/functional/conftest.py | 13 +++++++ tests/pytests/functional/states/test_pkg.py | 43 +++++++++++++++++++-- 4 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 changelog/61001.fixed.md diff --git a/changelog/61001.fixed.md b/changelog/61001.fixed.md new file mode 100644 index 000000000000..f9e6acf934d2 --- /dev/null +++ b/changelog/61001.fixed.md @@ -0,0 +1,2 @@ +Fixed an issue uninstalling packages on Windows using pkg.removed where there +are multiple versions of the same software installed diff --git a/salt/states/pkg.py b/salt/states/pkg.py index b0ab5f7ef736..1d56b545dc69 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -2884,7 +2884,7 @@ def _uninstall( try: pkg_params = __salt__["pkg_resource.parse_targets"]( - name, pkgs, normalize=normalize + name, pkgs, normalize=normalize, version=version, **kwargs )[0] except MinionError as exc: return { @@ -2951,7 +2951,7 @@ def _uninstall( new = __salt__["pkg.list_pkgs"](versions_as_list=True, **kwargs) failed = [] for param in pkg_params: - if __grains__["os_family"] in ["Suse", "RedHat"]: + if __grains__["os_family"] in ["Suse", "RedHat", "Windows"]: # Check if the package version set to be removed is actually removed: if param in new and not pkg_params[param]: failed.append(param) diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index 2fb2246b6338..5274ce23d356 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -4,6 +4,8 @@ import pytest from saltfactories.utils.functional import Loaders +import salt.utils.platform + log = logging.getLogger(__name__) @@ -70,6 +72,17 @@ def minion_opts( }, } ) + + if salt.utils.platform.is_windows(): + # We need to setup winrepo on Windows + minion_config_overrides.update( + { + "winrepo_source_dir": "salt://winrepo_ng", + "winrepo_dir_ng": str(state_tree / "winrepo_ng"), + "winrepo_dir": str(state_tree / "winrepo"), + } + ) + factory = salt_factories.salt_minion_daemon( minion_id, defaults=minion_config_defaults or None, diff --git a/tests/pytests/functional/states/test_pkg.py b/tests/pytests/functional/states/test_pkg.py index 8bfac2589667..f4cea3e0e97e 100644 --- a/tests/pytests/functional/states/test_pkg.py +++ b/tests/pytests/functional/states/test_pkg.py @@ -20,12 +20,17 @@ pytest.mark.slow_test, pytest.mark.skip_if_not_root, pytest.mark.destructive_test, + pytest.mark.windows_whitelisted, pytest.mark.timeout_unless_on_windows(240), ] @pytest.fixture(scope="module", autouse=True) def refresh_db(grains, modules): + + if salt.utils.platform.is_windows(): + modules.winrepo.update_git_repos() + modules.pkg.refresh_db() # If this is Arch Linux, check if pacman is in use by another process @@ -43,7 +48,7 @@ def refresh_db(grains, modules): def refresh_keys(grains, modules): if grains["os_family"] == "Arch": # We should be running this periodically when building new test runner - # images, otherwise this could take several minuets to complete. + # images, otherwise this could take several minutes to complete. proc = subprocess.run(["pacman-key", "--refresh-keys"], check=False) if proc.returncode != 0: pytest.fail("pacman-key --refresh-keys command failed.") @@ -53,7 +58,7 @@ def refresh_keys(grains, modules): def PKG_TARGETS(grains): _PKG_TARGETS = ["figlet", "sl"] if grains["os"] == "Windows": - _PKG_TARGETS = ["vlc", "putty"] + _PKG_TARGETS = ["7zip", "putty"] elif grains["os"] == "Amazon": if grains["osfinger"] == "Amazon Linux-2023": _PKG_TARGETS = ["lynx", "gnuplot-minimal"] @@ -106,7 +111,9 @@ def PKG_CAP_TARGETS(grains): @pytest.fixture def PKG_32_TARGETS(grains): _PKG_32_TARGETS = [] - if grains["os_family"] == "RedHat" and grains["oscodename"] != "Photon": + if grains["os"] == "Windows": + _PKG_32_TARGETS = ["npp", "nsis"] + elif grains["os_family"] == "RedHat" and grains["oscodename"] != "Photon": if grains["os"] == "CentOS": if grains["osmajorrelease"] == 5: _PKG_32_TARGETS = ["xz-devel.i386"] @@ -205,6 +212,19 @@ def run_command(*names): return run_command +@pytest.fixture(scope="function") +def install_7zip(modules): + try: + modules.pkg.install(name="7zip", version="22.01.00.0") + modules.pkg.install(name="7zip", version="19.00.00.0") + assert modules.pkg.version("7zip") == "19.00.00.0,22.01.00.0" + yield + finally: + modules.pkg.remove(name="7zip", version="19.00.00.0") + modules.pkg.remove(name="7zip", version="22.01.00.0") + assert modules.pkg.version("7zip") == "" + + @pytest.mark.requires_salt_modules("pkg.version") @pytest.mark.requires_salt_states("pkg.installed", "pkg.removed") @pytest.mark.slow_test @@ -268,7 +288,8 @@ def test_pkg_003_installed_multipkg(caplog, PKG_TARGETS, modules, states, grains try: ret = states.pkg.installed(name=None, pkgs=PKG_TARGETS, refresh=False) assert ret.result is True - assert "WARNING" not in caplog.text + if not salt.utils.platform.is_windows(): + assert "WARNING" not in caplog.text finally: ret = states.pkg.removed(name=None, pkgs=PKG_TARGETS) assert ret.result is True @@ -1091,3 +1112,17 @@ def test_pkg_purged_with_removed_pkg(grains, PKG_TARGETS, states, modules): "installed": {}, "removed": {target: {"new": "", "old": version}}, } + + +@pytest.mark.skip_unless_on_windows() +def test_pkg_removed_with_version_multiple(install_7zip, modules, states): + """ + This tests removing a specific version of a package when multiple versions + are installed. This is specific to Windows. The only version I could find + that allowed multiple installs of differing versions was 7zip, so we'll use + that. + """ + ret = states.pkg.removed(name="7zip", version="19.00.00.0") + assert ret.result is True + current = modules.pkg.version("7zip") + assert current == "22.01.00.0" From de33a4010721c2479d226d1ea53b1a0aea166405 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 21 Aug 2024 14:36:05 -0600 Subject: [PATCH 28/97] Fix test_win_pkg with real winrepo location --- tests/pytests/functional/modules/test_win_pkg.py | 2 +- tests/pytests/functional/states/test_pkg.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/pytests/functional/modules/test_win_pkg.py b/tests/pytests/functional/modules/test_win_pkg.py index b68895ef6253..6bcfaa9bd849 100644 --- a/tests/pytests/functional/modules/test_win_pkg.py +++ b/tests/pytests/functional/modules/test_win_pkg.py @@ -29,7 +29,7 @@ def pkg(modules): def test_refresh_db(pkg, pkg_def_contents, state_tree, minion_opts): assert len(pkg.get_package_info("my-software")) == 0 - repo_dir = state_tree / "win" / "repo-ng" + repo_dir = state_tree / "winrepo_ng" with pytest.helpers.temp_file("my-software.sls", pkg_def_contents, repo_dir): pkg.refresh_db() assert len(pkg.get_package_info("my-software")) == 1 diff --git a/tests/pytests/functional/states/test_pkg.py b/tests/pytests/functional/states/test_pkg.py index f4cea3e0e97e..a56f06667e44 100644 --- a/tests/pytests/functional/states/test_pkg.py +++ b/tests/pytests/functional/states/test_pkg.py @@ -111,14 +111,14 @@ def PKG_CAP_TARGETS(grains): @pytest.fixture def PKG_32_TARGETS(grains): _PKG_32_TARGETS = [] - if grains["os"] == "Windows": - _PKG_32_TARGETS = ["npp", "nsis"] - elif grains["os_family"] == "RedHat" and grains["oscodename"] != "Photon": + if grains["os_family"] == "RedHat" and grains["oscodename"] != "Photon": if grains["os"] == "CentOS": if grains["osmajorrelease"] == 5: _PKG_32_TARGETS = ["xz-devel.i386"] else: _PKG_32_TARGETS.append("xz-devel.i686") + elif grains["os"] == "Windows": + _PKG_32_TARGETS = ["npp", "nsis"] if not _PKG_32_TARGETS: pytest.skip("No 32 bit packages have been specified for testing") return _PKG_32_TARGETS From 7437fe9230bee4477eb6cef261bcf552bc2311eb Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 27 Aug 2024 09:32:09 -0600 Subject: [PATCH 29/97] Gate passing version for Windows --- salt/states/pkg.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 1d56b545dc69..71b0cd30febb 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -2883,9 +2883,14 @@ def _uninstall( } try: - pkg_params = __salt__["pkg_resource.parse_targets"]( - name, pkgs, normalize=normalize, version=version, **kwargs - )[0] + if salt.utils.platform.is_windows(): + pkg_params = __salt__["pkg_resource.parse_targets"]( + name, pkgs, normalize=normalize, version=version, **kwargs + )[0] + else: + pkg_params = __salt__["pkg_resource.parse_targets"]( + name, pkgs, normalize=normalize + )[0] except MinionError as exc: return { "name": name, From d3c0f996a4a84e91a1c7326605061e2235cb38cb Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 29 Aug 2024 11:21:14 -0600 Subject: [PATCH 30/97] Revert gating for Windows, use sys instead of salt.utils.platform --- salt/states/pkg.py | 11 +++-------- tests/pytests/functional/conftest.py | 7 +++---- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 71b0cd30febb..1d56b545dc69 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -2883,14 +2883,9 @@ def _uninstall( } try: - if salt.utils.platform.is_windows(): - pkg_params = __salt__["pkg_resource.parse_targets"]( - name, pkgs, normalize=normalize, version=version, **kwargs - )[0] - else: - pkg_params = __salt__["pkg_resource.parse_targets"]( - name, pkgs, normalize=normalize - )[0] + pkg_params = __salt__["pkg_resource.parse_targets"]( + name, pkgs, normalize=normalize, version=version, **kwargs + )[0] except MinionError as exc: return { "name": name, diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index 5274ce23d356..0a8219b8f717 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -1,11 +1,10 @@ import logging import shutil +import sys import pytest from saltfactories.utils.functional import Loaders -import salt.utils.platform - log = logging.getLogger(__name__) @@ -73,8 +72,8 @@ def minion_opts( } ) - if salt.utils.platform.is_windows(): - # We need to setup winrepo on Windows + if sys.platform.startswith("win"): + # We need to set up winrepo on Windows minion_config_overrides.update( { "winrepo_source_dir": "salt://winrepo_ng", From 36c65391a1aa63c6466c8105439f4f5a20156bf7 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 30 Aug 2024 09:27:20 -0600 Subject: [PATCH 31/97] Add some SSH settings to fix the SSH Broken Pipe with Amazon Linux --- tests/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9d673a3092c9..3fc062c73366 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1381,7 +1381,9 @@ def sshd_server(salt_factories, sshd_config_dir, salt_master, grains): "X11DisplayOffset": "10", "PrintMotd": "no", "PrintLastLog": "yes", - "TCPKeepAlive": "yes", + # https://unix.stackexchange.com/a/616355 + "ServerAliveInterval": "20", + "TCPKeepAlive": "no", "AcceptEnv": "LANG LC_*", "UsePAM": "yes", } From 72bb50cb279902b1151a4c3f21ac1d102cfda37d Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 4 Sep 2024 09:38:45 -0600 Subject: [PATCH 32/97] Revert SSH settings --- tests/conftest.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3fc062c73366..9d673a3092c9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1381,9 +1381,7 @@ def sshd_server(salt_factories, sshd_config_dir, salt_master, grains): "X11DisplayOffset": "10", "PrintMotd": "no", "PrintLastLog": "yes", - # https://unix.stackexchange.com/a/616355 - "ServerAliveInterval": "20", - "TCPKeepAlive": "no", + "TCPKeepAlive": "yes", "AcceptEnv": "LANG LC_*", "UsePAM": "yes", } From f0e2cf4689770ba4688a269610fcb06aa1ca7b2c Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 26 Aug 2024 11:16:33 -0600 Subject: [PATCH 33/97] Remove salt.utils.data from fileserver The salt.utils.data util was used to support python 2. Since we no longer support python 2, we can start removing this. This gives us some great speed increases as we're removing a lot of overhead and unnecessary function calls --- salt/fileserver/__init__.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/salt/fileserver/__init__.py b/salt/fileserver/__init__.py index fe4b3b8e4968..b71af97b12b9 100644 --- a/salt/fileserver/__init__.py +++ b/salt/fileserver/__init__.py @@ -11,7 +11,6 @@ from collections.abc import Sequence import salt.loader -import salt.utils.data import salt.utils.files import salt.utils.path import salt.utils.url @@ -147,13 +146,7 @@ def check_file_list_cache(opts, form, list_cache, w_lock): opts.get("fileserver_list_cache_time", 20), list_cache, ) - return ( - salt.utils.data.decode( - salt.payload.load(fp_).get(form, []) - ), - False, - False, - ) + return salt.payload.load(fp_).get(form, []), False, False elif _lock_cache(w_lock): # Set the w_lock and go refresh_cache = True @@ -189,7 +182,7 @@ def check_env_cache(opts, env_cache): try: with salt.utils.files.fopen(env_cache, "rb") as fp_: log.trace("Returning env cache data from %s", env_cache) - return salt.utils.data.decode(salt.payload.load(fp_)) + return salt.payload.load(fp_) except OSError: pass return None From 50187433cfe693e0883b24ba5c8d53e290312fb1 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 28 Aug 2024 13:42:03 -0600 Subject: [PATCH 34/97] Add changelog for 66835 --- changelog/66835.fixed.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog/66835.fixed.md diff --git a/changelog/66835.fixed.md b/changelog/66835.fixed.md new file mode 100644 index 000000000000..07420788c5f6 --- /dev/null +++ b/changelog/66835.fixed.md @@ -0,0 +1,2 @@ +Removed ``salt.utils.data.decode`` usage from the fileserver. This function was +necessary to support Python 2. This speeds up loading the list cache by 80-90x. \ No newline at end of file From f9c9787251285d77703ba39802f74d0c90a16d59 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 29 Aug 2024 09:14:40 -0600 Subject: [PATCH 35/97] Fix pre-commit --- changelog/66835.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/66835.fixed.md b/changelog/66835.fixed.md index 07420788c5f6..33d932b7fdf1 100644 --- a/changelog/66835.fixed.md +++ b/changelog/66835.fixed.md @@ -1,2 +1,2 @@ Removed ``salt.utils.data.decode`` usage from the fileserver. This function was -necessary to support Python 2. This speeds up loading the list cache by 80-90x. \ No newline at end of file +necessary to support Python 2. This speeds up loading the list cache by 80-90x. From 468b26e91867f114a4891a3d38f7313aac1227d7 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sun, 18 Aug 2024 15:18:32 -0700 Subject: [PATCH 36/97] initial work on log once option --- salt/_logging/impl.py | 10 ++++++++++ salt/grains/core.py | 1 + 2 files changed, 11 insertions(+) diff --git a/salt/_logging/impl.py b/salt/_logging/impl.py index 832c72b4769a..2a12ddf3f941 100644 --- a/salt/_logging/impl.py +++ b/salt/_logging/impl.py @@ -157,6 +157,9 @@ def set_log_record_factory(factory): class SaltLoggingClass(LOGGING_LOGGER_CLASS, metaclass=LoggingMixinMeta): + + ONCECACHE = set() + def __new__(cls, *args): """ We override `__new__` in our logging logger class in order to provide @@ -233,10 +236,16 @@ def _log( stack_info=False, stacklevel=1, exc_info_on_loglevel=None, + once=False, ): if extra is None: extra = {} + if once: + if str(args) in self.ONCECACHE: + return + self.ONCECACHE.add(str(args)) + # pylint: disable=no-member current_jid = RequestContext.current.get("data", {}).get("jid", None) log_fmt_jid = RequestContext.current.get("opts", {}).get("log_fmt_jid", None) @@ -265,6 +274,7 @@ def _log( exc_info_on_loglevel ) ) + # XXX: extra is never None if extra is None: extra = {"exc_info_on_loglevel": exc_info_on_loglevel} else: diff --git a/salt/grains/core.py b/salt/grains/core.py index 51646f6f9791..119e78c8e9bb 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -1275,6 +1275,7 @@ def _virtual(osdata): "cannot execute it. Grains output might not be " "accurate.", command, + once=True, ) return grains From 3140aecd2ba4cd9564000b1c5c54ea8d4dd510fa Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 6 Dec 2023 16:59:38 -0700 Subject: [PATCH 37/97] Add more loader dunders to cp module. Using cp module to dip our feet into more explicit loader dunders. Most all of the existing dunders can be imported already. The one outlier being used by the cp module is __opts__. --- salt/loader/dunder.py | 3 +++ salt/modules/cp.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/loader/dunder.py b/salt/loader/dunder.py index d3027098b5ae..321e6f9a69db 100644 --- a/salt/loader/dunder.py +++ b/salt/loader/dunder.py @@ -8,3 +8,6 @@ __file_client__ = loader_context.named_context("__file_client__", default=None) +__context__ = loader_context.named_context("__context__") +__pillar__ = loader_context.named_context("__pillar__") +__grains__ = loader_context.named_context("__grains__") diff --git a/salt/modules/cp.py b/salt/modules/cp.py index 25b0fcbae911..c9c72a4d026b 100644 --- a/salt/modules/cp.py +++ b/salt/modules/cp.py @@ -20,7 +20,7 @@ import salt.utils.templates import salt.utils.url from salt.exceptions import CommandExecutionError -from salt.loader.dunder import __file_client__ +from salt.loader.dunder import __context__, __file_client__, __grains__, __pillar__ log = logging.getLogger(__name__) From 96c59e07fc9a9ab5f971283361e1d0d13f2b5a72 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 7 Dec 2023 15:43:57 -0700 Subject: [PATCH 38/97] Add tests for opts dunder --- salt/loader/dunder.py | 1 + .../pytests/functional/loader/test_dunder.py | 49 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 tests/pytests/functional/loader/test_dunder.py diff --git a/salt/loader/dunder.py b/salt/loader/dunder.py index 321e6f9a69db..3b198b1497f8 100644 --- a/salt/loader/dunder.py +++ b/salt/loader/dunder.py @@ -8,6 +8,7 @@ __file_client__ = loader_context.named_context("__file_client__", default=None) +__opts__ = loader_context.named_context("__opts__") __context__ = loader_context.named_context("__context__") __pillar__ = loader_context.named_context("__pillar__") __grains__ = loader_context.named_context("__grains__") diff --git a/tests/pytests/functional/loader/test_dunder.py b/tests/pytests/functional/loader/test_dunder.py new file mode 100644 index 000000000000..4cd0d629cdcc --- /dev/null +++ b/tests/pytests/functional/loader/test_dunder.py @@ -0,0 +1,49 @@ +import pathlib + +import salt.loader.context +import salt.loader.lazy +import salt.utils.files +import tests.support.helpers + + +def test_opts_dunder_opts_without_import(tempdir): + """ + Test __opts__ without being imported. + + When a loaded module uses __opts__ but does not import it from + salt.loader.dunder the __opts__ object will be a dictionary. + """ + opts = {"optimization_order": [0, 1, 2]} + with salt.utils.files.fopen(pathlib.Path(tempdir.tempdir) / "mymod.py", "w") as fp: + fp.write( + tests.support.helpers.dedent( + """ + def mymethod(): + return type(__opts__) + """ + ) + ) + loader = salt.loader.lazy.LazyLoader([tempdir.tempdir], opts) + assert loader["mymod.mymethod"]() == dict + + +def test_opts_dunder_opts_with_import(tempdir): + """ + Test __opts__ when imported. + + When a loaded module uses __opts__ by importing it from + salt.loader.dunder the __opts__ object will be a NamedLoaderContext. + """ + opts = {"optimization_order": [0, 1, 2]} + with salt.utils.files.fopen(pathlib.Path(tempdir.tempdir) / "mymod.py", "w") as fp: + fp.write( + tests.support.helpers.dedent( + """ + from salt.loader.dunder import __opts__ + def mymethod(): + return type(__opts__) + """ + ) + ) + loader = salt.loader.lazy.LazyLoader([tempdir.tempdir], opts) + assert loader["mymod.mymethod"]() == salt.loader.context.NamedLoaderContext From fc06de8db1382a1d228e3a5470272e24055c34f6 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 7 Dec 2023 15:48:48 -0700 Subject: [PATCH 39/97] Try importing opts dunder --- salt/modules/cp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/cp.py b/salt/modules/cp.py index c9c72a4d026b..3349a083ecd2 100644 --- a/salt/modules/cp.py +++ b/salt/modules/cp.py @@ -20,7 +20,7 @@ import salt.utils.templates import salt.utils.url from salt.exceptions import CommandExecutionError -from salt.loader.dunder import __context__, __file_client__, __grains__, __pillar__ +from salt.loader.dunder import __context__, __file_client__, __grains__, __pillar__, __opts__ log = logging.getLogger(__name__) From b36f2760078f28c50bb5352372c68b53fcc4de4f Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 7 Dec 2023 18:31:52 -0700 Subject: [PATCH 40/97] Better handling of loader returns - Detect when a loader is returning a NamedLoaderContext and return the value. This will cut down on how often we need to call value() in loaded modules. - Add some tests to show newly defined behavior --- salt/loader/context.py | 14 ++++++++++++-- salt/modules/cp.py | 10 ++++++++-- tests/pytests/functional/loader/test_dunder.py | 13 ++++++++----- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/salt/loader/context.py b/salt/loader/context.py index 88a6472a8f3c..04e3bf094ebc 100644 --- a/salt/loader/context.py +++ b/salt/loader/context.py @@ -68,9 +68,19 @@ def value(self): loader = self.loader() if loader is None: return self.default - if self.name == "__context__": + elif self.name == "__context__": return loader.pack[self.name] - if self.name == loader.pack_self: + elif self.name == "__opts__": + # XXX: This behaviour tires to mimick what the loader does when + # __opts__ was not imported from dunder. It would be nice to just + # pass the value of __opts__ here. However, un-winding this + # behavior will be tricky. + opts = {} + if self.default: + opts.update(copy.deepcopy(self.default)) + opts.update(copy.deepcopy(loader.opts)) + return opts + elif self.name == loader.pack_self: return loader try: return loader.pack[self.name] diff --git a/salt/modules/cp.py b/salt/modules/cp.py index 3349a083ecd2..503b9976dbe1 100644 --- a/salt/modules/cp.py +++ b/salt/modules/cp.py @@ -20,7 +20,13 @@ import salt.utils.templates import salt.utils.url from salt.exceptions import CommandExecutionError -from salt.loader.dunder import __context__, __file_client__, __grains__, __pillar__, __opts__ +from salt.loader.dunder import ( + __context__, + __file_client__, + __grains__, + __opts__, + __pillar__, +) log = logging.getLogger(__name__) @@ -167,7 +173,7 @@ def _client(): """ if __file_client__: return __file_client__.value() - return salt.fileclient.get_file_client(__opts__) + return salt.fileclient.get_file_client(__opts__.value()) def _render_filenames(path, dest, saltenv, template, **kw): diff --git a/tests/pytests/functional/loader/test_dunder.py b/tests/pytests/functional/loader/test_dunder.py index 4cd0d629cdcc..76770784fbc3 100644 --- a/tests/pytests/functional/loader/test_dunder.py +++ b/tests/pytests/functional/loader/test_dunder.py @@ -6,7 +6,7 @@ import tests.support.helpers -def test_opts_dunder_opts_without_import(tempdir): +def xtest_opts_dunder_opts_without_import(tempdir): """ Test __opts__ without being imported. @@ -19,12 +19,12 @@ def test_opts_dunder_opts_without_import(tempdir): tests.support.helpers.dedent( """ def mymethod(): - return type(__opts__) + return __opts__ """ ) ) loader = salt.loader.lazy.LazyLoader([tempdir.tempdir], opts) - assert loader["mymod.mymethod"]() == dict + assert type(loader["mymod.mymethod"]()) == dict def test_opts_dunder_opts_with_import(tempdir): @@ -40,10 +40,13 @@ def test_opts_dunder_opts_with_import(tempdir): tests.support.helpers.dedent( """ from salt.loader.dunder import __opts__ - def mymethod(): + def optstype(): return type(__opts__) + def opts(): + return __opts__ """ ) ) loader = salt.loader.lazy.LazyLoader([tempdir.tempdir], opts) - assert loader["mymod.mymethod"]() == salt.loader.context.NamedLoaderContext + assert loader["mymod.optstype"]() == salt.loader.context.NamedLoaderContext + assert loader["mymod.opts"]() == opts From 96bc89f7217d30998292f7682a002e5fc5806122 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Fri, 8 Dec 2023 14:58:11 -0700 Subject: [PATCH 41/97] Fix broken tests --- salt/loader/context.py | 3 +++ tests/pytests/functional/loader/test_dunder.py | 14 ++++++-------- tests/pytests/unit/states/test_pkg.py | 3 ++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/salt/loader/context.py b/salt/loader/context.py index 04e3bf094ebc..a94e81d00a16 100644 --- a/salt/loader/context.py +++ b/salt/loader/context.py @@ -43,6 +43,9 @@ def __init__(self, name, loader_context, default=None): self.loader_context = loader_context self.default = default + def with_default(self, default): + return NamedLoaderContext(self.name, self.loader_context, default=default) + def loader(self): """ The LazyLoader in the current context. This will return None if there diff --git a/tests/pytests/functional/loader/test_dunder.py b/tests/pytests/functional/loader/test_dunder.py index 76770784fbc3..581d669a12f3 100644 --- a/tests/pytests/functional/loader/test_dunder.py +++ b/tests/pytests/functional/loader/test_dunder.py @@ -1,12 +1,10 @@ -import pathlib - import salt.loader.context import salt.loader.lazy import salt.utils.files import tests.support.helpers -def xtest_opts_dunder_opts_without_import(tempdir): +def test_opts_dunder_opts_without_import(tmp_path): """ Test __opts__ without being imported. @@ -14,7 +12,7 @@ def xtest_opts_dunder_opts_without_import(tempdir): salt.loader.dunder the __opts__ object will be a dictionary. """ opts = {"optimization_order": [0, 1, 2]} - with salt.utils.files.fopen(pathlib.Path(tempdir.tempdir) / "mymod.py", "w") as fp: + with salt.utils.files.fopen(tmp_path / "mymod.py", "w") as fp: fp.write( tests.support.helpers.dedent( """ @@ -23,11 +21,11 @@ def mymethod(): """ ) ) - loader = salt.loader.lazy.LazyLoader([tempdir.tempdir], opts) + loader = salt.loader.lazy.LazyLoader([tmp_path], opts) assert type(loader["mymod.mymethod"]()) == dict -def test_opts_dunder_opts_with_import(tempdir): +def test_opts_dunder_opts_with_import(tmp_path): """ Test __opts__ when imported. @@ -35,7 +33,7 @@ def test_opts_dunder_opts_with_import(tempdir): salt.loader.dunder the __opts__ object will be a NamedLoaderContext. """ opts = {"optimization_order": [0, 1, 2]} - with salt.utils.files.fopen(pathlib.Path(tempdir.tempdir) / "mymod.py", "w") as fp: + with salt.utils.files.fopen(tmp_path / "mymod.py", "w") as fp: fp.write( tests.support.helpers.dedent( """ @@ -47,6 +45,6 @@ def opts(): """ ) ) - loader = salt.loader.lazy.LazyLoader([tempdir.tempdir], opts) + loader = salt.loader.lazy.LazyLoader([tmp_path], opts) assert loader["mymod.optstype"]() == salt.loader.context.NamedLoaderContext assert loader["mymod.opts"]() == opts diff --git a/tests/pytests/unit/states/test_pkg.py b/tests/pytests/unit/states/test_pkg.py index 0255175005af..f9c566524df7 100644 --- a/tests/pytests/unit/states/test_pkg.py +++ b/tests/pytests/unit/states/test_pkg.py @@ -11,6 +11,7 @@ import salt.states.beacon as beaconstate import salt.states.pkg as pkg import salt.utils.state as state_utils +from salt.loader.dunder import __opts__ from salt.utils.event import SaltEvent from tests.support.mock import MagicMock, patch @@ -21,7 +22,7 @@ def configure_loader_modules(minion_opts): return { cp: { - "__opts__": minion_opts, + "__opts__": __opts__.with_default(minion_opts), }, pkg: { "__env__": "base", From 5ee6d3c2bcff85e8a98edc587c4a0aa2fe11a0e8 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 9 Dec 2023 14:54:18 -0700 Subject: [PATCH 42/97] Fix another test using __opts__ --- tests/pytests/functional/modules/test_aptpkg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pytests/functional/modules/test_aptpkg.py b/tests/pytests/functional/modules/test_aptpkg.py index 41ade2889de3..60b134966feb 100644 --- a/tests/pytests/functional/modules/test_aptpkg.py +++ b/tests/pytests/functional/modules/test_aptpkg.py @@ -13,6 +13,7 @@ import salt.modules.gpg as gpg import salt.utils.files import salt.utils.stringutils +from salt.loader.dunder import __opts__ from tests.support.mock import Mock, patch pytestmark = [ @@ -76,7 +77,7 @@ def configure_loader_modules(minion_opts): }, gpg: {}, cp: { - "__opts__": minion_opts, + "__opts__": __opts__.with_default(minion_opts), }, config: { "__opts__": minion_opts, From d4ee87338b37947be1b84b881d45a082c06727c4 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sun, 10 Dec 2023 02:38:53 -0700 Subject: [PATCH 43/97] Fix windows pkg tests --- tests/pytests/unit/modules/test_win_pkg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pytests/unit/modules/test_win_pkg.py b/tests/pytests/unit/modules/test_win_pkg.py index 3ae8f24f8dd6..a976b6d6083b 100644 --- a/tests/pytests/unit/modules/test_win_pkg.py +++ b/tests/pytests/unit/modules/test_win_pkg.py @@ -6,6 +6,7 @@ import pytest +import salt.loader.dunder import salt.modules.config as config import salt.modules.cp as cp import salt.modules.pkg_resource as pkg_resource @@ -57,7 +58,7 @@ def configure_loader_modules(minion_opts): opts = minion_opts opts["master_uri"] = "localhost" return { - cp: {"__opts__": opts}, + cp: {"__opts__": salt.loader.dunder.__opts__.with_default(opts)}, win_pkg: { "_get_latest_package_version": MagicMock(return_value="3.03"), "_get_package_info": MagicMock(return_value=pkg_info), From 0897afcc21b634472e40ad658d64cb139e7072e1 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 12 Sep 2024 15:22:35 -0700 Subject: [PATCH 44/97] Retrun opts because they get modified often --- salt/loader/context.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/salt/loader/context.py b/salt/loader/context.py index a94e81d00a16..1b330f18c851 100644 --- a/salt/loader/context.py +++ b/salt/loader/context.py @@ -5,6 +5,8 @@ import collections.abc import contextlib import copy +import logging +import traceback try: # Try the stdlib C extension first @@ -15,6 +17,8 @@ import salt.exceptions +log = logging.getLogger(__name__) + DEFAULT_CTX_VAR = "loader_ctxvar" loader_ctxvar = contextvars.ContextVar(DEFAULT_CTX_VAR) @@ -71,20 +75,12 @@ def value(self): loader = self.loader() if loader is None: return self.default + if self.name == loader.pack_self: + return loader elif self.name == "__context__": return loader.pack[self.name] elif self.name == "__opts__": - # XXX: This behaviour tires to mimick what the loader does when - # __opts__ was not imported from dunder. It would be nice to just - # pass the value of __opts__ here. However, un-winding this - # behavior will be tricky. - opts = {} - if self.default: - opts.update(copy.deepcopy(self.default)) - opts.update(copy.deepcopy(loader.opts)) - return opts - elif self.name == loader.pack_self: - return loader + return loader.pack[self.name] try: return loader.pack[self.name] except KeyError: From 0dc964fb507cdda2272ed0581dab08aa35330c5e Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Fri, 13 Sep 2024 00:17:58 -0700 Subject: [PATCH 45/97] Clean up imports --- salt/loader/context.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/salt/loader/context.py b/salt/loader/context.py index 1b330f18c851..38d0093a8baf 100644 --- a/salt/loader/context.py +++ b/salt/loader/context.py @@ -5,8 +5,6 @@ import collections.abc import contextlib import copy -import logging -import traceback try: # Try the stdlib C extension first @@ -17,8 +15,6 @@ import salt.exceptions -log = logging.getLogger(__name__) - DEFAULT_CTX_VAR = "loader_ctxvar" loader_ctxvar = contextvars.ContextVar(DEFAULT_CTX_VAR) From 77260acfd512d6620907749aec2a0b1357f1e703 Mon Sep 17 00:00:00 2001 From: ScriptAutomate Date: Fri, 13 Sep 2024 15:17:27 -0500 Subject: [PATCH 46/97] Drop Arch Linux; update latest golden images --- .github/workflows/ci.yml | 23 ----- .github/workflows/nightly.yml | 23 ----- .github/workflows/scheduled.yml | 23 ----- .github/workflows/staging.yml | 22 ----- .gitignore | 1 + changelog/66886.deprecated.md | 1 + cicd/golden-images.json | 122 ++++++++++++--------------- cicd/shared-gh-workflows-context.yml | 1 - tools/ci.py | 1 - tools/precommit/workflows.py | 1 - 10 files changed, 58 insertions(+), 160 deletions(-) create mode 100644 changelog/66886.deprecated.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b6de9cff239..4e88f05b4a24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1527,27 +1527,6 @@ jobs: workflow-slug: ci default-timeout: 180 - archlinux-lts: - name: Arch Linux LTS Test - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - build-ci-deps - uses: ./.github/workflows/test-action-linux.yml - with: - distro-slug: archlinux-lts - nox-session: ci-test-onedir - platform: linux - arch: x86_64 - nox-version: 2022.8.7 - gh-actions-python-version: "3.10" - testrun: ${{ needs.prepare-workflow.outputs.testrun }} - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.14 - skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} - workflow-slug: ci - default-timeout: 180 - debian-11: name: Debian 11 Test if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] && contains(fromJSON(needs.prepare-workflow.outputs.os-labels), 'debian-11') }} @@ -1995,7 +1974,6 @@ jobs: - amazonlinux-2-arm64 - amazonlinux-2023 - amazonlinux-2023-arm64 - - archlinux-lts - debian-11 - debian-11-arm64 - debian-12 @@ -2163,7 +2141,6 @@ jobs: - amazonlinux-2-arm64 - amazonlinux-2023 - amazonlinux-2023-arm64 - - archlinux-lts - debian-11 - debian-11-arm64 - debian-12 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 18e390d8ab7f..d7bb49730019 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1589,27 +1589,6 @@ jobs: workflow-slug: nightly default-timeout: 360 - archlinux-lts: - name: Arch Linux LTS Test - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - build-ci-deps - uses: ./.github/workflows/test-action-linux.yml - with: - distro-slug: archlinux-lts - nox-session: ci-test-onedir - platform: linux - arch: x86_64 - nox-version: 2022.8.7 - gh-actions-python-version: "3.10" - testrun: ${{ needs.prepare-workflow.outputs.testrun }} - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.14 - skip-code-coverage: false - workflow-slug: nightly - default-timeout: 360 - debian-11: name: Debian 11 Test if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} @@ -2057,7 +2036,6 @@ jobs: - amazonlinux-2-arm64 - amazonlinux-2023 - amazonlinux-2023-arm64 - - archlinux-lts - debian-11 - debian-11-arm64 - debian-12 @@ -2964,7 +2942,6 @@ jobs: - amazonlinux-2-arm64 - amazonlinux-2023 - amazonlinux-2023-arm64 - - archlinux-lts - debian-11 - debian-11-arm64 - debian-12 diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 767b8c17f46d..86e57fb71a42 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -1566,27 +1566,6 @@ jobs: workflow-slug: scheduled default-timeout: 360 - archlinux-lts: - name: Arch Linux LTS Test - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - build-ci-deps - uses: ./.github/workflows/test-action-linux.yml - with: - distro-slug: archlinux-lts - nox-session: ci-test-onedir - platform: linux - arch: x86_64 - nox-version: 2022.8.7 - gh-actions-python-version: "3.10" - testrun: ${{ needs.prepare-workflow.outputs.testrun }} - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.14 - skip-code-coverage: false - workflow-slug: scheduled - default-timeout: 360 - debian-11: name: Debian 11 Test if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} @@ -2034,7 +2013,6 @@ jobs: - amazonlinux-2-arm64 - amazonlinux-2023 - amazonlinux-2023-arm64 - - archlinux-lts - debian-11 - debian-11-arm64 - debian-12 @@ -2204,7 +2182,6 @@ jobs: - amazonlinux-2-arm64 - amazonlinux-2023 - amazonlinux-2023-arm64 - - archlinux-lts - debian-11 - debian-11-arm64 - debian-12 diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 2cf7786fe2c8..1c2546357e01 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -1574,27 +1574,6 @@ jobs: workflow-slug: staging default-timeout: 180 - archlinux-lts: - name: Arch Linux LTS Test - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - build-ci-deps - uses: ./.github/workflows/test-action-linux.yml - with: - distro-slug: archlinux-lts - nox-session: ci-test-onedir - platform: linux - arch: x86_64 - nox-version: 2022.8.7 - gh-actions-python-version: "3.10" - testrun: ${{ needs.prepare-workflow.outputs.testrun }} - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.14 - skip-code-coverage: true - workflow-slug: staging - default-timeout: 180 - debian-11: name: Debian 11 Test if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} @@ -2917,7 +2896,6 @@ jobs: - amazonlinux-2-arm64 - amazonlinux-2023 - amazonlinux-2023-arm64 - - archlinux-lts - debian-11 - debian-11-arm64 - debian-12 diff --git a/.gitignore b/.gitignore index 63b49a644872..73cc51c3d1ce 100644 --- a/.gitignore +++ b/.gitignore @@ -89,6 +89,7 @@ tests/unit/templates/roots # Pycharm .idea venv/ +.venv/ # VS Code .vscode diff --git a/changelog/66886.deprecated.md b/changelog/66886.deprecated.md new file mode 100644 index 000000000000..597c0aee10a4 --- /dev/null +++ b/changelog/66886.deprecated.md @@ -0,0 +1 @@ +Drop Arch Linux support diff --git a/cicd/golden-images.json b/cicd/golden-images.json index ca7818fdd6ba..b0504ad777a7 100644 --- a/cicd/golden-images.json +++ b/cicd/golden-images.json @@ -1,8 +1,8 @@ { "amazonlinux-2-arm64": { - "ami": "ami-0c98c023fba59d522", + "ami": "ami-0aab00f54b6cddde6", "ami_description": "CI Image of AmazonLinux 2 arm64", - "ami_name": "salt-project/ci/amazonlinux/2/arm64/20240509.1530", + "ami_name": "salt-project/ci/amazonlinux/2/arm64/20240912.2135", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -10,9 +10,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2": { - "ami": "ami-02cba95cfd7074794", + "ami": "ami-0fd6cec7bbcf52d36", "ami_description": "CI Image of AmazonLinux 2 x86_64", - "ami_name": "salt-project/ci/amazonlinux/2/x86_64/20240509.1530", + "ami_name": "salt-project/ci/amazonlinux/2/x86_64/20240912.2135", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -20,9 +20,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2023-arm64": { - "ami": "ami-0609f0e98f5a6b73d", + "ami": "ami-095e9e4757b5fca1a", "ami_description": "CI Image of AmazonLinux 2023 arm64", - "ami_name": "salt-project/ci/amazonlinux/2023/arm64/20240509.1529", + "ami_name": "salt-project/ci/amazonlinux/2023/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -30,29 +30,19 @@ "ssh_username": "ec2-user" }, "amazonlinux-2023": { - "ami": "ami-0554a801eb6dcc42c", + "ami": "ami-002d043f1a36bf06e", "ami_description": "CI Image of AmazonLinux 2023 x86_64", - "ami_name": "salt-project/ci/amazonlinux/2023/x86_64/20240509.1529", + "ami_name": "salt-project/ci/amazonlinux/2023/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", "is_windows": "false", "ssh_username": "ec2-user" }, - "archlinux-lts": { - "ami": "ami-01ad78f19930b9747", - "ami_description": "CI Image of ArchLinux lts x86_64", - "ami_name": "salt-project/ci/archlinux/lts/x86_64/20240509.1530", - "arch": "x86_64", - "cloudwatch-agent-available": "false", - "instance_type": "t3a.large", - "is_windows": "false", - "ssh_username": "arch" - }, "debian-11-arm64": { - "ami": "ami-0eff227d9a94d8692", + "ami": "ami-0ff63235fce7bea1d", "ami_description": "CI Image of Debian 11 arm64", - "ami_name": "salt-project/ci/debian/11/arm64/20240509.1529", + "ami_name": "salt-project/ci/debian/11/arm64/20240912.2135", "arch": "arm64", "cloudwatch-agent-available": "false", "instance_type": "m6g.large", @@ -60,9 +50,9 @@ "ssh_username": "admin" }, "debian-11": { - "ami": "ami-099b2a5a1fb995166", + "ami": "ami-08685bfca48beeb67", "ami_description": "CI Image of Debian 11 x86_64", - "ami_name": "salt-project/ci/debian/11/x86_64/20240509.1529", + "ami_name": "salt-project/ci/debian/11/x86_64/20240912.2135", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -70,9 +60,9 @@ "ssh_username": "admin" }, "debian-12-arm64": { - "ami": "ami-0ab6b0cc8488f8880", + "ami": "ami-07d383138f04b32ba", "ami_description": "CI Image of Debian 12 arm64", - "ami_name": "salt-project/ci/debian/12/arm64/20240509.1529", + "ami_name": "salt-project/ci/debian/12/arm64/20240912.2135", "arch": "arm64", "cloudwatch-agent-available": "false", "instance_type": "m6g.large", @@ -80,9 +70,9 @@ "ssh_username": "admin" }, "debian-12": { - "ami": "ami-0e1f5b55325249c4e", + "ami": "ami-0867ec74072fd97a0", "ami_description": "CI Image of Debian 12 x86_64", - "ami_name": "salt-project/ci/debian/12/x86_64/20240509.1530", + "ami_name": "salt-project/ci/debian/12/x86_64/20240912.2135", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -90,9 +80,9 @@ "ssh_username": "admin" }, "fedora-40-arm64": { - "ami": "ami-064df327a55f83953", + "ami": "ami-03be8e03c17f1abeb", "ami_description": "CI Image of Fedora 40 arm64", - "ami_name": "salt-project/ci/fedora/40/arm64/20240509.1530", + "ami_name": "salt-project/ci/fedora/40/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -100,9 +90,9 @@ "ssh_username": "fedora" }, "fedora-40": { - "ami": "ami-08d8dbd4f063788de", + "ami": "ami-060a59b30809758b2", "ami_description": "CI Image of Fedora 40 x86_64", - "ami_name": "salt-project/ci/fedora/40/x86_64/20240509.1530", + "ami_name": "salt-project/ci/fedora/40/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -110,9 +100,9 @@ "ssh_username": "fedora" }, "opensuse-15": { - "ami": "ami-0f82d5ab3015af6ad", + "ami": "ami-0aaf63315ada5365b", "ami_description": "CI Image of Opensuse 15 x86_64", - "ami_name": "salt-project/ci/opensuse/15/x86_64/20240509.1529", + "ami_name": "salt-project/ci/opensuse/15/x86_64/20240912.2135", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -120,9 +110,9 @@ "ssh_username": "ec2-user" }, "photonos-4-arm64": { - "ami": "ami-0ea152c346cb8e13b", + "ami": "ami-0d425acec9d0d78a5", "ami_description": "CI Image of PhotonOS 4 arm64", - "ami_name": "salt-project/ci/photonos/4/arm64/20240509.1530", + "ami_name": "salt-project/ci/photonos/4/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -130,9 +120,9 @@ "ssh_username": "root" }, "photonos-4": { - "ami": "ami-09b55d0bf3a1aa7e5", + "ami": "ami-056d988807f8b586d", "ami_description": "CI Image of PhotonOS 4 x86_64", - "ami_name": "salt-project/ci/photonos/4/x86_64/20240509.1530", + "ami_name": "salt-project/ci/photonos/4/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -140,9 +130,9 @@ "ssh_username": "root" }, "photonos-5-arm64": { - "ami": "ami-09de4952bc9fc068a", + "ami": "ami-059f47b459d04544a", "ami_description": "CI Image of PhotonOS 5 arm64", - "ami_name": "salt-project/ci/photonos/5/arm64/20240509.1530", + "ami_name": "salt-project/ci/photonos/5/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -150,9 +140,9 @@ "ssh_username": "root" }, "photonos-5": { - "ami": "ami-0c3375a583643fc77", + "ami": "ami-06424daf7c85ffff0", "ami_description": "CI Image of PhotonOS 5 x86_64", - "ami_name": "salt-project/ci/photonos/5/x86_64/20240509.1530", + "ami_name": "salt-project/ci/photonos/5/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -160,9 +150,9 @@ "ssh_username": "root" }, "rockylinux-8-arm64": { - "ami": "ami-0662cc201cada14b8", + "ami": "ami-0a21b175629f1a793", "ami_description": "CI Image of RockyLinux 8 arm64", - "ami_name": "salt-project/ci/rockylinux/8/arm64/20240509.1530", + "ami_name": "salt-project/ci/rockylinux/8/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -170,9 +160,9 @@ "ssh_username": "rocky" }, "rockylinux-8": { - "ami": "ami-071ca70a907d79e05", + "ami": "ami-01032695e18f0fe85", "ami_description": "CI Image of RockyLinux 8 x86_64", - "ami_name": "salt-project/ci/rockylinux/8/x86_64/20240509.1530", + "ami_name": "salt-project/ci/rockylinux/8/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -180,9 +170,9 @@ "ssh_username": "rocky" }, "rockylinux-9-arm64": { - "ami": "ami-065842dfdf03a1a03", + "ami": "ami-0c9147ca5f07effc6", "ami_description": "CI Image of RockyLinux 9 arm64", - "ami_name": "salt-project/ci/rockylinux/9/arm64/20240509.1530", + "ami_name": "salt-project/ci/rockylinux/9/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -190,9 +180,9 @@ "ssh_username": "rocky" }, "rockylinux-9": { - "ami": "ami-09f5d6df00e99ba16", + "ami": "ami-01a72f34d198efc4a", "ami_description": "CI Image of RockyLinux 9 x86_64", - "ami_name": "salt-project/ci/rockylinux/9/x86_64/20240509.1530", + "ami_name": "salt-project/ci/rockylinux/9/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -200,9 +190,9 @@ "ssh_username": "rocky" }, "ubuntu-20.04-arm64": { - "ami": "ami-00171fa604b826054", + "ami": "ami-0bf8ea4c07a88d6c5", "ami_description": "CI Image of Ubuntu 20.04 arm64", - "ami_name": "salt-project/ci/ubuntu/20.04/arm64/20240509.1530", + "ami_name": "salt-project/ci/ubuntu/20.04/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -210,9 +200,9 @@ "ssh_username": "ubuntu" }, "ubuntu-20.04": { - "ami": "ami-07ddfbdc489064022", + "ami": "ami-08a84f7455622c3d5", "ami_description": "CI Image of Ubuntu 20.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/20.04/x86_64/20240509.1530", + "ami_name": "salt-project/ci/ubuntu/20.04/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -220,9 +210,9 @@ "ssh_username": "ubuntu" }, "ubuntu-22.04-arm64": { - "ami": "ami-0e6b6fc1dd298e055", + "ami": "ami-0415a2d2279277d61", "ami_description": "CI Image of Ubuntu 22.04 arm64", - "ami_name": "salt-project/ci/ubuntu/22.04/arm64/20240509.1530", + "ami_name": "salt-project/ci/ubuntu/22.04/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -230,9 +220,9 @@ "ssh_username": "ubuntu" }, "ubuntu-22.04": { - "ami": "ami-0736289579c0d01ba", + "ami": "ami-055513129ce06397c", "ami_description": "CI Image of Ubuntu 22.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/22.04/x86_64/20240509.1530", + "ami_name": "salt-project/ci/ubuntu/22.04/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -240,9 +230,9 @@ "ssh_username": "ubuntu" }, "ubuntu-24.04-arm64": { - "ami": "ami-015058823f69446b3", + "ami": "ami-035ef6d54ec25b0fa", "ami_description": "CI Image of Ubuntu 24.04 arm64", - "ami_name": "salt-project/ci/ubuntu/24.04/arm64/20240509.1530", + "ami_name": "salt-project/ci/ubuntu/24.04/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -250,9 +240,9 @@ "ssh_username": "ubuntu" }, "ubuntu-24.04": { - "ami": "ami-0eb04152e7cafaaf9", + "ami": "ami-0a287b781a487ec65", "ami_description": "CI Image of Ubuntu 24.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/24.04/x86_64/20240509.1530", + "ami_name": "salt-project/ci/ubuntu/24.04/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -260,9 +250,9 @@ "ssh_username": "ubuntu" }, "windows-2016": { - "ami": "ami-06026cb4d83072df5", + "ami": "ami-030cdb60764141f56", "ami_description": "CI Image of Windows 2016 x86_64", - "ami_name": "salt-project/ci/windows/2016/x86_64/20240509.1530", + "ami_name": "salt-project/ci/windows/2016/x86_64/20240913.1756", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", @@ -270,9 +260,9 @@ "ssh_username": "Administrator" }, "windows-2019": { - "ami": "ami-095a9256ec0e8261c", + "ami": "ami-08f10b0d4914572de", "ami_description": "CI Image of Windows 2019 x86_64", - "ami_name": "salt-project/ci/windows/2019/x86_64/20240509.1530", + "ami_name": "salt-project/ci/windows/2019/x86_64/20240913.1756", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", @@ -280,9 +270,9 @@ "ssh_username": "Administrator" }, "windows-2022": { - "ami": "ami-0d295c0711e513c05", + "ami": "ami-07eda52ffbd76a4c6", "ami_description": "CI Image of Windows 2022 x86_64", - "ami_name": "salt-project/ci/windows/2022/x86_64/20240509.1530", + "ami_name": "salt-project/ci/windows/2022/x86_64/20240913.1756", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", diff --git a/cicd/shared-gh-workflows-context.yml b/cicd/shared-gh-workflows-context.yml index 9cd0641c739b..038071329d9e 100644 --- a/cicd/shared-gh-workflows-context.yml +++ b/cicd/shared-gh-workflows-context.yml @@ -4,7 +4,6 @@ relenv_version: "0.17.0" mandatory_os_slugs: - rockylinux-9 - amazonlinux-2023-arm64 - - archlinux-lts - photonos-5-arm64 - macos-12 - ubuntu-24.04-arm64 diff --git a/tools/ci.py b/tools/ci.py index 391214e69a38..0791fabe90bb 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -1010,7 +1010,6 @@ def get_pkg_downloads_matrix(ctx: Context): "photon", ) linux_skip_pkg_download_tests = ( - "archlinux-lts", "opensuse-15", "windows", ) diff --git a/tools/precommit/workflows.py b/tools/precommit/workflows.py index aa844b904ccc..b60371aa80cb 100644 --- a/tools/precommit/workflows.py +++ b/tools/precommit/workflows.py @@ -51,7 +51,6 @@ display_name="Amazon Linux 2023 Arm64", arch="arm64", ), - Linux(slug="archlinux-lts", display_name="Arch Linux LTS", arch="x86_64"), Linux(slug="debian-11", display_name="Debian 11", arch="x86_64"), Linux(slug="debian-11-arm64", display_name="Debian 11 Arm64", arch="arm64"), Linux(slug="debian-12", display_name="Debian 12", arch="x86_64"), From c54a8ced4fd4eeab482af3e76267659418770353 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 16 Sep 2024 13:00:52 -0600 Subject: [PATCH 47/97] Test on systems where 7zip exists --- tests/pytests/functional/states/test_pkg.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/pytests/functional/states/test_pkg.py b/tests/pytests/functional/states/test_pkg.py index a56f06667e44..230491d77c60 100644 --- a/tests/pytests/functional/states/test_pkg.py +++ b/tests/pytests/functional/states/test_pkg.py @@ -58,7 +58,7 @@ def refresh_keys(grains, modules): def PKG_TARGETS(grains): _PKG_TARGETS = ["figlet", "sl"] if grains["os"] == "Windows": - _PKG_TARGETS = ["7zip", "putty"] + _PKG_TARGETS = ["npp_x64", "winrar"] elif grains["os"] == "Amazon": if grains["osfinger"] == "Amazon Linux-2023": _PKG_TARGETS = ["lynx", "gnuplot-minimal"] @@ -118,7 +118,7 @@ def PKG_32_TARGETS(grains): else: _PKG_32_TARGETS.append("xz-devel.i686") elif grains["os"] == "Windows": - _PKG_32_TARGETS = ["npp", "nsis"] + _PKG_32_TARGETS = ["npp", "putty"] if not _PKG_32_TARGETS: pytest.skip("No 32 bit packages have been specified for testing") return _PKG_32_TARGETS @@ -217,12 +217,16 @@ def install_7zip(modules): try: modules.pkg.install(name="7zip", version="22.01.00.0") modules.pkg.install(name="7zip", version="19.00.00.0") - assert modules.pkg.version("7zip") == "19.00.00.0,22.01.00.0" + versions = modules.pkg.version("7zip") + assert "19.00.00.0" in versions + assert "22.01.00.0" in versions yield finally: modules.pkg.remove(name="7zip", version="19.00.00.0") modules.pkg.remove(name="7zip", version="22.01.00.0") - assert modules.pkg.version("7zip") == "" + versions = modules.pkg.version("7zip") + assert "19.00.00.0" not in versions + assert "22.01.00.0" not in versions @pytest.mark.requires_salt_modules("pkg.version") @@ -1125,4 +1129,4 @@ def test_pkg_removed_with_version_multiple(install_7zip, modules, states): ret = states.pkg.removed(name="7zip", version="19.00.00.0") assert ret.result is True current = modules.pkg.version("7zip") - assert current == "22.01.00.0" + assert "22.01.00.0" in current From 26e31d23d525e3271d54d5f2072ee0324c2f1559 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 28 Aug 2024 12:23:41 -0600 Subject: [PATCH 48/97] Manage symlinks after dirs and files Force symlink on keep_symlinks --- salt/states/file.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 4adf6e12529b..04c490e97109 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -4668,12 +4668,16 @@ def manage_directory(path): name, source, keep_symlinks, include_pat, exclude_pat, maxdepth, include_empty ) + for dirname in mng_dirs: + manage_directory(dirname) + for dest, src in mng_files: + manage_file(dest, src, replace) for srelpath, ltarget in mng_symlinks: _ret = symlink( os.path.join(name, srelpath), ltarget, makedirs=True, - force=force_symlinks, + force=force_symlinks or keep_symlinks, user=user, group=group, mode=sym_mode, @@ -4681,10 +4685,6 @@ def manage_directory(path): if not _ret: continue merge_ret(os.path.join(name, srelpath), _ret) - for dirname in mng_dirs: - manage_directory(dirname) - for dest, src in mng_files: - manage_file(dest, src, replace) if clean: # TODO: Use directory(clean=True) instead From 572e955388ec8b3550786f1c2b11fa07adb3b824 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 30 Aug 2024 11:55:46 -0600 Subject: [PATCH 49/97] Add tests and changelog --- changelog/64630.fixed.md | 3 + .../functional/states/file/test_recurse.py | 89 +++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 changelog/64630.fixed.md diff --git a/changelog/64630.fixed.md b/changelog/64630.fixed.md new file mode 100644 index 000000000000..f49c58d4c2e0 --- /dev/null +++ b/changelog/64630.fixed.md @@ -0,0 +1,3 @@ +Fixed an intermittent issue with file.recurse where the state would +report failure even on success. Makes sure symlinks are created +after the target file is created diff --git a/tests/pytests/functional/states/file/test_recurse.py b/tests/pytests/functional/states/file/test_recurse.py index c735d5128dac..53583b26b2d4 100644 --- a/tests/pytests/functional/states/file/test_recurse.py +++ b/tests/pytests/functional/states/file/test_recurse.py @@ -7,6 +7,19 @@ ] +@pytest.fixture(scope="module") +def symlink(state_tree): + # Create directory structure + source_dir = state_tree / "test_symlink" + if not source_dir.is_dir(): + source_dir.mkdir() + source_file = source_dir / "source_file.txt" + source_file.write_text("This is the source file...") + symlink_file = source_dir / "symlink" + symlink_file.symlink_to(source_file) + yield + + @pytest.mark.parametrize("test", (False, True)) def test_recurse(file, tmp_path, grail, test): """ @@ -249,3 +262,79 @@ def test_issue_2726_mode_kwarg(modules, tmp_path, state_tree): ret = modules.state.template_str("\n".join(good_template)) for state_run in ret: assert state_run.result is True + + +def test_issue_64630_keep_symlinks_true(file, symlink, tmp_path): + """ + Make sure that symlinks are created and that there isn't an error + """ + target_dir = tmp_path / "test_symlink" # Target for the file.recurse state + target_file = target_dir / "source_file.txt" + target_symlink = target_dir / "symlink" + + ret = file.recurse( + name=str(target_dir), source=f"salt://{target_dir.name}", keep_symlinks=True + ) + assert ret.result is True + + assert target_dir.exists() + assert target_file.is_file() + assert target_symlink.is_symlink() + + +def test_issue_64630_keep_symlinks_false(file, symlink, tmp_path): + """ + Make sure that symlinks are created and that there isn't an error + """ + target_dir = tmp_path / "test_symlink" # Target for the file.recurse state + target_file = target_dir / "source_file.txt" + target_symlink = target_dir / "symlink" + + ret = file.recurse( + name=str(target_dir), source=f"salt://{target_dir.name}", keep_symlinks=False + ) + assert ret.result is True + + assert target_dir.exists() + assert target_file.is_file() + assert target_symlink.is_file() + assert target_file.read_text() == target_symlink.read_text() + + +def test_issue_64630_force_symlinks_true(file, symlink, tmp_path): + """ + Make sure that symlinks are created and that there isn't an error + """ + target_dir = tmp_path / "test_symlink" # Target for the file.recurse state + target_file = target_dir / "source_file.txt" + target_symlink = target_dir / "symlink" + + ret = file.recurse( + name=str(target_dir), source=f"salt://{target_dir.name}", force_symlinks=True + ) + assert ret.result is True + + assert target_dir.exists() + assert target_file.is_file() + assert target_symlink.is_file() + + +def test_issue_64630_force_symlinks_keep_symlinks_true(file, symlink, tmp_path): + """ + Make sure that symlinks are created and that there isn't an error + """ + target_dir = tmp_path / "test_symlink" # Target for the file.recurse state + target_file = target_dir / "source_file.txt" + target_symlink = target_dir / "symlink" + + ret = file.recurse( + name=str(target_dir), + source=f"salt://{target_dir.name}", + force_symlinks=True, + keep_symlinks=True, + ) + assert ret.result is True + + assert target_dir.exists() + assert target_file.is_file() + assert target_symlink.is_symlink() From bbb0013138bc8d6c5769f6211da5dd9773a3cf82 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 6 Sep 2024 13:54:05 -0600 Subject: [PATCH 50/97] Make sure symlinks are done last --- salt/fileserver/roots.py | 2 +- salt/states/file.py | 23 +++++-- .../functional/states/file/test_recurse.py | 65 +++++++++++++++---- 3 files changed, 72 insertions(+), 18 deletions(-) diff --git a/salt/fileserver/roots.py b/salt/fileserver/roots.py index e81f37dcf029..cb27396b9790 100644 --- a/salt/fileserver/roots.py +++ b/salt/fileserver/roots.py @@ -325,7 +325,7 @@ def file_hash(load, fnd): def _file_lists(load, form): """ - Return a dict containing the file lists for files, dirs, emtydirs and symlinks + Return a dict containing the file lists for files, dirs, empty dirs and symlinks """ if "env" in load: # "env" is not supported; Use "saltenv". diff --git a/salt/states/file.py b/salt/states/file.py index 04c490e97109..2e943f0797c0 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -557,7 +557,18 @@ def process_symlinks(filenames, symlinks): managed_directories.add(mdest) keep.add(mdest) - return managed_files, managed_directories, managed_symlinks, keep + # Sets are randomly ordered. We need to make sure symlinks are always at the + # end, so we need to use a list + new_managed_files = list(managed_files) + # Now let's move all the symlinks to the end + for symlink in managed_symlinks: + for file in managed_files: + if file[0].endswith(symlink[0]): + new_managed_files.append( + new_managed_files.pop(new_managed_files.index(file)) + ) + + return new_managed_files, managed_directories, managed_symlinks, keep def _gen_keep_files(name, require, walk_d=None): @@ -4668,16 +4679,12 @@ def manage_directory(path): name, source, keep_symlinks, include_pat, exclude_pat, maxdepth, include_empty ) - for dirname in mng_dirs: - manage_directory(dirname) - for dest, src in mng_files: - manage_file(dest, src, replace) for srelpath, ltarget in mng_symlinks: _ret = symlink( os.path.join(name, srelpath), ltarget, makedirs=True, - force=force_symlinks or keep_symlinks, + force=force_symlinks, user=user, group=group, mode=sym_mode, @@ -4685,6 +4692,10 @@ def manage_directory(path): if not _ret: continue merge_ret(os.path.join(name, srelpath), _ret) + for dirname in mng_dirs: + manage_directory(dirname) + for dest, src in mng_files: + manage_file(dest, src, replace) if clean: # TODO: Use directory(clean=True) instead diff --git a/tests/pytests/functional/states/file/test_recurse.py b/tests/pytests/functional/states/file/test_recurse.py index 53583b26b2d4..04370c046afa 100644 --- a/tests/pytests/functional/states/file/test_recurse.py +++ b/tests/pytests/functional/states/file/test_recurse.py @@ -8,16 +8,35 @@ @pytest.fixture(scope="module") -def symlink(state_tree): +def symlink_scenario_1(state_tree): # Create directory structure - source_dir = state_tree / "test_symlink" + dir_name = "symlink_scenario_1" + source_dir = state_tree / dir_name if not source_dir.is_dir(): source_dir.mkdir() source_file = source_dir / "source_file.txt" source_file.write_text("This is the source file...") symlink_file = source_dir / "symlink" symlink_file.symlink_to(source_file) - yield + yield dir_name + + +@pytest.fixture(scope="module") +def symlink_scenario_2(state_tree): + # Create directory structure + dir_name = "symlink_scenario_2" + source_dir = state_tree / dir_name / "test" + if not source_dir.is_dir(): + source_dir.mkdir(parents=True) + test1 = source_dir / "test1" + test2 = source_dir / "test2" + test3 = source_dir / "test3" + test_link = source_dir / "test" + test1.touch() + test2.touch() + test3.touch() + test_link.symlink_to(test3) + yield dir_name @pytest.mark.parametrize("test", (False, True)) @@ -264,11 +283,11 @@ def test_issue_2726_mode_kwarg(modules, tmp_path, state_tree): assert state_run.result is True -def test_issue_64630_keep_symlinks_true(file, symlink, tmp_path): +def test_issue_64630_keep_symlinks_true(file, symlink_scenario_1, tmp_path): """ Make sure that symlinks are created and that there isn't an error """ - target_dir = tmp_path / "test_symlink" # Target for the file.recurse state + target_dir = tmp_path / symlink_scenario_1 # Target for the file.recurse state target_file = target_dir / "source_file.txt" target_symlink = target_dir / "symlink" @@ -282,11 +301,11 @@ def test_issue_64630_keep_symlinks_true(file, symlink, tmp_path): assert target_symlink.is_symlink() -def test_issue_64630_keep_symlinks_false(file, symlink, tmp_path): +def test_issue_64630_keep_symlinks_false(file, symlink_scenario_1, tmp_path): """ Make sure that symlinks are created and that there isn't an error """ - target_dir = tmp_path / "test_symlink" # Target for the file.recurse state + target_dir = tmp_path / symlink_scenario_1 # Target for the file.recurse state target_file = target_dir / "source_file.txt" target_symlink = target_dir / "symlink" @@ -301,11 +320,11 @@ def test_issue_64630_keep_symlinks_false(file, symlink, tmp_path): assert target_file.read_text() == target_symlink.read_text() -def test_issue_64630_force_symlinks_true(file, symlink, tmp_path): +def test_issue_64630_force_symlinks_true(file, symlink_scenario_1, tmp_path): """ Make sure that symlinks are created and that there isn't an error """ - target_dir = tmp_path / "test_symlink" # Target for the file.recurse state + target_dir = tmp_path / symlink_scenario_1 # Target for the file.recurse state target_file = target_dir / "source_file.txt" target_symlink = target_dir / "symlink" @@ -319,11 +338,13 @@ def test_issue_64630_force_symlinks_true(file, symlink, tmp_path): assert target_symlink.is_file() -def test_issue_64630_force_symlinks_keep_symlinks_true(file, symlink, tmp_path): +def test_issue_64630_force_symlinks_keep_symlinks_true( + file, symlink_scenario_1, tmp_path +): """ Make sure that symlinks are created and that there isn't an error """ - target_dir = tmp_path / "test_symlink" # Target for the file.recurse state + target_dir = tmp_path / symlink_scenario_1 # Target for the file.recurse state target_file = target_dir / "source_file.txt" target_symlink = target_dir / "symlink" @@ -338,3 +359,25 @@ def test_issue_64630_force_symlinks_keep_symlinks_true(file, symlink, tmp_path): assert target_dir.exists() assert target_file.is_file() assert target_symlink.is_symlink() + + +def test_issue_62117(file, symlink_scenario_2, tmp_path): + target_dir = tmp_path / symlink_scenario_2 / "test" + target_file_1 = target_dir / "test1" + target_file_2 = target_dir / "test2" + target_file_3 = target_dir / "test3" + target_symlink = target_dir / "test" + + ret = file.recurse( + name=str(target_dir), + source=f"salt://{target_dir.parent.name}/test", + clean=True, + keep_symlinks=True, + ) + assert ret.result is True + + assert target_dir.exists() + assert target_file_1.is_file() + assert target_file_2.is_file() + assert target_file_3.is_file() + assert target_symlink.is_symlink() From 294a81a05c1c941a0bde2b376df14e6285f85556 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 11 Sep 2024 14:58:41 -0600 Subject: [PATCH 51/97] Try to handle similar names --- salt/states/file.py | 30 ++++--- .../functional/states/file/test_recurse.py | 83 +++++++++++++++++-- 2 files changed, 94 insertions(+), 19 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 2e943f0797c0..717dafb47200 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -563,7 +563,7 @@ def process_symlinks(filenames, symlinks): # Now let's move all the symlinks to the end for symlink in managed_symlinks: for file in managed_files: - if file[0].endswith(symlink[0]): + if file[0].endswith(os.sep + symlink[0]): new_managed_files.append( new_managed_files.pop(new_managed_files.index(file)) ) @@ -4443,18 +4443,26 @@ def recurse( or immediate subdirectories keep_symlinks - Keep symlinks when copying from the source. This option will cause - the copy operation to terminate at the symlink. If desire behavior - similar to rsync, then set this to True. This option is not taken - in account if ``fileserver_followsymlinks`` is set to False. + + Determines how symbolic links (symlinks) are handled during the copying + process. When set to ``True``, the copy operation will copy the symlink + itself, rather than the file or directory it points to. When set to + ``False``, the operation will follow the symlink and copy the target + file or directory. If you want behavior similar to rsync, set this + option to ``True``. + + However, if the ``fileserver_followsymlinks`` option is set to ``False``, + the ``keep_symlinks`` setting will be ignored, and symlinks will not be + copied at all. force_symlinks - Force symlink creation. This option will force the symlink creation. - If a file or directory is obstructing symlink creation it will be - recursively removed so that symlink creation can proceed. This - option is usually not needed except in special circumstances. This - option is not taken in account if ``fileserver_followsymlinks`` is - set to False. + + Controls the creation of symlinks when using ``keep_symlinks``. When set + to ``True``, it forces the creation of symlinks by removing any existing + files or directories that might be obstructing their creation. This + removal is done recursively if a directory is blocking the symlink. This + option is only used when ``keep_symlinks`` is passed and is ignored if + ``fileserver_followsymlinks`` is set to ``False``. win_owner The owner of the symlink and directories if ``makedirs`` is True. If diff --git a/tests/pytests/functional/states/file/test_recurse.py b/tests/pytests/functional/states/file/test_recurse.py index 04370c046afa..9b69bbf5fffd 100644 --- a/tests/pytests/functional/states/file/test_recurse.py +++ b/tests/pytests/functional/states/file/test_recurse.py @@ -39,6 +39,28 @@ def symlink_scenario_2(state_tree): yield dir_name +@pytest.fixture(scope="module") +def symlink_scenario_3(state_tree): + # Create directory structure + dir_name = "symlink_scenario_3" + source_dir = state_tree / dir_name + if not source_dir.is_dir(): + source_dir.mkdir(parents=True) + # Create a file with the same name but is not a symlink + source_file = source_dir / "not_a_symlink" / "symlink" + source_file.parent.mkdir(parents=True) + source_file.write_text("This is the source file...") + # Create other fluff files + just_a_file = source_dir / "just_a_file.txt" + just_a_file.touch() + dummy_file = source_dir / "notasymlink" + dummy_file.touch() + # Create symlink to source with the same name + symlink_file = source_dir / "symlink" + symlink_file.symlink_to(source_file) + yield dir_name + + @pytest.mark.parametrize("test", (False, True)) def test_recurse(file, tmp_path, grail, test): """ @@ -285,7 +307,8 @@ def test_issue_2726_mode_kwarg(modules, tmp_path, state_tree): def test_issue_64630_keep_symlinks_true(file, symlink_scenario_1, tmp_path): """ - Make sure that symlinks are created and that there isn't an error + Make sure that symlinks are created and that there isn't an error when there + are no conflicting target files """ target_dir = tmp_path / symlink_scenario_1 # Target for the file.recurse state target_file = target_dir / "source_file.txt" @@ -303,7 +326,7 @@ def test_issue_64630_keep_symlinks_true(file, symlink_scenario_1, tmp_path): def test_issue_64630_keep_symlinks_false(file, symlink_scenario_1, tmp_path): """ - Make sure that symlinks are created and that there isn't an error + Make sure that symlinks are created as files and that there isn't an error """ target_dir = tmp_path / symlink_scenario_1 # Target for the file.recurse state target_file = target_dir / "source_file.txt" @@ -320,34 +343,53 @@ def test_issue_64630_keep_symlinks_false(file, symlink_scenario_1, tmp_path): assert target_file.read_text() == target_symlink.read_text() -def test_issue_64630_force_symlinks_true(file, symlink_scenario_1, tmp_path): +def test_issue_64630_keep_symlinks_conflicting_force_symlinks_false( + file, symlink_scenario_1, tmp_path +): """ - Make sure that symlinks are created and that there isn't an error + Make sure that symlinks are not created when there is a conflict. The state + should return False """ target_dir = tmp_path / symlink_scenario_1 # Target for the file.recurse state target_file = target_dir / "source_file.txt" target_symlink = target_dir / "symlink" + # Create the conflicting file + target_symlink.parent.mkdir(parents=True) + target_symlink.touch() + assert target_symlink.is_file() + ret = file.recurse( - name=str(target_dir), source=f"salt://{target_dir.name}", force_symlinks=True + name=str(target_dir), + source=f"salt://{target_dir.name}", + keep_symlinks=True, + force_symlinks=False, ) - assert ret.result is True + # We expect it to fail + assert ret.result is False + # And files not to be created properly assert target_dir.exists() assert target_file.is_file() assert target_symlink.is_file() -def test_issue_64630_force_symlinks_keep_symlinks_true( +def test_issue_64630_keep_symlinks_conflicting_force_symlinks_true( file, symlink_scenario_1, tmp_path ): """ - Make sure that symlinks are created and that there isn't an error + Make sure that symlinks are created when there is a conflict with an + existing file. """ target_dir = tmp_path / symlink_scenario_1 # Target for the file.recurse state target_file = target_dir / "source_file.txt" target_symlink = target_dir / "symlink" + # Create the conflicting file + target_symlink.parent.mkdir(parents=True) + target_symlink.touch() + assert target_symlink.is_file() + ret = file.recurse( name=str(target_dir), source=f"salt://{target_dir.name}", @@ -361,6 +403,31 @@ def test_issue_64630_force_symlinks_keep_symlinks_true( assert target_symlink.is_symlink() +def test_issue_64630_keep_symlinks_similar_names(file, symlink_scenario_3, tmp_path): + """ + Make sure that symlinks are created when there is a file that shares part + of the name of the actual symlink file. I'm not sure what I'm testing here + as I couldn't really get this to fail either way + """ + target_dir = tmp_path / symlink_scenario_3 # Target for the file.recurse state + # symlink target, but has the same name as the symlink itself + target_source = target_dir / "not_a_symlink" / "symlink" + target_symlink = target_dir / "symlink" + decoy_file = target_dir / "notasymlink" + just_a_file = target_dir / "just_a_file.txt" + + ret = file.recurse( + name=str(target_dir), source=f"salt://{target_dir.name}", keep_symlinks=True + ) + assert ret.result is True + + assert target_dir.exists() + assert target_source.is_file() + assert decoy_file.is_file() + assert just_a_file.is_file() + assert target_symlink.is_symlink() + + def test_issue_62117(file, symlink_scenario_2, tmp_path): target_dir = tmp_path / symlink_scenario_2 / "test" target_file_1 = target_dir / "test1" From 554da0c21efa1e891bc21ed25bd9e142782c35f0 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 12 Sep 2024 09:49:19 -0600 Subject: [PATCH 52/97] Compare full paths --- salt/states/file.py | 21 +++++++--- .../pytests/unit/states/file/test_recurse.py | 42 +++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 tests/pytests/unit/states/file/test_recurse.py diff --git a/salt/states/file.py b/salt/states/file.py index 717dafb47200..e4d9b7ece985 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -282,6 +282,7 @@ def run(): import itertools import logging import os +import pathlib import posixpath import re import shutil @@ -557,15 +558,23 @@ def process_symlinks(filenames, symlinks): managed_directories.add(mdest) keep.add(mdest) - # Sets are randomly ordered. We need to make sure symlinks are always at the - # end, so we need to use a list + # Sets are randomly ordered. We need to use a list so we can make sure + # symlinks are always at the end. This is necessary because the file must + # exist before we can create a symlink to it. See issue: + # https://github.com/saltstack/salt/issues/64630 new_managed_files = list(managed_files) # Now let's move all the symlinks to the end - for symlink in managed_symlinks: - for file in managed_files: - if file[0].endswith(os.sep + symlink[0]): + for link_src_relpath, _ in managed_symlinks: + for file_dest, file_src in managed_files: + # We need to convert relpath to fullpath. We're using pathlib to + # be platform-agnostic + symlink_full_path = pathlib.Path(f"{name}\\{link_src_relpath}") + file_dest_full_path = pathlib.Path(file_dest) + if symlink_full_path == file_dest_full_path: new_managed_files.append( - new_managed_files.pop(new_managed_files.index(file)) + new_managed_files.pop( + new_managed_files.index((file_dest, file_src)) + ) ) return new_managed_files, managed_directories, managed_symlinks, keep diff --git a/tests/pytests/unit/states/file/test_recurse.py b/tests/pytests/unit/states/file/test_recurse.py new file mode 100644 index 000000000000..dbbf1e1d2fec --- /dev/null +++ b/tests/pytests/unit/states/file/test_recurse.py @@ -0,0 +1,42 @@ +import logging +import pathlib + +import pytest + +import salt.states.file as filestate +from tests.support.mock import MagicMock, patch + +log = logging.getLogger(__name__) + + +@pytest.fixture +def configure_loader_modules(): + return {filestate: {"__salt__": {}, "__opts__": {}, "__env__": "base"}} + + +def test__gen_recurse_managed_files(): + """ + Test _gen_recurse_managed_files to make sure it puts symlinks at the end of the list of files. + """ + target_dir = pathlib.Path("\\some\\path\\target") + cp_list_master = MagicMock( + return_value=[ + "target/symlink", + "target/just_a_file.txt", + "target/not_a_symlink/symlink", + "target/notasymlink", + ], + ) + cp_list_master_symlinks = MagicMock( + return_value={"target/symlink": f"{target_dir}\\not_a_symlink\\symlink"} + ) + patch_salt = { + "cp.list_master": cp_list_master, + "cp.list_master_symlinks": cp_list_master_symlinks, + } + with patch.dict(filestate.__salt__, patch_salt): + files, dirs, links, keep = filestate._gen_recurse_managed_files( + name=str(target_dir), source=f"salt://{target_dir.name}", keep_symlinks=True + ) + expected = ("\\some\\path\\target\\symlink", "salt://target/symlink?saltenv=base") + assert files[-1] == expected From a5c3c184423b76823c8c35c147be58677077144a Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 12 Sep 2024 13:27:44 -0600 Subject: [PATCH 53/97] Fix unit test on Linux --- salt/states/file.py | 2 +- tests/pytests/unit/states/file/test_recurse.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index e4d9b7ece985..6d8c43b02b08 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -568,7 +568,7 @@ def process_symlinks(filenames, symlinks): for file_dest, file_src in managed_files: # We need to convert relpath to fullpath. We're using pathlib to # be platform-agnostic - symlink_full_path = pathlib.Path(f"{name}\\{link_src_relpath}") + symlink_full_path = pathlib.Path(f"{name}{os.sep}{link_src_relpath}") file_dest_full_path = pathlib.Path(file_dest) if symlink_full_path == file_dest_full_path: new_managed_files.append( diff --git a/tests/pytests/unit/states/file/test_recurse.py b/tests/pytests/unit/states/file/test_recurse.py index dbbf1e1d2fec..53e6e0fd22f1 100644 --- a/tests/pytests/unit/states/file/test_recurse.py +++ b/tests/pytests/unit/states/file/test_recurse.py @@ -1,4 +1,5 @@ import logging +import os import pathlib import pytest @@ -18,7 +19,7 @@ def test__gen_recurse_managed_files(): """ Test _gen_recurse_managed_files to make sure it puts symlinks at the end of the list of files. """ - target_dir = pathlib.Path("\\some\\path\\target") + target_dir = pathlib.Path(f"{os.sep}some{os.sep}path{os.sep}target") cp_list_master = MagicMock( return_value=[ "target/symlink", @@ -28,7 +29,9 @@ def test__gen_recurse_managed_files(): ], ) cp_list_master_symlinks = MagicMock( - return_value={"target/symlink": f"{target_dir}\\not_a_symlink\\symlink"} + return_value={ + "target/symlink": f"{target_dir}{os.sep}not_a_symlink{os.sep}symlink" + } ) patch_salt = { "cp.list_master": cp_list_master, @@ -38,5 +41,8 @@ def test__gen_recurse_managed_files(): files, dirs, links, keep = filestate._gen_recurse_managed_files( name=str(target_dir), source=f"salt://{target_dir.name}", keep_symlinks=True ) - expected = ("\\some\\path\\target\\symlink", "salt://target/symlink?saltenv=base") + expected = ( + f"{os.sep}some{os.sep}path{os.sep}target{os.sep}symlink", + "salt://target/symlink?saltenv=base", + ) assert files[-1] == expected From 99639ca5c9b5b7ba9dc88e63a86f7fa4fb871b23 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 17 Sep 2024 10:51:53 -0600 Subject: [PATCH 54/97] Update contributing guide and PR template --- .github/PULL_REQUEST_TEMPLATE.md | 16 +++-- CONTRIBUTING.rst | 78 ++++++++++++++++-------- doc/topics/development/pull_requests.rst | 19 ++++-- 3 files changed, 78 insertions(+), 35 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index f06d37e9e140..d707b6c88481 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -11,7 +11,9 @@ Remove this section if not relevant ### Merge requirements satisfied? **[NOTICE] Bug fixes or features added to Salt require tests.** - + - [ ] Docs - [ ] Changelog - https://docs.saltproject.io/en/master/topics/development/changelog.html - [ ] Tests written/updated @@ -19,7 +21,13 @@ Remove this section if not relevant ### Commits signed with GPG? Yes/No -Please review [Salt's Contributing Guide](https://docs.saltproject.io/en/master/topics/development/contributing.html) for best practices, including the -[PR Guidelines](https://docs.saltproject.io/en/master/topics/development/pull_requests.html). + -See GitHub's [page on GPG signing](https://help.github.com/articles/signing-commits-using-gpg/) for more information about signing commits with GPG. + + + diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 5379f4485728..c9c459ef5584 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,30 +1,56 @@ -============ -Contributing -============ +============================================== +Contributing to Salt: A Guide for Contributors +============================================== + +So, you want to contribute to the Salt project? That's fantastic! There are many +ways you can help improve Salt: + +- Use Salt and report bugs with clear, detailed descriptions. +- Join a `working group `__ to + collaborate with other contributors. +- Answer questions on platforms like `IRC `__, + the `community Discord `__, + the `salt-users mailing list `__, + `Server Fault `__, + or `r/saltstack on Reddit `__. +- Fix bugs or contribute to the `documentation `__. +- Submit workarounds, patches, or code (even without tests). +- Share your experiences and solutions to problems you've solved using Salt. + +Choosing the Right Branch for Your Pull Request +=============================================== + +We appreciate your contributions to the project! To ensure a smooth and +efficient workflow, please follow these guidelines when submitting a Pull +Request. Each type of contribution—whether it's fixing a bug, adding a feature, +updating documentation, or fixing tests—should be targeted at the appropriate +branch. This helps us manage changes effectively and maintain stability across +versions. + +- **Bug Fixes:** + + Create your Pull Request against the oldest supported branch where the bug + exists. This ensures that the fix can be applied to all relevant versions. + +- **New Features**: + + For new features or enhancements, create your Pull Request against the master + branch. + +- **Documentation Updates:** + + Documentation changes should be made against the master branch, unless they + are related to a bug fix, in which case they should follow the same branch as + the bug fix. + +- **Test Fixes:** + + Pull Requests that fix broken or failing tests should be created against the + oldest supported branch where the issue occurs. + +Setting Up Your Salt Development Environment +============================================ -So you want to contribute to the Salt project? Excellent! You can help -in a number of ways: - -- Use Salt and open well-written bug reports. -- Join a `working group `__. -- Answer questions on `irc `__, - the `community Discord `__, - the `salt-users mailing - list `__, - `Server Fault `__, - or `r/saltstack on Reddit `__. -- Fix bugs. -- `Improve the documentation `__. -- Provide workarounds, patches, or other code without tests. -- Tell other people about problems you solved using Salt. - -If you'd like to update docs or fix an issue, you're going to need the -Salt repo. The best way to contribute is using -`Git `__. - - -Environment setup -================= To hack on Salt or the docs you're going to need to set up your development environment. If you already have a workflow that you're comfortable with, you can use that, but otherwise this is an opinionated diff --git a/doc/topics/development/pull_requests.rst b/doc/topics/development/pull_requests.rst index 4b6ffee9135e..c1cb30a035c0 100644 --- a/doc/topics/development/pull_requests.rst +++ b/doc/topics/development/pull_requests.rst @@ -193,12 +193,21 @@ By default, PRs run a limited subset of the test suite against the following operating systems: * Linux: + - Latest ``Amazon Linux Arm64`` + - Latest ``Amazon Linux x86_64`` + - Latest ``Debian Linux Arm64`` + - Latest ``Debian Linux x86_64`` + - Latest ``Photon OS Arm64`` + - Latest ``Photon OS x86_64`` + - Latest ``Rocky Linux Arm64`` - Latest ``Rocky Linux x86_64`` - - Latest ``Amazon Linux aarch64`` - - Latest ``Ubuntu LTS arm64`` - - Latest ``Arch Linux x86_64`` -* Latest ``Windows Server x86_64`` -* Latest ``MacOS arm64`` + - Latest ``Ubuntu LTS Arm64`` + - Latest ``Ubuntu LTS x86_64`` +* Windows Server: + - Latest ``Windows Server x86_64`` +* macOS: + - Latest ``MacOS Arm64`` + - Latest ``MacOS x86_64`` Optional OS additions --------------------- From 6c38c1a253a0e34072a20d3a637b1339fe4b3f4a Mon Sep 17 00:00:00 2001 From: Marek Czernek Date: Fri, 30 Aug 2024 11:37:27 +0200 Subject: [PATCH 55/97] Fix test_system flaky setup_teardown fn --- tests/pytests/functional/modules/test_system.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/pytests/functional/modules/test_system.py b/tests/pytests/functional/modules/test_system.py index 07f34e8a5164..5944507c1ee0 100644 --- a/tests/pytests/functional/modules/test_system.py +++ b/tests/pytests/functional/modules/test_system.py @@ -5,10 +5,12 @@ import signal import subprocess import textwrap +import time import pytest import salt.utils.files +from salt.exceptions import CommandExecutionError pytestmark = [ pytest.mark.skip_unless_on_linux, @@ -76,7 +78,13 @@ def setup_teardown_vars(file, service, system): file.remove("/etc/machine-info") if _systemd_timesyncd_available_: - res = service.start("systemd-timesyncd") + try: + res = service.start("systemd-timesyncd") + except CommandExecutionError: + # We possibly did too many restarts in too short time + # Wait 10s (default systemd timeout) and try again + time.sleep(10) + res = service.start("systemd-timesyncd") assert res From eba0b174d7d36f5c318b7c4a76a600c2666dc111 Mon Sep 17 00:00:00 2001 From: Niklas Rousset Date: Mon, 26 Aug 2024 22:18:20 +0200 Subject: [PATCH 56/97] Add test cases for stdin_raw_newlines functionality --- tests/integration/modules/test_cmdmod.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/integration/modules/test_cmdmod.py b/tests/integration/modules/test_cmdmod.py index 4f2a72c45608..7d76aafcb743 100644 --- a/tests/integration/modules/test_cmdmod.py +++ b/tests/integration/modules/test_cmdmod.py @@ -78,6 +78,18 @@ def test_run(self): ), "four\nfive", ) + self.assertEqual( + self.run_function( + "cmd.run", ["cat"], stdin="one\\ntwo", stdin_raw_newlines=False + ), + "one\ntwo", + ) + self.assertEqual( + self.run_function( + "cmd.run", ["cat"], stdin="one\\ntwo", stdin_raw_newlines=True + ), + "one\\ntwo", + ) self.assertEqual( self.run_function( "cmd.run", ['echo "a=b" | sed -e s/=/:/g'], python_shell=True From 9da6764eb86adc9e43a113a901a44d1780fe669a Mon Sep 17 00:00:00 2001 From: Niklas Rousset Date: Mon, 26 Aug 2024 22:28:22 +0200 Subject: [PATCH 57/97] Convert stdin string to bytes regardless of stdin_raw_newlines --- changelog/62501.fixed.md | 1 + salt/utils/timed_subprocess.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 changelog/62501.fixed.md diff --git a/changelog/62501.fixed.md b/changelog/62501.fixed.md new file mode 100644 index 000000000000..5b9b04603220 --- /dev/null +++ b/changelog/62501.fixed.md @@ -0,0 +1 @@ +Convert stdin string to bytes regardless of stdin_raw_newlines diff --git a/salt/utils/timed_subprocess.py b/salt/utils/timed_subprocess.py index 627d3f712ed1..c41d1a7377bc 100644 --- a/salt/utils/timed_subprocess.py +++ b/salt/utils/timed_subprocess.py @@ -33,9 +33,8 @@ def __init__(self, args, **kwargs): if not self.stdin_raw_newlines: # Translate a newline submitted as '\n' on the CLI to an actual # newline character. - self.stdin = salt.utils.stringutils.to_bytes( - self.stdin.replace("\\n", "\n") - ) + self.stdin = self.stdin.replace("\\n", "\n") + self.stdin = salt.utils.stringutils.to_bytes(self.stdin) kwargs["stdin"] = subprocess.PIPE if not self.with_communicate: From 73bb35d943ba70b8ccac9c3c1f96a85226c4595e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 27 Aug 2024 16:04:41 -0600 Subject: [PATCH 58/97] Correct bash-completion for Debian / Ubuntu --- .../salt-common.bash-completion} | 187 ++++----- pkg/debian/salt-common.install | 2 +- pkg/rpm/salt.bash | 376 +++++++++++++++++- pkg/rpm/salt.spec | 2 +- 4 files changed, 449 insertions(+), 118 deletions(-) rename pkg/{common/salt.bash => debian/salt-common.bash-completion} (63%) mode change 120000 => 100644 pkg/rpm/salt.bash diff --git a/pkg/common/salt.bash b/pkg/debian/salt-common.bash-completion similarity index 63% rename from pkg/common/salt.bash rename to pkg/debian/salt-common.bash-completion index 35fe0695dbe8..aba866bc7958 100644 --- a/pkg/common/salt.bash +++ b/pkg/debian/salt-common.bash-completion @@ -5,86 +5,47 @@ # TODO: solve somehow completion for salt -G pythonversion:[tab] # (not sure what to do with lists) # TODO: --range[tab] -- how? +# TODO: -E --exsel[tab] -- how? # TODO: --compound[tab] -- how? # TODO: use history to extract some words, esp. if ${cur} is empty -# TODO: TEST EVERYTHING a lot +# TODO: TEST EVERYTING a lot +# TODO: cache results of some functions? where? how long? # TODO: is it ok to use '--timeout 2' ? _salt_get_grains(){ if [ "$1" = 'local' ] ; then - salt-call --log-level=error --out=txt -- grains.ls | sed 's/^.*\[//' | tr -d ",']" |sed 's:\([a-z0-9]\) :\1\: :g' + salt-call --out=txt -- grains.ls | sed 's/^.*\[//' | tr -d ",']" |sed 's:\([a-z0-9]\) :\1\: :g' else - salt '*' --timeout 2 --hide-timeout --log-level=error --out=txt -- grains.ls | sed 's/^.*\[//' | tr -d ",']" |sed 's:\([a-z0-9]\) :\1\: :g' + salt '*' --timeout 2 --out=txt -- grains.ls | sed 's/^.*\[//' | tr -d ",']" |sed 's:\([a-z0-9]\) :\1\: :g' fi } _salt_get_grain_values(){ if [ "$1" = 'local' ] ; then - salt-call --log-level=error --out=txt -- grains.item $1 |sed 's/^\S*:\s//' |grep -v '^\s*$' + salt-call --out=txt -- grains.item $1 |sed 's/^\S*:\s//' |grep -v '^\s*$' else - salt '*' --timeout 2 --hide-timeout --log-level=error --out=txt -- grains.item $1 |sed 's/^\S*:\s//' |grep -v '^\s*$' + salt '*' --timeout 2 --out=txt -- grains.item $1 |sed 's/^\S*:\s//' |grep -v '^\s*$' fi } -_salt_get_keys(){ - for type in $*; do - # remove header from data: - salt-key --no-color -l $type | tail -n+2 - done -} - -_salt_list_functions(){ - # salt-call: get all functions on this minion - # salt: get all functions on all minions - # sed: remove all array overhead and convert to newline separated list - # sort: chop out doubled entries, so overhead is minimal later during actual completion - if [ "$1" = 'local' ] ; then - salt-call --log-level=quiet --out=txt -- sys.list_functions \ - | sed "s/^.*\[//;s/[],']//g;s/ /\n/g" \ - | sort -u - else - salt '*' --timeout 2 --hide-timeout --log-level=quiet --out=txt -- sys.list_functions \ - | sed "s/^.*\[//;s/[],']//g;s/ /\n/g" \ - | sort -u - fi -} - -_salt_get_coms() { - CACHE_DIR="$HOME/.cache/salt-${1}-comp-cache_functions" - local _salt_cache_functions=${SALT_COMP_CACHE_FUNCTIONS:=$CACHE_DIR} - local _salt_cache_timeout=${SALT_COMP_CACHE_TIMEOUT:='last hour'} - - if [ ! -d "$(dirname ${_salt_cache_functions})" ]; then - mkdir -p "$(dirname ${_salt_cache_functions})" - fi - - # Regenerate cache if timed out - if [[ "$(stat --format=%Z ${_salt_cache_functions} 2>/dev/null)" -lt "$(date --date="${_salt_cache_timeout}" +%s)" ]]; then - _salt_list_functions $1 > "${_salt_cache_functions}" - fi - - # filter results, to only print the part to next dot (or end of function) - sed 's/^\('${cur}'\(\.\|[^.]*\)\)\?.*/\1/' "${_salt_cache_functions}" | sort -u -} _salt(){ - local cur prev opts _salt_grains _salt_coms pprev ppprev COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" if [ ${COMP_CWORD} -gt 2 ]; then - pprev="${COMP_WORDS[COMP_CWORD-2]}" + pprev="${COMP_WORDS[COMP_CWORD-2]}" fi if [ ${COMP_CWORD} -gt 3 ]; then - ppprev="${COMP_WORDS[COMP_CWORD-3]}" + ppprev="${COMP_WORDS[COMP_CWORD-3]}" fi opts="-h --help -d --doc --documentation --version --versions-report -c \ --config-dir= -v --verbose -t --timeout= -s --static -b --batch= \ --batch-size= -E --pcre -L --list -G --grain --grain-pcre -N \ - --nodegroup -R --range -C --compound -I --pillar \ + --nodegroup -R --range -C --compound -X --exsel -I --pillar \ --return= -a --auth= --eauth= --extended-auth= -T --make-token -S \ --ipcidr --out=pprint --out=yaml --out=overstatestage --out=json \ --out=raw --out=highstate --out=key --out=txt --no-color --out-indent= " @@ -98,7 +59,7 @@ _salt(){ case "${pprev}" in -G|--grain|--grain-pcre) if [ "${cur}" = ":" ]; then - COMPREPLY=($(compgen -W "`_salt_get_grain_values ${prev}`")) + COMPREPLY=($(compgen -W "`_salt_get_grain_values ${prev}`" )) return 0 fi ;; @@ -126,17 +87,17 @@ _salt(){ return 0 ;; salt) - COMPREPLY=($(compgen -W "\'*\' ${opts} $(_salt_get_keys acc)" -- ${cur})) + COMPREPLY=($(compgen -W "\'*\' ${opts} `salt-key --no-color -l acc`" -- ${cur})) return 0 ;; -E|--pcre) - COMPREPLY=($(compgen -W "$(_salt_get_keys acc)" -- ${cur})) + COMPREPLY=($(compgen -W "`salt-key --no-color -l acc`" -- ${cur})) return 0 ;; -G|--grain|--grain-pcre) COMPREPLY=($(compgen -W "$(_salt_get_grains)" -- ${cur})) return 0 - ;; + ;; -C|--compound) COMPREPLY=() # TODO: finish this one? how? return 0 @@ -149,18 +110,17 @@ _salt(){ COMPREPLY=($(compgen -W "1 2 3 4 5 6 7 8 9 10 15 20 30 40 50 60 70 80 90 100 120 150 200")) return 0 ;; + -X|--exsel) # TODO: finish this one? how? + return 0 + ;; -N|--nodegroup) - MASTER_CONFIG='/etc/salt/master' + MASTER_CONFIG='/etc/salt/master' COMPREPLY=($(compgen -W "`awk -F ':' 'BEGIN {print_line = 0}; /^nodegroups/ {print_line = 1;getline } print_line && /^ */ {print $1} /^[^ ]/ {print_line = 0}' <${MASTER_CONFIG}`" -- ${cur})) return 0 ;; esac - _salt_coms=$(_salt_get_coms remote) - - # If there are still dots in the suggestion, do not append space - grep "^${cur}.*\." "${_salt_coms}" &>/dev/null && compopt -o nospace - + _salt_coms="$(salt '*' --timeout 2 --out=txt -- sys.list_functions | sed 's/^.*\[//' | tr -d ",']" )" all="${opts} ${_salt_coms}" COMPREPLY=( $(compgen -W "${all}" -- ${cur}) ) @@ -202,15 +162,15 @@ _saltkey(){ case "${prev}" in -a|--accept) - COMPREPLY=($(compgen -W "$(_salt_get_keys un rej)" -- ${cur})) + COMPREPLY=($(compgen -W "$(salt-key -l un --no-color; salt-key -l rej --no-color)" -- ${cur})) return 0 ;; -r|--reject) - COMPREPLY=($(compgen -W "$(_salt_get_keys acc)" -- ${cur})) + COMPREPLY=($(compgen -W "$(salt-key -l acc --no-color)" -- ${cur})) return 0 ;; -d|--delete) - COMPREPLY=($(compgen -W "$(_salt_get_keys acc un rej)" -- ${cur})) + COMPREPLY=($(compgen -W "$(salt-key -l acc --no-color; salt-key -l un --no-color; salt-key -l rej --no-color)" -- ${cur})) return 0 ;; -c|--config) @@ -229,7 +189,7 @@ _saltkey(){ return 0 ;; -p|--print) - COMPREPLY=($(compgen -W "$(_salt_get_keys acc un rej)" -- ${cur})) + COMPREPLY=($(compgen -W "$(salt-key -l acc --no-color; salt-key -l un --no-color; salt-key -l rej --no-color)" -- ${cur})) return 0 ;; -l|--list) @@ -237,7 +197,7 @@ _saltkey(){ return 0 ;; --accept-all) - return 0 + return 0 ;; esac COMPREPLY=($(compgen -W "${opts} " -- ${cur})) @@ -276,26 +236,22 @@ _saltcall(){ case ${prev} in -m|--module-dirs) COMPREPLY=( $(compgen -d ${cur} )) + return 0 + ;; + -l|--log-level) + COMPREPLY=( $(compgen -W "info none garbage trace warning error debug" -- ${cur})) + return 0 + ;; + -g|grains) return 0 - ;; - -l|--log-level) - COMPREPLY=( $(compgen -W "info none garbage trace warning error debug" -- ${cur})) - return 0 - ;; - -g|grains) - return 0 - ;; - salt-call) + ;; + salt-call) COMPREPLY=($(compgen -W "${opts}" -- ${cur})) - return 0 - ;; + return 0 + ;; esac - _salt_coms=$(_salt_get_coms local) - - # If there are still dots in the suggestion, do not append space - grep "^${cur}.*\." "${_salt_coms}" &>/dev/null && compopt -o nospace - + _salt_coms="$(salt-call --out=txt -- sys.list_functions|sed 's/^.*\[//' | tr -d ",']" )" COMPREPLY=( $(compgen -W "${opts} ${_salt_coms}" -- ${cur} )) return 0 } @@ -311,7 +267,7 @@ _saltcp(){ opts="-t --timeout= -s --static -b --batch= --batch-size= \ -h --help --version --versions-report -c --config-dir= \ -E --pcre -L --list -G --grain --grain-pcre -N --nodegroup \ - -R --range -C --compound -I --pillar \ + -R --range -C --compound -X --exsel -I --pillar \ --out=pprint --out=yaml --out=overstatestage --out=json --out=raw \ --out=highstate --out=key --out=txt --no-color --out-indent= " if [[ "${cur}" == -* ]] ; then @@ -327,45 +283,46 @@ _saltcp(){ fi case ${prev} in - salt-cp) - COMPREPLY=($(compgen -W "${opts} $(_salt_get_keys acc)" -- ${cur})) - return 0 - ;; + salt-cp) + COMPREPLY=($(compgen -W "${opts} `salt-key -l acc --no-color`" -- ${cur})) + return 0 + ;; -t|--timeout) - # those numbers are just a hint + # those numbers are just a hint COMPREPLY=($(compgen -W "2 3 4 8 10 15 20 25 30 40 60 90 120 180 240 300" -- ${cur} )) + return 0 + ;; + -E|--pcre) + COMPREPLY=($(compgen -W "`salt-key -l acc --no-color`" -- ${cur})) return 0 - ;; - -E|--pcre) - COMPREPLY=($(compgen -W "$(_salt_get_keys acc)" -- ${cur})) - return 0 - ;; - -L|--list) - # IMPROVEMENTS ARE WELCOME - prefpart="${cur%,*}," - postpart=${cur##*,} - filt="^\($(echo ${cur}| sed 's:,:\\|:g')\)$" - helper=($(_salt_get_keys acc | grep -v "${filt}" | sed "s/^/${prefpart}/")) - COMPREPLY=($(compgen -W "${helper[*]}" -- ${cur})) - return 0 - ;; - -G|--grain|--grain-pcre) + ;; + -L|--list) + # IMPROVEMENTS ARE WELCOME + prefpart="${cur%,*}," + postpart=${cur##*,} + filt="^\($(echo ${cur}| sed 's:,:\\|:g')\)$" + helper=($(salt-key -l acc --no-color | grep -v "${filt}" | sed "s/^/${prefpart}/")) + COMPREPLY=($(compgen -W "${helper[*]}" -- ${cur})) + + return 0 + ;; + -G|--grain|--grain-pcre) COMPREPLY=($(compgen -W "$(_salt_get_grains)" -- ${cur})) return 0 - ;; - # FIXME - -R|--range) - # FIXME ?? - return 0 - ;; - -C|--compound) - # FIXME ?? - return 0 - ;; - -c|--config) - COMPREPLY=($(compgen -f -- ${cur})) - return 0 - ;; + ;; + # FIXME + -R|--range) + # FIXME ?? + return 0 + ;; + -C|--compound) + # FIXME ?? + return 0 + ;; + -c|--config) + COMPREPLY=($(compgen -f -- ${cur})) + return 0 + ;; esac # default is using opts: diff --git a/pkg/debian/salt-common.install b/pkg/debian/salt-common.install index 4f8dac552ece..ef3c2b520dec 100644 --- a/pkg/debian/salt-common.install +++ b/pkg/debian/salt-common.install @@ -3,9 +3,9 @@ pkg/common/fish-completions/salt-cp.fish /usr/share/fish/vendor_completions.d pkg/common/fish-completions/salt-call.fish /usr/share/fish/vendor_completions.d pkg/common/fish-completions/salt-syndic.fish /usr/share/fish/vendor_completions.d pkg/common/fish-completions/salt_common.fish /usr/share/fish/vendor_completions.d -pkg/common/salt.bash /usr/share/bash-completions/completions/salt-common.bash pkg/common/fish-completions/salt-minion.fish /usr/share/fish/vendor_completions.d pkg/common/fish-completions/salt-key.fish /usr/share/fish/vendor_completions.d pkg/common/fish-completions/salt-master.fish /usr/share/fish/vendor_completions.d pkg/common/fish-completions/salt-run.fish /usr/share/fish/vendor_completions.d pkg/common/fish-completions/salt.fish /usr/share/fish/vendor_completions.d +pkg/debian/salt-common.bash-completion /usr/share/bash-completion/completions/salt-common diff --git a/pkg/rpm/salt.bash b/pkg/rpm/salt.bash deleted file mode 120000 index 98ee56c40cd6..000000000000 --- a/pkg/rpm/salt.bash +++ /dev/null @@ -1 +0,0 @@ -../common/salt.bash \ No newline at end of file diff --git a/pkg/rpm/salt.bash b/pkg/rpm/salt.bash new file mode 100644 index 000000000000..35fe0695dbe8 --- /dev/null +++ b/pkg/rpm/salt.bash @@ -0,0 +1,375 @@ +# written by David Pravec +# - feel free to /msg alekibango on IRC if you want to talk about this file + +# TODO: check if --config|-c was used and use configured config file for queries +# TODO: solve somehow completion for salt -G pythonversion:[tab] +# (not sure what to do with lists) +# TODO: --range[tab] -- how? +# TODO: --compound[tab] -- how? +# TODO: use history to extract some words, esp. if ${cur} is empty +# TODO: TEST EVERYTHING a lot +# TODO: is it ok to use '--timeout 2' ? + + +_salt_get_grains(){ + if [ "$1" = 'local' ] ; then + salt-call --log-level=error --out=txt -- grains.ls | sed 's/^.*\[//' | tr -d ",']" |sed 's:\([a-z0-9]\) :\1\: :g' + else + salt '*' --timeout 2 --hide-timeout --log-level=error --out=txt -- grains.ls | sed 's/^.*\[//' | tr -d ",']" |sed 's:\([a-z0-9]\) :\1\: :g' + fi +} + +_salt_get_grain_values(){ + if [ "$1" = 'local' ] ; then + salt-call --log-level=error --out=txt -- grains.item $1 |sed 's/^\S*:\s//' |grep -v '^\s*$' + else + salt '*' --timeout 2 --hide-timeout --log-level=error --out=txt -- grains.item $1 |sed 's/^\S*:\s//' |grep -v '^\s*$' + fi +} + +_salt_get_keys(){ + for type in $*; do + # remove header from data: + salt-key --no-color -l $type | tail -n+2 + done +} + +_salt_list_functions(){ + # salt-call: get all functions on this minion + # salt: get all functions on all minions + # sed: remove all array overhead and convert to newline separated list + # sort: chop out doubled entries, so overhead is minimal later during actual completion + if [ "$1" = 'local' ] ; then + salt-call --log-level=quiet --out=txt -- sys.list_functions \ + | sed "s/^.*\[//;s/[],']//g;s/ /\n/g" \ + | sort -u + else + salt '*' --timeout 2 --hide-timeout --log-level=quiet --out=txt -- sys.list_functions \ + | sed "s/^.*\[//;s/[],']//g;s/ /\n/g" \ + | sort -u + fi +} + +_salt_get_coms() { + CACHE_DIR="$HOME/.cache/salt-${1}-comp-cache_functions" + local _salt_cache_functions=${SALT_COMP_CACHE_FUNCTIONS:=$CACHE_DIR} + local _salt_cache_timeout=${SALT_COMP_CACHE_TIMEOUT:='last hour'} + + if [ ! -d "$(dirname ${_salt_cache_functions})" ]; then + mkdir -p "$(dirname ${_salt_cache_functions})" + fi + + # Regenerate cache if timed out + if [[ "$(stat --format=%Z ${_salt_cache_functions} 2>/dev/null)" -lt "$(date --date="${_salt_cache_timeout}" +%s)" ]]; then + _salt_list_functions $1 > "${_salt_cache_functions}" + fi + + # filter results, to only print the part to next dot (or end of function) + sed 's/^\('${cur}'\(\.\|[^.]*\)\)\?.*/\1/' "${_salt_cache_functions}" | sort -u +} + +_salt(){ + + local cur prev opts _salt_grains _salt_coms pprev ppprev + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + if [ ${COMP_CWORD} -gt 2 ]; then + pprev="${COMP_WORDS[COMP_CWORD-2]}" + fi + if [ ${COMP_CWORD} -gt 3 ]; then + ppprev="${COMP_WORDS[COMP_CWORD-3]}" + fi + + opts="-h --help -d --doc --documentation --version --versions-report -c \ + --config-dir= -v --verbose -t --timeout= -s --static -b --batch= \ + --batch-size= -E --pcre -L --list -G --grain --grain-pcre -N \ + --nodegroup -R --range -C --compound -I --pillar \ + --return= -a --auth= --eauth= --extended-auth= -T --make-token -S \ + --ipcidr --out=pprint --out=yaml --out=overstatestage --out=json \ + --out=raw --out=highstate --out=key --out=txt --no-color --out-indent= " + + if [[ "${cur}" == -* ]] ; then + COMPREPLY=($(compgen -W "${opts}" -- ${cur})) + return 0 + fi + + # 2 special cases for filling up grain values + case "${pprev}" in + -G|--grain|--grain-pcre) + if [ "${cur}" = ":" ]; then + COMPREPLY=($(compgen -W "`_salt_get_grain_values ${prev}`")) + return 0 + fi + ;; + esac + case "${ppprev}" in + -G|--grain|--grain-pcre) + if [ "${prev}" = ":" ]; then + COMPREPLY=( $(compgen -W "`_salt_get_grain_values ${pprev}`" -- ${cur}) ) + return 0 + fi + ;; + esac + + if [ "${cur}" = "=" ] && [[ "${prev}" == --* ]]; then + cur="" + fi + if [ "${prev}" = "=" ] && [[ "${pprev}" == --* ]]; then + prev="${pprev}" + fi + + case "${prev}" in + + -c|--config) + COMPREPLY=($(compgen -f -- ${cur})) + return 0 + ;; + salt) + COMPREPLY=($(compgen -W "\'*\' ${opts} $(_salt_get_keys acc)" -- ${cur})) + return 0 + ;; + -E|--pcre) + COMPREPLY=($(compgen -W "$(_salt_get_keys acc)" -- ${cur})) + return 0 + ;; + -G|--grain|--grain-pcre) + COMPREPLY=($(compgen -W "$(_salt_get_grains)" -- ${cur})) + return 0 + ;; + -C|--compound) + COMPREPLY=() # TODO: finish this one? how? + return 0 + ;; + -t|--timeout) + COMPREPLY=($( compgen -W "1 2 3 4 5 6 7 8 9 10 15 20 30 40 60 90 120 180" -- ${cur})) + return 0 + ;; + -b|--batch|--batch-size) + COMPREPLY=($(compgen -W "1 2 3 4 5 6 7 8 9 10 15 20 30 40 50 60 70 80 90 100 120 150 200")) + return 0 + ;; + -N|--nodegroup) + MASTER_CONFIG='/etc/salt/master' + COMPREPLY=($(compgen -W "`awk -F ':' 'BEGIN {print_line = 0}; /^nodegroups/ {print_line = 1;getline } print_line && /^ */ {print $1} /^[^ ]/ {print_line = 0}' <${MASTER_CONFIG}`" -- ${cur})) + return 0 + ;; + esac + + _salt_coms=$(_salt_get_coms remote) + + # If there are still dots in the suggestion, do not append space + grep "^${cur}.*\." "${_salt_coms}" &>/dev/null && compopt -o nospace + + all="${opts} ${_salt_coms}" + COMPREPLY=( $(compgen -W "${all}" -- ${cur}) ) + + return 0 +} + +complete -F _salt salt + + +_saltkey(){ + local cur prev opts prev pprev + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + opts="-c --config-dir= -h --help --version --versions-report -q --quiet \ + -y --yes --gen-keys= --gen-keys-dir= --keysize= --key-logfile= \ + -l --list= -L --list-all -a --accept= -A --accept-all \ + -r --reject= -R --reject-all -p --print= -P --print-all \ + -d --delete= -D --delete-all -f --finger= -F --finger-all \ + --out=pprint --out=yaml --out=overstatestage --out=json --out=raw \ + --out=highstate --out=key --out=txt --no-color --out-indent= " + if [ ${COMP_CWORD} -gt 2 ]; then + pprev="${COMP_WORDS[COMP_CWORD-2]}" + fi + if [ ${COMP_CWORD} -gt 3 ]; then + ppprev="${COMP_WORDS[COMP_CWORD-3]}" + fi + if [[ "${cur}" == -* ]] ; then + COMPREPLY=($(compgen -W "${opts}" -- ${cur})) + return 0 + fi + + if [ "${cur}" = "=" ] && [[ "${prev}" == --* ]]; then + cur="" + fi + if [ "${prev}" = "=" ] && [[ "${pprev}" == --* ]]; then + prev="${pprev}" + fi + + case "${prev}" in + -a|--accept) + COMPREPLY=($(compgen -W "$(_salt_get_keys un rej)" -- ${cur})) + return 0 + ;; + -r|--reject) + COMPREPLY=($(compgen -W "$(_salt_get_keys acc)" -- ${cur})) + return 0 + ;; + -d|--delete) + COMPREPLY=($(compgen -W "$(_salt_get_keys acc un rej)" -- ${cur})) + return 0 + ;; + -c|--config) + COMPREPLY=($(compgen -f -- ${cur})) + return 0 + ;; + --keysize) + COMPREPLY=($(compgen -W "2048 3072 4096 5120 6144" -- ${cur})) + return 0 + ;; + --gen-keys) + return 0 + ;; + --gen-keys-dir) + COMPREPLY=($(compgen -d -- ${cur})) + return 0 + ;; + -p|--print) + COMPREPLY=($(compgen -W "$(_salt_get_keys acc un rej)" -- ${cur})) + return 0 + ;; + -l|--list) + COMPREPLY=($(compgen -W "pre un acc accepted unaccepted rej rejected all" -- ${cur})) + return 0 + ;; + --accept-all) + return 0 + ;; + esac + COMPREPLY=($(compgen -W "${opts} " -- ${cur})) + return 0 +} + +complete -F _saltkey salt-key + +_saltcall(){ + local cur prev opts _salt_coms pprev ppprev + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + opts="-h --help -d --doc --documentation --version --versions-report \ + -m --module-dirs= -g --grains --return= --local -c --config-dir= -l --log-level= \ + --out=pprint --out=yaml --out=overstatestage --out=json --out=raw \ + --out=highstate --out=key --out=txt --no-color --out-indent= " + if [ ${COMP_CWORD} -gt 2 ]; then + pprev="${COMP_WORDS[COMP_CWORD-2]}" + fi + if [ ${COMP_CWORD} -gt 3 ]; then + ppprev="${COMP_WORDS[COMP_CWORD-3]}" + fi + if [[ "${cur}" == -* ]] ; then + COMPREPLY=($(compgen -W "${opts}" -- ${cur})) + return 0 + fi + + if [ "${cur}" = "=" ] && [[ ${prev} == --* ]]; then + cur="" + fi + if [ "${prev}" = "=" ] && [[ ${pprev} == --* ]]; then + prev="${pprev}" + fi + + case ${prev} in + -m|--module-dirs) + COMPREPLY=( $(compgen -d ${cur} )) + return 0 + ;; + -l|--log-level) + COMPREPLY=( $(compgen -W "info none garbage trace warning error debug" -- ${cur})) + return 0 + ;; + -g|grains) + return 0 + ;; + salt-call) + COMPREPLY=($(compgen -W "${opts}" -- ${cur})) + return 0 + ;; + esac + + _salt_coms=$(_salt_get_coms local) + + # If there are still dots in the suggestion, do not append space + grep "^${cur}.*\." "${_salt_coms}" &>/dev/null && compopt -o nospace + + COMPREPLY=( $(compgen -W "${opts} ${_salt_coms}" -- ${cur} )) + return 0 +} + +complete -F _saltcall salt-call + + +_saltcp(){ + local cur prev opts target prefpart postpart helper filt pprev ppprev + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + opts="-t --timeout= -s --static -b --batch= --batch-size= \ + -h --help --version --versions-report -c --config-dir= \ + -E --pcre -L --list -G --grain --grain-pcre -N --nodegroup \ + -R --range -C --compound -I --pillar \ + --out=pprint --out=yaml --out=overstatestage --out=json --out=raw \ + --out=highstate --out=key --out=txt --no-color --out-indent= " + if [[ "${cur}" == -* ]] ; then + COMPREPLY=($(compgen -W "${opts}" -- ${cur})) + return 0 + fi + + if [ "${cur}" = "=" ] && [[ "${prev}" == --* ]]; then + cur="" + fi + if [ "${prev}" = "=" ] && [[ "${pprev}" == --* ]]; then + prev=${pprev} + fi + + case ${prev} in + salt-cp) + COMPREPLY=($(compgen -W "${opts} $(_salt_get_keys acc)" -- ${cur})) + return 0 + ;; + -t|--timeout) + # those numbers are just a hint + COMPREPLY=($(compgen -W "2 3 4 8 10 15 20 25 30 40 60 90 120 180 240 300" -- ${cur} )) + return 0 + ;; + -E|--pcre) + COMPREPLY=($(compgen -W "$(_salt_get_keys acc)" -- ${cur})) + return 0 + ;; + -L|--list) + # IMPROVEMENTS ARE WELCOME + prefpart="${cur%,*}," + postpart=${cur##*,} + filt="^\($(echo ${cur}| sed 's:,:\\|:g')\)$" + helper=($(_salt_get_keys acc | grep -v "${filt}" | sed "s/^/${prefpart}/")) + COMPREPLY=($(compgen -W "${helper[*]}" -- ${cur})) + return 0 + ;; + -G|--grain|--grain-pcre) + COMPREPLY=($(compgen -W "$(_salt_get_grains)" -- ${cur})) + return 0 + ;; + # FIXME + -R|--range) + # FIXME ?? + return 0 + ;; + -C|--compound) + # FIXME ?? + return 0 + ;; + -c|--config) + COMPREPLY=($(compgen -f -- ${cur})) + return 0 + ;; + esac + + # default is using opts: + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) +} + +complete -F _saltcp salt-cp diff --git a/pkg/rpm/salt.spec b/pkg/rpm/salt.spec index f1707cb5dd07..ad21a07e2614 100644 --- a/pkg/rpm/salt.spec +++ b/pkg/rpm/salt.spec @@ -283,7 +283,7 @@ install -p -m 0644 %{_salt_src}/pkg/common/logrotate/salt-common %{buildroot}%{_ # Bash completion mkdir -p %{buildroot}%{_sysconfdir}/bash_completion.d/ -install -p -m 0644 %{_salt_src}/pkg/common/salt.bash %{buildroot}%{_sysconfdir}/bash_completion.d/salt.bash +install -p -m 0644 %{_salt_src}/pkg/rpm/salt.bash %{buildroot}%{_sysconfdir}/bash_completion.d/salt.bash # Fish completion (TBD remove -v) mkdir -p %{buildroot}%{fish_dir} From 2532a088cda28d7ef7f2e2a26ddc978ba4f64998 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 27 Aug 2024 16:12:33 -0600 Subject: [PATCH 59/97] Added changelog for fix --- changelog/66560.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/66560.fixed.md diff --git a/changelog/66560.fixed.md b/changelog/66560.fixed.md new file mode 100644 index 000000000000..e71230f25b44 --- /dev/null +++ b/changelog/66560.fixed.md @@ -0,0 +1 @@ +Correct bash-completion for Debian / Ubuntu From 02c196ad13381df9060b83f8be8f4810b1a26b8d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 28 Aug 2024 10:32:56 -0600 Subject: [PATCH 60/97] Add symbolic links for bash-completion --- pkg/debian/salt-common.dirs | 1 + pkg/debian/salt-common.install | 4 ++++ pkg/debian/salt-common.links | 7 +++++++ pkg/debian/salt-master.links | 4 ++++ pkg/debian/salt-minion.links | 2 ++ 5 files changed, 18 insertions(+) diff --git a/pkg/debian/salt-common.dirs b/pkg/debian/salt-common.dirs index 381ec1f48ce0..f818dc8d15b5 100644 --- a/pkg/debian/salt-common.dirs +++ b/pkg/debian/salt-common.dirs @@ -2,6 +2,7 @@ /var/log/salt /var/run/salt /usr/share/fish/vendor_completions.d +/usr/share/zsh/vendor-completions /opt/saltstack/salt /etc/salt /etc/logrotate.d diff --git a/pkg/debian/salt-common.install b/pkg/debian/salt-common.install index ef3c2b520dec..4e79cc91d9ad 100644 --- a/pkg/debian/salt-common.install +++ b/pkg/debian/salt-common.install @@ -1,3 +1,6 @@ +#! /usr/bin/dh-exec + +usr/lib pkg/common/logrotate/salt-common /etc/logrotate.d pkg/common/fish-completions/salt-cp.fish /usr/share/fish/vendor_completions.d pkg/common/fish-completions/salt-call.fish /usr/share/fish/vendor_completions.d @@ -9,3 +12,4 @@ pkg/common/fish-completions/salt-master.fish /usr/share/fish/vendor_completions. pkg/common/fish-completions/salt-run.fish /usr/share/fish/vendor_completions.d pkg/common/fish-completions/salt.fish /usr/share/fish/vendor_completions.d pkg/debian/salt-common.bash-completion /usr/share/bash-completion/completions/salt-common +pkg/salt.zsh => usr/share/zsh/vendor-completions/_salt diff --git a/pkg/debian/salt-common.links b/pkg/debian/salt-common.links index ef1cd42e5dd1..cddd400ceeb0 100644 --- a/pkg/debian/salt-common.links +++ b/pkg/debian/salt-common.links @@ -1,2 +1,9 @@ +# permissions on /var/log/salt to permit adm group ownership +salt-common: non-standard-dir-perm + +# minor formatting error in table in man page +salt-common: manpage-has-errors-from-man + opt/saltstack/salt/salt-pip /usr/bin/salt-pip opt/saltstack/salt/salt-call /usr/bin/salt-call +usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt-call diff --git a/pkg/debian/salt-master.links b/pkg/debian/salt-master.links index e6c0ef2446ad..9588e277100f 100644 --- a/pkg/debian/salt-master.links +++ b/pkg/debian/salt-master.links @@ -4,3 +4,7 @@ opt/saltstack/salt/salt-cp /usr/bin/salt-cp opt/saltstack/salt/salt-key /usr/bin/salt-key opt/saltstack/salt/salt-run /usr/bin/salt-run opt/saltstack/salt/spm /usr/bin/spm +usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt +usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt-cp +usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt-key +usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt-run diff --git a/pkg/debian/salt-minion.links b/pkg/debian/salt-minion.links index 9dae19eb1d3a..ad3236c8f1d3 100644 --- a/pkg/debian/salt-minion.links +++ b/pkg/debian/salt-minion.links @@ -1,2 +1,4 @@ opt/saltstack/salt/salt-minion /usr/bin/salt-minion opt/saltstack/salt/salt-proxy /usr/bin/salt-proxy +usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt-minion +usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt-proxy From fd9001804593e5704ca177743269e99cdf2086ae Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 28 Aug 2024 13:22:24 -0600 Subject: [PATCH 61/97] Corrected files for zsh and salt-proxy --- pkg/debian/salt-common.dirs | 1 - pkg/debian/salt-common.install | 2 -- pkg/debian/salt-minion.links | 1 - 3 files changed, 4 deletions(-) diff --git a/pkg/debian/salt-common.dirs b/pkg/debian/salt-common.dirs index f818dc8d15b5..381ec1f48ce0 100644 --- a/pkg/debian/salt-common.dirs +++ b/pkg/debian/salt-common.dirs @@ -2,7 +2,6 @@ /var/log/salt /var/run/salt /usr/share/fish/vendor_completions.d -/usr/share/zsh/vendor-completions /opt/saltstack/salt /etc/salt /etc/logrotate.d diff --git a/pkg/debian/salt-common.install b/pkg/debian/salt-common.install index 4e79cc91d9ad..372cf029a08b 100644 --- a/pkg/debian/salt-common.install +++ b/pkg/debian/salt-common.install @@ -1,6 +1,5 @@ #! /usr/bin/dh-exec -usr/lib pkg/common/logrotate/salt-common /etc/logrotate.d pkg/common/fish-completions/salt-cp.fish /usr/share/fish/vendor_completions.d pkg/common/fish-completions/salt-call.fish /usr/share/fish/vendor_completions.d @@ -12,4 +11,3 @@ pkg/common/fish-completions/salt-master.fish /usr/share/fish/vendor_completions. pkg/common/fish-completions/salt-run.fish /usr/share/fish/vendor_completions.d pkg/common/fish-completions/salt.fish /usr/share/fish/vendor_completions.d pkg/debian/salt-common.bash-completion /usr/share/bash-completion/completions/salt-common -pkg/salt.zsh => usr/share/zsh/vendor-completions/_salt diff --git a/pkg/debian/salt-minion.links b/pkg/debian/salt-minion.links index ad3236c8f1d3..546a2c8e4b10 100644 --- a/pkg/debian/salt-minion.links +++ b/pkg/debian/salt-minion.links @@ -1,4 +1,3 @@ opt/saltstack/salt/salt-minion /usr/bin/salt-minion opt/saltstack/salt/salt-proxy /usr/bin/salt-proxy usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt-minion -usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt-proxy From 9c712e09eacd8f663b6407c29d7523eed35beb36 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 29 Aug 2024 11:13:35 -0600 Subject: [PATCH 62/97] Correct symbolic link generation to corrspond with Debina/Ubuntu fork of Salt --- pkg/debian/salt-common.links | 1 - pkg/debian/salt-master.links | 4 ---- pkg/debian/salt-minion.links | 1 - 3 files changed, 6 deletions(-) diff --git a/pkg/debian/salt-common.links b/pkg/debian/salt-common.links index cddd400ceeb0..29d5cb3e7e0f 100644 --- a/pkg/debian/salt-common.links +++ b/pkg/debian/salt-common.links @@ -6,4 +6,3 @@ salt-common: manpage-has-errors-from-man opt/saltstack/salt/salt-pip /usr/bin/salt-pip opt/saltstack/salt/salt-call /usr/bin/salt-call -usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt-call diff --git a/pkg/debian/salt-master.links b/pkg/debian/salt-master.links index 9588e277100f..e6c0ef2446ad 100644 --- a/pkg/debian/salt-master.links +++ b/pkg/debian/salt-master.links @@ -4,7 +4,3 @@ opt/saltstack/salt/salt-cp /usr/bin/salt-cp opt/saltstack/salt/salt-key /usr/bin/salt-key opt/saltstack/salt/salt-run /usr/bin/salt-run opt/saltstack/salt/spm /usr/bin/spm -usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt -usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt-cp -usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt-key -usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt-run diff --git a/pkg/debian/salt-minion.links b/pkg/debian/salt-minion.links index 546a2c8e4b10..9dae19eb1d3a 100644 --- a/pkg/debian/salt-minion.links +++ b/pkg/debian/salt-minion.links @@ -1,3 +1,2 @@ opt/saltstack/salt/salt-minion /usr/bin/salt-minion opt/saltstack/salt/salt-proxy /usr/bin/salt-proxy -usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt-minion From 4be168509ff1508dfb7816ae6f962eebe8b46bb5 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 3 Sep 2024 14:52:01 -0600 Subject: [PATCH 63/97] Testing building with bash-completion and systemd --- pkg/debian/rules | 2 +- pkg/debian/salt-api.postinst | 42 ++++++++++++++++----------------- pkg/debian/salt-master.postinst | 42 ++++++++++++++++----------------- pkg/debian/salt-master.preinst | 20 ++++++++-------- pkg/debian/salt-minion.postinst | 42 ++++++++++++++++----------------- pkg/debian/salt-minion.preinst | 18 +++++++------- 6 files changed, 83 insertions(+), 83 deletions(-) diff --git a/pkg/debian/rules b/pkg/debian/rules index 6ed990418e4d..26d0112a5e5f 100755 --- a/pkg/debian/rules +++ b/pkg/debian/rules @@ -4,7 +4,7 @@ DH_VERBOSE = 1 .PHONY: override_dh_strip %: - dh $@ + dh $@ --with python3,bash-completion --with=systemd # dh_auto_clean tries to invoke distutils causing failures. override_dh_auto_clean: diff --git a/pkg/debian/salt-api.postinst b/pkg/debian/salt-api.postinst index 3b78211922ad..fecd7e5d336f 100644 --- a/pkg/debian/salt-api.postinst +++ b/pkg/debian/salt-api.postinst @@ -12,26 +12,26 @@ case "$1" in fi chown $RET:$RET /var/log/salt/api fi - if command -v systemctl; then - db_get salt-api/active - RESLT=$(echo "$RET" | cut -d ' ' -f 1) - if [ "$RESLT" != 10 ]; then - systemctl daemon-reload - if [ "$RESLT" = "active" ]; then - systemctl restart salt-api - fi - db_get salt-api/enabled - RESLT=$(echo "$RET" | cut -d ' ' -f 1) - if [ "$RESLT" = "disabled" ]; then - systemctl disable salt-api - else - systemctl enable salt-api - fi - else - systemctl daemon-reload - systemctl restart salt-api - systemctl enable salt-api - fi - fi + ## DGM if command -v systemctl; then + ## DGM db_get salt-api/active + ## DGM RESLT=$(echo "$RET" | cut -d ' ' -f 1) + ## DGM if [ "$RESLT" != 10 ]; then + ## DGM systemctl daemon-reload + ## DGM if [ "$RESLT" = "active" ]; then + ## DGM systemctl restart salt-api + ## DGM fi + ## DGM db_get salt-api/enabled + ## DGM RESLT=$(echo "$RET" | cut -d ' ' -f 1) + ## DGM if [ "$RESLT" = "disabled" ]; then + ## DGM systemctl disable salt-api + ## DGM else + ## DGM systemctl enable salt-api + ## DGM fi + ## DGM else + ## DGM systemctl daemon-reload + ## DGM systemctl restart salt-api + ## DGM systemctl enable salt-api + ## DGM fi + ## DGM fi ;; esac diff --git a/pkg/debian/salt-master.postinst b/pkg/debian/salt-master.postinst index be7064f9bad2..2b1b003a2761 100644 --- a/pkg/debian/salt-master.postinst +++ b/pkg/debian/salt-master.postinst @@ -16,26 +16,26 @@ case "$1" in fi chown -R $RET:$RET /etc/salt/pki/master /etc/salt/master.d /var/log/salt/master /var/log/salt/key /var/cache/salt/master /var/run/salt/master fi - if command -v systemctl; then - db_get salt-master/active - RESLT=$(echo "$RET" | cut -d ' ' -f 1) - if [ "$RESLT" != 10 ]; then - systemctl daemon-reload - if [ "$RESLT" = "active" ]; then - systemctl restart salt-master - fi - db_get salt-master/enabled - RESLT=$(echo "$RET" | cut -d ' ' -f 1) - if [ "$RESLT" = "disabled" ]; then - systemctl disable salt-master - else - systemctl enable salt-master - fi - else - systemctl daemon-reload - systemctl restart salt-master - systemctl enable salt-master - fi - fi + ## DGM if command -v systemctl; then + ## DGM db_get salt-master/active + ## DGM RESLT=$(echo "$RET" | cut -d ' ' -f 1) + ## DGM if [ "$RESLT" != 10 ]; then + ## DGM systemctl daemon-reload + ## DGM if [ "$RESLT" = "active" ]; then + ## DGM systemctl restart salt-master + ## DGM fi + ## DGM db_get salt-master/enabled + ## DGM RESLT=$(echo "$RET" | cut -d ' ' -f 1) + ## DGM if [ "$RESLT" = "disabled" ]; then + ## DGM systemctl disable salt-master + ## DGM else + ## DGM systemctl enable salt-master + ## DGM fi + ## DGM else + ## DGM systemctl daemon-reload + ## DGM systemctl restart salt-master + ## DGM systemctl enable salt-master + ## DGM fi + ## DGM fi ;; esac diff --git a/pkg/debian/salt-master.preinst b/pkg/debian/salt-master.preinst index af978b8e508e..ef1a5dcf10cd 100644 --- a/pkg/debian/salt-master.preinst +++ b/pkg/debian/salt-master.preinst @@ -31,16 +31,16 @@ case "$1" in db_set salt-master/user $CUR_USER chown -R $CUR_USER:$CUR_GROUP /etc/salt/pki/master /etc/salt/master.d /var/log/salt/master \ /var/log/salt/key /var/cache/salt/master /var/run/salt/master - if command -v systemctl; then - SM_ENABLED=$(systemctl show -p UnitFileState salt-master | cut -d '=' -f 2) - db_set salt-master/enabled $SM_ENABLED - SM_ACTIVE=$(systemctl is-active salt-master) - db_set salt-master/active $SM_ACTIVE - else - db_set salt-master/enabled enabled - db_set salt-master/active active - - fi + ## DGM if command -v systemctl; then + ## DGM SM_ENABLED=$(systemctl show -p UnitFileState salt-master | cut -d '=' -f 2) + ## DGM db_set salt-master/enabled $SM_ENABLED + ## DGM SM_ACTIVE=$(systemctl is-active salt-master) + ## DGM db_set salt-master/active $SM_ACTIVE + ## DGM else + ## DGM db_set salt-master/enabled enabled + ## DGM db_set salt-master/active active + + ## DGM fi ;; esac diff --git a/pkg/debian/salt-minion.postinst b/pkg/debian/salt-minion.postinst index 13d1cf509016..bd2e90b1b69a 100644 --- a/pkg/debian/salt-minion.postinst +++ b/pkg/debian/salt-minion.postinst @@ -16,26 +16,26 @@ case "$1" in fi chown -R $RET:$RET /etc/salt/pki/minion /etc/salt/minion.d /var/log/salt/minion /var/cache/salt/minion /var/run/salt/minion fi - if command -v systemctl; then - db_get salt-minion/active - RESLT=$(echo "$RET" | cut -d ' ' -f 1) - if [ "$RESLT" != 10 ]; then - systemctl daemon-reload - if [ "$RESLT" = "active" ]; then - systemctl restart salt-minion - fi - db_get salt-minion/enabled - RESLT=$(echo "$RET" | cut -d ' ' -f 1) - if [ "$RESLT" = "disabled" ]; then - systemctl disable salt-minion - else - systemctl enable salt-minion - fi - else - systemctl daemon-reload - systemctl restart salt-minion - systemctl enable salt-minion - fi - fi + ## DGM if command -v systemctl; then + ## DGM db_get salt-minion/active + ## DGM RESLT=$(echo "$RET" | cut -d ' ' -f 1) + ## DGM if [ "$RESLT" != 10 ]; then + ## DGM systemctl daemon-reload + ## DGM if [ "$RESLT" = "active" ]; then + ## DGM systemctl restart salt-minion + ## DGM fi + ## DGM db_get salt-minion/enabled + ## DGM RESLT=$(echo "$RET" | cut -d ' ' -f 1) + ## DGM if [ "$RESLT" = "disabled" ]; then + ## DGM systemctl disable salt-minion + ## DGM else + ## DGM systemctl enable salt-minion + ## DGM fi + ## DGM else + ## DGM systemctl daemon-reload + ## DGM systemctl restart salt-minion + ## DGM systemctl enable salt-minion + ## DGM fi + ## DGM fi ;; esac diff --git a/pkg/debian/salt-minion.preinst b/pkg/debian/salt-minion.preinst index 4a4cd949c642..3538a6df740b 100644 --- a/pkg/debian/salt-minion.preinst +++ b/pkg/debian/salt-minion.preinst @@ -16,15 +16,15 @@ case "$1" in db_set salt-minion/user $CUR_USER chown -R $CUR_USER:$CUR_GROUP /etc/salt/pki/minion /etc/salt/minion.d /var/log/salt/minion \ /var/cache/salt/minion /var/run/salt/minion - if command -v systemctl; then - SM_ENABLED=$(systemctl show -p UnitFileState salt-minion | cut -d '=' -f 2) - db_set salt-minion/enabled $SM_ENABLED - SM_ACTIVE=$(systemctl is-active salt-minion) - db_set salt-minion/active $SM_ACTIVE - else - db_set salt-minion/enabled enabled - db_set salt-minion/active active + ## DGM if command -v systemctl; then + ## DGM SM_ENABLED=$(systemctl show -p UnitFileState salt-minion | cut -d '=' -f 2) + ## DGM db_set salt-minion/enabled $SM_ENABLED + ## DGM SM_ACTIVE=$(systemctl is-active salt-minion) + ## DGM db_set salt-minion/active $SM_ACTIVE + ## DGM else + ## DGM db_set salt-minion/enabled enabled + ## DGM db_set salt-minion/active active - fi + ## DGM fi ;; esac From 974bad005481164fb06c5f78013ba74d17510093 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 4 Sep 2024 10:35:28 -0600 Subject: [PATCH 64/97] Updated debian rules to not include python3 --- pkg/debian/rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/debian/rules b/pkg/debian/rules index 26d0112a5e5f..a624c2e43810 100755 --- a/pkg/debian/rules +++ b/pkg/debian/rules @@ -4,7 +4,7 @@ DH_VERBOSE = 1 .PHONY: override_dh_strip %: - dh $@ --with python3,bash-completion --with=systemd + dh $@ --with bash-completion,systemd # dh_auto_clean tries to invoke distutils causing failures. override_dh_auto_clean: From 142b7ac258ed274487730eef967593011f89f1ed Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 4 Sep 2024 14:05:27 -0600 Subject: [PATCH 65/97] Restore systemd handling in preinst and postinst for Debina/Ubuntu --- pkg/debian/salt-api.postinst | 42 ++++++++++++++++----------------- pkg/debian/salt-master.postinst | 42 ++++++++++++++++----------------- pkg/debian/salt-master.preinst | 19 +++++++-------- pkg/debian/salt-minion.postinst | 42 ++++++++++++++++----------------- pkg/debian/salt-minion.preinst | 19 +++++++-------- 5 files changed, 81 insertions(+), 83 deletions(-) diff --git a/pkg/debian/salt-api.postinst b/pkg/debian/salt-api.postinst index fecd7e5d336f..3b78211922ad 100644 --- a/pkg/debian/salt-api.postinst +++ b/pkg/debian/salt-api.postinst @@ -12,26 +12,26 @@ case "$1" in fi chown $RET:$RET /var/log/salt/api fi - ## DGM if command -v systemctl; then - ## DGM db_get salt-api/active - ## DGM RESLT=$(echo "$RET" | cut -d ' ' -f 1) - ## DGM if [ "$RESLT" != 10 ]; then - ## DGM systemctl daemon-reload - ## DGM if [ "$RESLT" = "active" ]; then - ## DGM systemctl restart salt-api - ## DGM fi - ## DGM db_get salt-api/enabled - ## DGM RESLT=$(echo "$RET" | cut -d ' ' -f 1) - ## DGM if [ "$RESLT" = "disabled" ]; then - ## DGM systemctl disable salt-api - ## DGM else - ## DGM systemctl enable salt-api - ## DGM fi - ## DGM else - ## DGM systemctl daemon-reload - ## DGM systemctl restart salt-api - ## DGM systemctl enable salt-api - ## DGM fi - ## DGM fi + if command -v systemctl; then + db_get salt-api/active + RESLT=$(echo "$RET" | cut -d ' ' -f 1) + if [ "$RESLT" != 10 ]; then + systemctl daemon-reload + if [ "$RESLT" = "active" ]; then + systemctl restart salt-api + fi + db_get salt-api/enabled + RESLT=$(echo "$RET" | cut -d ' ' -f 1) + if [ "$RESLT" = "disabled" ]; then + systemctl disable salt-api + else + systemctl enable salt-api + fi + else + systemctl daemon-reload + systemctl restart salt-api + systemctl enable salt-api + fi + fi ;; esac diff --git a/pkg/debian/salt-master.postinst b/pkg/debian/salt-master.postinst index 2b1b003a2761..be7064f9bad2 100644 --- a/pkg/debian/salt-master.postinst +++ b/pkg/debian/salt-master.postinst @@ -16,26 +16,26 @@ case "$1" in fi chown -R $RET:$RET /etc/salt/pki/master /etc/salt/master.d /var/log/salt/master /var/log/salt/key /var/cache/salt/master /var/run/salt/master fi - ## DGM if command -v systemctl; then - ## DGM db_get salt-master/active - ## DGM RESLT=$(echo "$RET" | cut -d ' ' -f 1) - ## DGM if [ "$RESLT" != 10 ]; then - ## DGM systemctl daemon-reload - ## DGM if [ "$RESLT" = "active" ]; then - ## DGM systemctl restart salt-master - ## DGM fi - ## DGM db_get salt-master/enabled - ## DGM RESLT=$(echo "$RET" | cut -d ' ' -f 1) - ## DGM if [ "$RESLT" = "disabled" ]; then - ## DGM systemctl disable salt-master - ## DGM else - ## DGM systemctl enable salt-master - ## DGM fi - ## DGM else - ## DGM systemctl daemon-reload - ## DGM systemctl restart salt-master - ## DGM systemctl enable salt-master - ## DGM fi - ## DGM fi + if command -v systemctl; then + db_get salt-master/active + RESLT=$(echo "$RET" | cut -d ' ' -f 1) + if [ "$RESLT" != 10 ]; then + systemctl daemon-reload + if [ "$RESLT" = "active" ]; then + systemctl restart salt-master + fi + db_get salt-master/enabled + RESLT=$(echo "$RET" | cut -d ' ' -f 1) + if [ "$RESLT" = "disabled" ]; then + systemctl disable salt-master + else + systemctl enable salt-master + fi + else + systemctl daemon-reload + systemctl restart salt-master + systemctl enable salt-master + fi + fi ;; esac diff --git a/pkg/debian/salt-master.preinst b/pkg/debian/salt-master.preinst index ef1a5dcf10cd..a96f9dd67678 100644 --- a/pkg/debian/salt-master.preinst +++ b/pkg/debian/salt-master.preinst @@ -31,16 +31,15 @@ case "$1" in db_set salt-master/user $CUR_USER chown -R $CUR_USER:$CUR_GROUP /etc/salt/pki/master /etc/salt/master.d /var/log/salt/master \ /var/log/salt/key /var/cache/salt/master /var/run/salt/master - ## DGM if command -v systemctl; then - ## DGM SM_ENABLED=$(systemctl show -p UnitFileState salt-master | cut -d '=' -f 2) - ## DGM db_set salt-master/enabled $SM_ENABLED - ## DGM SM_ACTIVE=$(systemctl is-active salt-master) - ## DGM db_set salt-master/active $SM_ACTIVE - ## DGM else - ## DGM db_set salt-master/enabled enabled - ## DGM db_set salt-master/active active - - ## DGM fi + if command -v systemctl; then + SM_ENABLED=$(systemctl show -p UnitFileState salt-master | cut -d '=' -f 2) + db_set salt-master/enabled $SM_ENABLED + SM_ACTIVE=$(systemctl is-active salt-master) + db_set salt-master/active $SM_ACTIVE + else + db_set salt-master/enabled enabled + db_set salt-master/active active + fi ;; esac diff --git a/pkg/debian/salt-minion.postinst b/pkg/debian/salt-minion.postinst index bd2e90b1b69a..13d1cf509016 100644 --- a/pkg/debian/salt-minion.postinst +++ b/pkg/debian/salt-minion.postinst @@ -16,26 +16,26 @@ case "$1" in fi chown -R $RET:$RET /etc/salt/pki/minion /etc/salt/minion.d /var/log/salt/minion /var/cache/salt/minion /var/run/salt/minion fi - ## DGM if command -v systemctl; then - ## DGM db_get salt-minion/active - ## DGM RESLT=$(echo "$RET" | cut -d ' ' -f 1) - ## DGM if [ "$RESLT" != 10 ]; then - ## DGM systemctl daemon-reload - ## DGM if [ "$RESLT" = "active" ]; then - ## DGM systemctl restart salt-minion - ## DGM fi - ## DGM db_get salt-minion/enabled - ## DGM RESLT=$(echo "$RET" | cut -d ' ' -f 1) - ## DGM if [ "$RESLT" = "disabled" ]; then - ## DGM systemctl disable salt-minion - ## DGM else - ## DGM systemctl enable salt-minion - ## DGM fi - ## DGM else - ## DGM systemctl daemon-reload - ## DGM systemctl restart salt-minion - ## DGM systemctl enable salt-minion - ## DGM fi - ## DGM fi + if command -v systemctl; then + db_get salt-minion/active + RESLT=$(echo "$RET" | cut -d ' ' -f 1) + if [ "$RESLT" != 10 ]; then + systemctl daemon-reload + if [ "$RESLT" = "active" ]; then + systemctl restart salt-minion + fi + db_get salt-minion/enabled + RESLT=$(echo "$RET" | cut -d ' ' -f 1) + if [ "$RESLT" = "disabled" ]; then + systemctl disable salt-minion + else + systemctl enable salt-minion + fi + else + systemctl daemon-reload + systemctl restart salt-minion + systemctl enable salt-minion + fi + fi ;; esac diff --git a/pkg/debian/salt-minion.preinst b/pkg/debian/salt-minion.preinst index 3538a6df740b..51be48e0677f 100644 --- a/pkg/debian/salt-minion.preinst +++ b/pkg/debian/salt-minion.preinst @@ -16,15 +16,14 @@ case "$1" in db_set salt-minion/user $CUR_USER chown -R $CUR_USER:$CUR_GROUP /etc/salt/pki/minion /etc/salt/minion.d /var/log/salt/minion \ /var/cache/salt/minion /var/run/salt/minion - ## DGM if command -v systemctl; then - ## DGM SM_ENABLED=$(systemctl show -p UnitFileState salt-minion | cut -d '=' -f 2) - ## DGM db_set salt-minion/enabled $SM_ENABLED - ## DGM SM_ACTIVE=$(systemctl is-active salt-minion) - ## DGM db_set salt-minion/active $SM_ACTIVE - ## DGM else - ## DGM db_set salt-minion/enabled enabled - ## DGM db_set salt-minion/active active - - ## DGM fi + if command -v systemctl; then + SM_ENABLED=$(systemctl show -p UnitFileState salt-minion | cut -d '=' -f 2) + db_set salt-minion/enabled $SM_ENABLED + SM_ACTIVE=$(systemctl is-active salt-minion) + db_set salt-minion/active $SM_ACTIVE + else + db_set salt-minion/enabled enabled + db_set salt-minion/active active + fi ;; esac From 0483e44f195df4443d175c35eb23713688401c6f Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 12 Sep 2024 11:22:31 -0600 Subject: [PATCH 66/97] Updated symbolic links for bash completion as per original issuer comments --- pkg/debian/salt-common.links | 1 + pkg/debian/salt-master.links | 3 +++ 2 files changed, 4 insertions(+) diff --git a/pkg/debian/salt-common.links b/pkg/debian/salt-common.links index 29d5cb3e7e0f..cddd400ceeb0 100644 --- a/pkg/debian/salt-common.links +++ b/pkg/debian/salt-common.links @@ -6,3 +6,4 @@ salt-common: manpage-has-errors-from-man opt/saltstack/salt/salt-pip /usr/bin/salt-pip opt/saltstack/salt/salt-call /usr/bin/salt-call +usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt-call diff --git a/pkg/debian/salt-master.links b/pkg/debian/salt-master.links index e6c0ef2446ad..77c8bdc67b29 100644 --- a/pkg/debian/salt-master.links +++ b/pkg/debian/salt-master.links @@ -4,3 +4,6 @@ opt/saltstack/salt/salt-cp /usr/bin/salt-cp opt/saltstack/salt/salt-key /usr/bin/salt-key opt/saltstack/salt/salt-run /usr/bin/salt-run opt/saltstack/salt/spm /usr/bin/spm +usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt +usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt-cp +usr/share/bash-completion/completions/salt-common usr/share/bash-completion/completions/salt-key From 94fdf082ecd574373d9920f21fd8eec75663a75c Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 17 Sep 2024 12:20:53 -0600 Subject: [PATCH 67/97] Adjust salt-common.install for bash-completion --- pkg/debian/salt-common.install | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/debian/salt-common.install b/pkg/debian/salt-common.install index 372cf029a08b..63f1d5a1287a 100644 --- a/pkg/debian/salt-common.install +++ b/pkg/debian/salt-common.install @@ -10,4 +10,3 @@ pkg/common/fish-completions/salt-key.fish /usr/share/fish/vendor_completions.d pkg/common/fish-completions/salt-master.fish /usr/share/fish/vendor_completions.d pkg/common/fish-completions/salt-run.fish /usr/share/fish/vendor_completions.d pkg/common/fish-completions/salt.fish /usr/share/fish/vendor_completions.d -pkg/debian/salt-common.bash-completion /usr/share/bash-completion/completions/salt-common From c1dd74aa69031213f16b56fe4f9e5a5fb71429ef Mon Sep 17 00:00:00 2001 From: Imran Iqbal Date: Mon, 26 Aug 2024 18:00:18 +0100 Subject: [PATCH 68/97] docs(modules/system): fix `system.get_computer_name` CLI example --- salt/modules/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/system.py b/salt/modules/system.py index bcdce7a434c6..c2bd8f8da78c 100644 --- a/salt/modules/system.py +++ b/salt/modules/system.py @@ -633,7 +633,7 @@ def get_computer_name(): .. code-block:: bash - salt '*' network.get_hostname + salt '*' system.get_computer_name """ return __salt__["network.get_hostname"]() From 6c3365f67cd294d2ae7077fa17ed9012acebaaa4 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 7 Aug 2024 13:30:04 -0500 Subject: [PATCH 69/97] Move grain_opts to subdict to prevent overwriting core grains --- salt/grains/opts.py | 2 +- tests/pytests/unit/grains/test_opts.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 tests/pytests/unit/grains/test_opts.py diff --git a/salt/grains/opts.py b/salt/grains/opts.py index c014f484bcbf..3d63d41b7e9f 100644 --- a/salt/grains/opts.py +++ b/salt/grains/opts.py @@ -11,5 +11,5 @@ def opts(): if __opts__.get("grain_opts", False) or ( isinstance(__pillar__, dict) and __pillar__.get("grain_opts", False) ): - return __opts__ + return {"opts": __opts__} return {} diff --git a/tests/pytests/unit/grains/test_opts.py b/tests/pytests/unit/grains/test_opts.py new file mode 100644 index 000000000000..35fc8a06b207 --- /dev/null +++ b/tests/pytests/unit/grains/test_opts.py @@ -0,0 +1,20 @@ +""" +tests.pytests.unit.grains.test_opts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +""" + +import salt.grains.opts as opts +from tests.support.mock import patch + + +def test_grain_opts_does_not_overwrite_core_grains(tmp_path): + """ + Tests that enabling grain_opts doesn't overwrite the core grains + + See: https://github.com/saltstack/salt/issues/66784 + """ + dunder_opts = {"grain_opts": True} + + with patch.object(opts, "__opts__", dunder_opts, create=True): + with patch.object(opts, "__pillar__", {}, create=True): + assert opts.opts() == {"opts": dunder_opts} From d4b295719499f182d8c12cc731d0d4078dcf761b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 7 Aug 2024 13:38:00 -0500 Subject: [PATCH 70/97] Add changelog file --- changelog/66784.fixed.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog/66784.fixed.md diff --git a/changelog/66784.fixed.md b/changelog/66784.fixed.md new file mode 100644 index 000000000000..afe9df887a54 --- /dev/null +++ b/changelog/66784.fixed.md @@ -0,0 +1,2 @@ +Fixed an issue where enabling `grain_opts` in the minion config would cause +some core grains to be overwritten. From 748943cd59790258977405c78227dbae742e9ca8 Mon Sep 17 00:00:00 2001 From: sunxingboo Date: Fri, 9 Aug 2024 08:35:13 +0800 Subject: [PATCH 71/97] Corrected the description of the two parameters mkey and key in the MWorker constructor. --- salt/master.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/master.py b/salt/master.py index e6ba5f1d8d8c..fcf8bab0c349 100644 --- a/salt/master.py +++ b/salt/master.py @@ -950,8 +950,8 @@ def __init__(self, opts, mkey, key, req_channels, **kwargs): Create a salt master worker process :param dict opts: The salt options - :param dict mkey: The user running the salt master and the AES key - :param dict key: The user running the salt master and the RSA key + :param dict mkey: The user running the salt master and the RSA key + :param dict key: The user running the salt master and the AES key :rtype: MWorker :return: Master worker From 698979fe55c0582ccc7369644a3a5dd91166ab4f Mon Sep 17 00:00:00 2001 From: hurzhurz Date: Thu, 1 Aug 2024 11:48:28 +0000 Subject: [PATCH 72/97] fix nacl.keygen for not yet existing sk_file or pk_file --- changelog/66772.fixed.md | 1 + salt/utils/nacl.py | 8 ++++---- tests/pytests/unit/utils/test_nacl.py | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 changelog/66772.fixed.md diff --git a/changelog/66772.fixed.md b/changelog/66772.fixed.md new file mode 100644 index 000000000000..2f9f40ee523d --- /dev/null +++ b/changelog/66772.fixed.md @@ -0,0 +1 @@ +Fixed nacl.keygen for not yet existing sk_file or pk_file diff --git a/salt/utils/nacl.py b/salt/utils/nacl.py index cac3455d1a68..63d97e6f9abb 100644 --- a/salt/utils/nacl.py +++ b/salt/utils/nacl.py @@ -182,12 +182,12 @@ def keygen(sk_file=None, pk_file=None, **kwargs): with salt.utils.files.fopen(sk_file, "rb") as keyf: sk = salt.utils.stringutils.to_unicode(keyf.read()).rstrip("\n") sk = base64.b64decode(sk) - kp = nacl.public.PublicKey(sk) + kp = nacl.public.PrivateKey(sk) with salt.utils.files.fopen(pk_file, "wb") as keyf: - keyf.write(base64.b64encode(kp.encode())) + keyf.write(base64.b64encode(kp.public_key.encode())) return f"saved pk_file: {pk_file}" - kp = nacl.public.PublicKey.generate() + kp = nacl.public.PrivateKey.generate() with salt.utils.files.fopen(sk_file, "wb") as keyf: keyf.write(base64.b64encode(kp.encode())) if salt.utils.platform.is_windows(): @@ -200,7 +200,7 @@ def keygen(sk_file=None, pk_file=None, **kwargs): # chmod 0600 file os.chmod(sk_file, 1536) with salt.utils.files.fopen(pk_file, "wb") as keyf: - keyf.write(base64.b64encode(kp.encode())) + keyf.write(base64.b64encode(kp.public_key.encode())) return f"saved sk_file:{sk_file} pk_file: {pk_file}" diff --git a/tests/pytests/unit/utils/test_nacl.py b/tests/pytests/unit/utils/test_nacl.py index 5c60d880b2fd..91be6855487b 100644 --- a/tests/pytests/unit/utils/test_nacl.py +++ b/tests/pytests/unit/utils/test_nacl.py @@ -73,6 +73,20 @@ def test_keygen_keyfile(test_keygen): ret = nacl.keygen(keyfile=fpath) assert f"saved pk_file: {fpath}.pub" == ret + with salt.utils.files.fopen(str(fpath) + ".pub", "rb") as rfh: + assert test_keygen["pk"] == rfh.read() + salt.utils.files.remove(str(fpath) + ".pub") + + +def test_keygen_nonexistent_sk_file(): + """ + test nacl.keygen function + with nonexistent/new sk_file + """ + with pytest.helpers.temp_file("test_keygen_sk_file") as fpath: + salt.utils.files.remove(str(fpath)) + ret = nacl.keygen(sk_file=str(fpath)) + assert f"saved sk_file:{fpath} pk_file: {fpath}.pub" == ret salt.utils.files.remove(str(fpath) + ".pub") From 059d1e0269af2450bcc0689231023a2b445abe98 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 6 Sep 2024 10:48:29 -0600 Subject: [PATCH 73/97] Testing previous version 4.3.6 instead of latest 4.4.0 for action/upload-artifact/merge --- .github/workflows/ci.yml | 3 ++- .github/workflows/nightly.yml | 3 ++- .github/workflows/scheduled.yml | 3 ++- .github/workflows/templates/ci.yml.jinja | 3 ++- .github/workflows/test-action-linux.yml | 9 ++++++--- .github/workflows/test-action-macos.yml | 9 ++++++--- .github/workflows/test-action-windows.yml | 9 ++++++--- .github/workflows/test-packages-action-macos.yml | 3 ++- .github/workflows/test-packages-action-windows.yml | 3 ++- 9 files changed, 30 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e88f05b4a24..c9b82e509333 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2016,7 +2016,8 @@ jobs: - name: Merge All Code Coverage Test Run Artifacts continue-on-error: true - uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4 + uses: actions/upload-artifact/merge@v4.3.6 with: name: all-testrun-coverage-artifacts pattern: all-testrun-coverage-artifacts-* diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d7bb49730019..824b75a40203 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2078,7 +2078,8 @@ jobs: - name: Merge All Code Coverage Test Run Artifacts continue-on-error: true - uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4 + uses: actions/upload-artifact/merge@v4.3.6 with: name: all-testrun-coverage-artifacts pattern: all-testrun-coverage-artifacts-* diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 86e57fb71a42..15c0f111ddef 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -2055,7 +2055,8 @@ jobs: - name: Merge All Code Coverage Test Run Artifacts continue-on-error: true - uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4 + uses: actions/upload-artifact/merge@v4.3.6 with: name: all-testrun-coverage-artifacts pattern: all-testrun-coverage-artifacts-* diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index 91713863f187..1597ddff9e29 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -359,7 +359,8 @@ - name: Merge All Code Coverage Test Run Artifacts continue-on-error: true - uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4 + uses: actions/upload-artifact/merge@v4.3.6 with: name: all-testrun-coverage-artifacts pattern: all-testrun-coverage-artifacts-* diff --git a/.github/workflows/test-action-linux.yml b/.github/workflows/test-action-linux.yml index 38a743944033..6db00f9b3b50 100644 --- a/.github/workflows/test-action-linux.yml +++ b/.github/workflows/test-action-linux.yml @@ -328,7 +328,8 @@ jobs: - name: Merge JUnit XML Test Run Artifacts if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true - uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4 + uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* @@ -338,7 +339,8 @@ jobs: - name: Merge Log Test Run Artifacts if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true - uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4 + uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} pattern: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* @@ -348,7 +350,8 @@ jobs: - name: Merge Code Coverage Test Run Artifacts if: ${{ inputs.skip-code-coverage == false }} continue-on-error: true - uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4 + uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index 1e3a873a13c2..6a767e4bfbeb 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -358,7 +358,8 @@ jobs: - name: Merge JUnit XML Test Run Artifacts if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true - uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4 + uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* @@ -368,7 +369,8 @@ jobs: - name: Merge Log Test Run Artifacts if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true - uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4 + uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* @@ -378,7 +380,8 @@ jobs: - name: Merge Code Coverage Test Run Artifacts if: ${{ inputs.skip-code-coverage == false }} continue-on-error: true - uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4 + uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* diff --git a/.github/workflows/test-action-windows.yml b/.github/workflows/test-action-windows.yml index 385b6eed95f6..231c01f177c8 100644 --- a/.github/workflows/test-action-windows.yml +++ b/.github/workflows/test-action-windows.yml @@ -329,7 +329,8 @@ jobs: - name: Merge JUnit XML Test Run Artifacts if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true - uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4 + uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* @@ -339,7 +340,8 @@ jobs: - name: Merge Log Test Run Artifacts if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true - uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4 + uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* @@ -349,7 +351,8 @@ jobs: - name: Merge Code Coverage Test Run Artifacts if: ${{ inputs.skip-code-coverage == false }} continue-on-error: true - uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4 + uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* diff --git a/.github/workflows/test-packages-action-macos.yml b/.github/workflows/test-packages-action-macos.yml index aca0b4cc244c..d4aacfc07704 100644 --- a/.github/workflows/test-packages-action-macos.yml +++ b/.github/workflows/test-packages-action-macos.yml @@ -239,7 +239,8 @@ jobs: - name: Merge Test Run Artifacts continue-on-error: true - uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4 + uses: actions/upload-artifact/merge@v4.3.6 with: name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }} pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-* diff --git a/.github/workflows/test-packages-action-windows.yml b/.github/workflows/test-packages-action-windows.yml index 985ff96de820..ece1efd6f888 100644 --- a/.github/workflows/test-packages-action-windows.yml +++ b/.github/workflows/test-packages-action-windows.yml @@ -246,7 +246,8 @@ jobs: t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t" - name: Merge Test Run Artifacts - uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4 + uses: actions/upload-artifact/merge@v4.3.6 continue-on-error: true with: name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }} From 9cef9a6ac00b4540b50e8d594309033ec18cf631 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 9 Sep 2024 13:13:50 -0600 Subject: [PATCH 74/97] Revert to using v4 and alter pattern (leave off dash) --- .github/workflows/ci.yml | 7 ++++--- .github/workflows/nightly.yml | 7 ++++--- .github/workflows/scheduled.yml | 7 ++++--- .github/workflows/templates/ci.yml.jinja | 7 ++++--- .github/workflows/test-action-linux.yml | 21 +++++++++++-------- .github/workflows/test-action-macos.yml | 18 +++++++++------- .github/workflows/test-action-windows.yml | 21 +++++++++++-------- .../workflows/test-packages-action-macos.yml | 7 ++++--- .../test-packages-action-windows.yml | 7 ++++--- 9 files changed, 58 insertions(+), 44 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9b82e509333..c1daf82f53fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2016,11 +2016,12 @@ jobs: - name: Merge All Code Coverage Test Run Artifacts continue-on-error: true - ## DGM uses: actions/upload-artifact/merge@v4 - uses: actions/upload-artifact/merge@v4.3.6 + uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: all-testrun-coverage-artifacts - pattern: all-testrun-coverage-artifacts-* + ## DGM pattern: all-testrun-coverage-artifacts-* + pattern: all-testrun-coverage-artifacts* separate-directories: false delete-merged: true diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 824b75a40203..86c109aeb822 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2078,11 +2078,12 @@ jobs: - name: Merge All Code Coverage Test Run Artifacts continue-on-error: true - ## DGM uses: actions/upload-artifact/merge@v4 - uses: actions/upload-artifact/merge@v4.3.6 + uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: all-testrun-coverage-artifacts - pattern: all-testrun-coverage-artifacts-* + ## DGM pattern: all-testrun-coverage-artifacts-* + pattern: all-testrun-coverage-artifacts* separate-directories: false delete-merged: true diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 15c0f111ddef..94c9dd7cdede 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -2055,11 +2055,12 @@ jobs: - name: Merge All Code Coverage Test Run Artifacts continue-on-error: true - ## DGM uses: actions/upload-artifact/merge@v4 - uses: actions/upload-artifact/merge@v4.3.6 + uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: all-testrun-coverage-artifacts - pattern: all-testrun-coverage-artifacts-* + ## DGM pattern: all-testrun-coverage-artifacts-* + pattern: all-testrun-coverage-artifacts* separate-directories: false delete-merged: true diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index 1597ddff9e29..b4fdc783cee2 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -359,11 +359,12 @@ - name: Merge All Code Coverage Test Run Artifacts continue-on-error: true - ## DGM uses: actions/upload-artifact/merge@v4 - uses: actions/upload-artifact/merge@v4.3.6 + uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: all-testrun-coverage-artifacts - pattern: all-testrun-coverage-artifacts-* + ## DGM pattern: all-testrun-coverage-artifacts-* + pattern: all-testrun-coverage-artifacts* separate-directories: false delete-merged: true diff --git a/.github/workflows/test-action-linux.yml b/.github/workflows/test-action-linux.yml index 6db00f9b3b50..28695ab91d90 100644 --- a/.github/workflows/test-action-linux.yml +++ b/.github/workflows/test-action-linux.yml @@ -328,33 +328,36 @@ jobs: - name: Merge JUnit XML Test Run Artifacts if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true - ## DGM uses: actions/upload-artifact/merge@v4 - uses: actions/upload-artifact/merge@v4.3.6 + uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} - pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* + ## DGM pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* + pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}* separate-directories: false delete-merged: true - name: Merge Log Test Run Artifacts if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true - ## DGM uses: actions/upload-artifact/merge@v4 - uses: actions/upload-artifact/merge@v4.3.6 + uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} - pattern: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* + ## DGM pattern: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* + pattern: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}* separate-directories: false delete-merged: true - name: Merge Code Coverage Test Run Artifacts if: ${{ inputs.skip-code-coverage == false }} continue-on-error: true - ## DGM uses: actions/upload-artifact/merge@v4 - uses: actions/upload-artifact/merge@v4.3.6 + uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} - pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* + ## DGM pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* + pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}* separate-directories: false delete-merged: true diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index 6a767e4bfbeb..7c1b04d3de7f 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -358,8 +358,8 @@ jobs: - name: Merge JUnit XML Test Run Artifacts if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true - ## DGM uses: actions/upload-artifact/merge@v4 - uses: actions/upload-artifact/merge@v4.3.6 + uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* @@ -369,22 +369,24 @@ jobs: - name: Merge Log Test Run Artifacts if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true - ## DGM uses: actions/upload-artifact/merge@v4 - uses: actions/upload-artifact/merge@v4.3.6 + uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + ## DGM pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* separate-directories: false delete-merged: true - name: Merge Code Coverage Test Run Artifacts if: ${{ inputs.skip-code-coverage == false }} continue-on-error: true - ## DGM uses: actions/upload-artifact/merge@v4 - uses: actions/upload-artifact/merge@v4.3.6 + uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + ## DGM pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* separate-directories: false delete-merged: true diff --git a/.github/workflows/test-action-windows.yml b/.github/workflows/test-action-windows.yml index 231c01f177c8..2188d3707d99 100644 --- a/.github/workflows/test-action-windows.yml +++ b/.github/workflows/test-action-windows.yml @@ -329,33 +329,36 @@ jobs: - name: Merge JUnit XML Test Run Artifacts if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true - ## DGM uses: actions/upload-artifact/merge@v4 - uses: actions/upload-artifact/merge@v4.3.6 + uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + ## DGM pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* separate-directories: false delete-merged: true - name: Merge Log Test Run Artifacts if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true - ## DGM uses: actions/upload-artifact/merge@v4 - uses: actions/upload-artifact/merge@v4.3.6 + uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + ## DGM pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* separate-directories: false delete-merged: true - name: Merge Code Coverage Test Run Artifacts if: ${{ inputs.skip-code-coverage == false }} continue-on-error: true - ## DGM uses: actions/upload-artifact/merge@v4 - uses: actions/upload-artifact/merge@v4.3.6 + uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + ## DGM pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* separate-directories: false delete-merged: true diff --git a/.github/workflows/test-packages-action-macos.yml b/.github/workflows/test-packages-action-macos.yml index d4aacfc07704..4a147ae13d41 100644 --- a/.github/workflows/test-packages-action-macos.yml +++ b/.github/workflows/test-packages-action-macos.yml @@ -239,11 +239,12 @@ jobs: - name: Merge Test Run Artifacts continue-on-error: true - ## DGM uses: actions/upload-artifact/merge@v4 - uses: actions/upload-artifact/merge@v4.3.6 + uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }} - pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-* + ## DGM pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-* + pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}* separate-directories: true delete-merged: true diff --git a/.github/workflows/test-packages-action-windows.yml b/.github/workflows/test-packages-action-windows.yml index ece1efd6f888..e6592b9fde8d 100644 --- a/.github/workflows/test-packages-action-windows.yml +++ b/.github/workflows/test-packages-action-windows.yml @@ -246,12 +246,13 @@ jobs: t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t" - name: Merge Test Run Artifacts - ## DGM uses: actions/upload-artifact/merge@v4 - uses: actions/upload-artifact/merge@v4.3.6 + uses: actions/upload-artifact/merge@v4 + ## DGM uses: actions/upload-artifact/merge@v4.3.6 continue-on-error: true with: name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }} - pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-* + ## DGM pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-* + pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}* separate-directories: true delete-merged: true From 3827456452a866ceba72fed61dac0be60c52dd6c Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 10 Sep 2024 14:45:22 -0600 Subject: [PATCH 75/97] removing commented out 4.3.6, and added debugging --- .github/workflows/ci.yml | 4 +--- .github/workflows/nightly.yml | 4 +--- .github/workflows/scheduled.yml | 4 +--- .github/workflows/templates/ci.yml.jinja | 4 +--- .github/workflows/test-action-linux.yml | 5 ++--- .github/workflows/test-action-macos.yml | 3 --- .github/workflows/test-action-windows.yml | 3 --- .github/workflows/test-packages-action-macos.yml | 1 - .github/workflows/test-packages-action-windows.yml | 1 - 9 files changed, 6 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1daf82f53fe..4e88f05b4a24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2017,11 +2017,9 @@ jobs: - name: Merge All Code Coverage Test Run Artifacts continue-on-error: true uses: actions/upload-artifact/merge@v4 - ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: all-testrun-coverage-artifacts - ## DGM pattern: all-testrun-coverage-artifacts-* - pattern: all-testrun-coverage-artifacts* + pattern: all-testrun-coverage-artifacts-* separate-directories: false delete-merged: true diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 86c109aeb822..d7bb49730019 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2079,11 +2079,9 @@ jobs: - name: Merge All Code Coverage Test Run Artifacts continue-on-error: true uses: actions/upload-artifact/merge@v4 - ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: all-testrun-coverage-artifacts - ## DGM pattern: all-testrun-coverage-artifacts-* - pattern: all-testrun-coverage-artifacts* + pattern: all-testrun-coverage-artifacts-* separate-directories: false delete-merged: true diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 94c9dd7cdede..86e57fb71a42 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -2056,11 +2056,9 @@ jobs: - name: Merge All Code Coverage Test Run Artifacts continue-on-error: true uses: actions/upload-artifact/merge@v4 - ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: all-testrun-coverage-artifacts - ## DGM pattern: all-testrun-coverage-artifacts-* - pattern: all-testrun-coverage-artifacts* + pattern: all-testrun-coverage-artifacts-* separate-directories: false delete-merged: true diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index b4fdc783cee2..91713863f187 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -360,11 +360,9 @@ - name: Merge All Code Coverage Test Run Artifacts continue-on-error: true uses: actions/upload-artifact/merge@v4 - ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: all-testrun-coverage-artifacts - ## DGM pattern: all-testrun-coverage-artifacts-* - pattern: all-testrun-coverage-artifacts* + pattern: all-testrun-coverage-artifacts-* separate-directories: false delete-merged: true diff --git a/.github/workflows/test-action-linux.yml b/.github/workflows/test-action-linux.yml index 28695ab91d90..d671aa26c90b 100644 --- a/.github/workflows/test-action-linux.yml +++ b/.github/workflows/test-action-linux.yml @@ -276,6 +276,8 @@ jobs: if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}${{ inputs.fips && '.fips' || '' }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }}.grp${{ matrix.test-group || '1' }} fi + # DGM + tree -a artifacts - name: Destroy VM if: always() @@ -329,7 +331,6 @@ jobs: if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true uses: actions/upload-artifact/merge@v4 - ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} ## DGM pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* @@ -341,7 +342,6 @@ jobs: if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true uses: actions/upload-artifact/merge@v4 - ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} ## DGM pattern: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* @@ -353,7 +353,6 @@ jobs: if: ${{ inputs.skip-code-coverage == false }} continue-on-error: true uses: actions/upload-artifact/merge@v4 - ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} ## DGM pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index 7c1b04d3de7f..434256aa058f 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -359,7 +359,6 @@ jobs: if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true uses: actions/upload-artifact/merge@v4 - ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* @@ -370,7 +369,6 @@ jobs: if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true uses: actions/upload-artifact/merge@v4 - ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} ## DGM pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* @@ -382,7 +380,6 @@ jobs: if: ${{ inputs.skip-code-coverage == false }} continue-on-error: true uses: actions/upload-artifact/merge@v4 - ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} ## DGM pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* diff --git a/.github/workflows/test-action-windows.yml b/.github/workflows/test-action-windows.yml index 2188d3707d99..a1ecbe2ef132 100644 --- a/.github/workflows/test-action-windows.yml +++ b/.github/workflows/test-action-windows.yml @@ -330,7 +330,6 @@ jobs: if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true uses: actions/upload-artifact/merge@v4 - ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} ## DGM pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* @@ -342,7 +341,6 @@ jobs: if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' continue-on-error: true uses: actions/upload-artifact/merge@v4 - ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} ## DGM pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* @@ -354,7 +352,6 @@ jobs: if: ${{ inputs.skip-code-coverage == false }} continue-on-error: true uses: actions/upload-artifact/merge@v4 - ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} ## DGM pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* diff --git a/.github/workflows/test-packages-action-macos.yml b/.github/workflows/test-packages-action-macos.yml index 4a147ae13d41..df3696cce17f 100644 --- a/.github/workflows/test-packages-action-macos.yml +++ b/.github/workflows/test-packages-action-macos.yml @@ -240,7 +240,6 @@ jobs: - name: Merge Test Run Artifacts continue-on-error: true uses: actions/upload-artifact/merge@v4 - ## DGM uses: actions/upload-artifact/merge@v4.3.6 with: name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }} ## DGM pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-* diff --git a/.github/workflows/test-packages-action-windows.yml b/.github/workflows/test-packages-action-windows.yml index e6592b9fde8d..c91dc5973a3c 100644 --- a/.github/workflows/test-packages-action-windows.yml +++ b/.github/workflows/test-packages-action-windows.yml @@ -247,7 +247,6 @@ jobs: - name: Merge Test Run Artifacts uses: actions/upload-artifact/merge@v4 - ## DGM uses: actions/upload-artifact/merge@v4.3.6 continue-on-error: true with: name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }} From d77a0d5fc906c38859213559fa7c4572433967c9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 10 Sep 2024 16:55:51 -0600 Subject: [PATCH 76/97] Trying including hidden files for coverage on Linux - Test --- .github/workflows/test-action-linux.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-action-linux.yml b/.github/workflows/test-action-linux.yml index d671aa26c90b..60c802c0a837 100644 --- a/.github/workflows/test-action-linux.yml +++ b/.github/workflows/test-action-linux.yml @@ -291,6 +291,8 @@ jobs: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }} path: | artifacts/coverage/ + # DGM + include-hidden-files: true - name: Upload JUnit XML Test Run Artifacts if: always() && steps.download-artifacts-from-vm.outcome == 'success' From ac09c433b34740a3e56752399010febd162d5845 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Sep 2024 13:26:09 -0600 Subject: [PATCH 77/97] Trying other debug methods --- .github/workflows/ci.yml | 3 ++- .github/workflows/nightly.yml | 3 ++- .github/workflows/scheduled.yml | 3 ++- .github/workflows/templates/ci.yml.jinja | 3 ++- .github/workflows/test-action-macos.yml | 5 ++++- .github/workflows/test-action-windows.yml | 2 ++ .github/workflows/test-packages-action-linux.yml | 3 ++- 7 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e88f05b4a24..392d13294311 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2019,7 +2019,8 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: all-testrun-coverage-artifacts - pattern: all-testrun-coverage-artifacts-* + ## DGM pattern: all-testrun-coverage-artifacts-* + pattern: all-testrun-coverage-artifacts* separate-directories: false delete-merged: true diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d7bb49730019..5033f37a7940 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2081,7 +2081,8 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: all-testrun-coverage-artifacts - pattern: all-testrun-coverage-artifacts-* + ## DGM pattern: all-testrun-coverage-artifacts-* + pattern: all-testrun-coverage-artifacts* separate-directories: false delete-merged: true diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 86e57fb71a42..6fd4c1b15ee3 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -2058,7 +2058,8 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: all-testrun-coverage-artifacts - pattern: all-testrun-coverage-artifacts-* + ## DGM pattern: all-testrun-coverage-artifacts-* + pattern: all-testrun-coverage-artifacts* separate-directories: false delete-merged: true diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index 91713863f187..a035888a2430 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -362,7 +362,8 @@ uses: actions/upload-artifact/merge@v4 with: name: all-testrun-coverage-artifacts - pattern: all-testrun-coverage-artifacts-* + ## DGM pattern: all-testrun-coverage-artifacts-* + pattern: all-testrun-coverage-artifacts* separate-directories: false delete-merged: true diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index 434256aa058f..a65dc59118b8 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -319,6 +319,8 @@ jobs: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ env.TIMESTAMP }} path: | artifacts/coverage/ + # DGM + include-hidden-files: true - name: Upload JUnit XML Test Run Artifacts if: always() && steps.download-artifacts-from-vm.outcome == 'success' @@ -361,7 +363,8 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + ## DGM pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* separate-directories: false delete-merged: true diff --git a/.github/workflows/test-action-windows.yml b/.github/workflows/test-action-windows.yml index a1ecbe2ef132..192f73a1b5c4 100644 --- a/.github/workflows/test-action-windows.yml +++ b/.github/workflows/test-action-windows.yml @@ -289,6 +289,8 @@ jobs: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }} path: | artifacts/coverage/ + # DGM + include-hidden-files: true - name: Upload JUnit XML Test Run Artifacts if: always() && steps.download-artifacts-from-vm.outcome == 'success' diff --git a/.github/workflows/test-packages-action-linux.yml b/.github/workflows/test-packages-action-linux.yml index 36b4d7818d47..dfde25f51866 100644 --- a/.github/workflows/test-packages-action-linux.yml +++ b/.github/workflows/test-packages-action-linux.yml @@ -251,7 +251,8 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: pkg-testrun-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.pkg-type }} - pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.pkg-type }}-* + ## DGM pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.pkg-type }}-* + pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.pkg-type }}* separate-directories: true delete-merged: true From 58083b7cef476b0e3f6d8e6f0c2389e3fc1cf4d9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 12 Sep 2024 14:33:24 -0600 Subject: [PATCH 78/97] Debugging --- .github/workflows/test-action-linux.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-action-linux.yml b/.github/workflows/test-action-linux.yml index 60c802c0a837..c9eda0026f1a 100644 --- a/.github/workflows/test-action-linux.yml +++ b/.github/workflows/test-action-linux.yml @@ -354,6 +354,7 @@ jobs: - name: Merge Code Coverage Test Run Artifacts if: ${{ inputs.skip-code-coverage == false }} continue-on-error: true + needs: upload uses: actions/upload-artifact/merge@v4 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} @@ -361,14 +362,22 @@ jobs: pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}* separate-directories: false delete-merged: true + # DGM + overwrite: true - name: Download Code Coverage Test Run Artifacts uses: actions/download-artifact@v4 if: ${{ inputs.skip-code-coverage == false }} id: download-coverage-artifacts with: - name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} + ## DGM name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} path: artifacts/coverage/ + pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}* + merge-multiple: true + + - name: DGM Show Downloaded Test Run Artifacts + run: | + tree -a artifacts - name: Show Downloaded Test Run Artifacts if: ${{ inputs.skip-code-coverage == false }} From 51cc9b8b2e22b549820abcbcc2a668722237290b Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 13 Sep 2024 10:04:57 -0600 Subject: [PATCH 79/97] change to force build, since github stalled it overnight --- .github/workflows/test-action-linux.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-action-linux.yml b/.github/workflows/test-action-linux.yml index c9eda0026f1a..ebe9567133eb 100644 --- a/.github/workflows/test-action-linux.yml +++ b/.github/workflows/test-action-linux.yml @@ -375,6 +375,7 @@ jobs: pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}* merge-multiple: true + # DGM - name: DGM Show Downloaded Test Run Artifacts run: | tree -a artifacts From 51099d9901b0ff8786455436f74daa4fc8db4eb6 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 13 Sep 2024 11:35:53 -0600 Subject: [PATCH 80/97] Correct logic error - removed 'needs' --- .github/workflows/test-action-linux.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-action-linux.yml b/.github/workflows/test-action-linux.yml index ebe9567133eb..c19338625a6c 100644 --- a/.github/workflows/test-action-linux.yml +++ b/.github/workflows/test-action-linux.yml @@ -354,7 +354,6 @@ jobs: - name: Merge Code Coverage Test Run Artifacts if: ${{ inputs.skip-code-coverage == false }} continue-on-error: true - needs: upload uses: actions/upload-artifact/merge@v4 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} From 6e0fc168bb4d1a6853e5266bd6be108da2ac1858 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 16 Sep 2024 11:37:00 -0600 Subject: [PATCH 81/97] Update Windows and MacOS actions for Download Code Coverage Test Run Artifacts for merge-multiple, v4 update --- .github/workflows/test-action-linux.yml | 2 +- .github/workflows/test-action-macos.yml | 11 ++++++++++- .github/workflows/test-action-windows.yml | 11 ++++++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-action-linux.yml b/.github/workflows/test-action-linux.yml index c19338625a6c..6f1b2458b921 100644 --- a/.github/workflows/test-action-linux.yml +++ b/.github/workflows/test-action-linux.yml @@ -362,7 +362,7 @@ jobs: separate-directories: false delete-merged: true # DGM - overwrite: true + ## DGM delete this test # overwrite: true - name: Download Code Coverage Test Run Artifacts uses: actions/download-artifact@v4 diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index a65dc59118b8..a7217f270840 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -311,6 +311,8 @@ jobs: if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }} fi + # DGM + tree -a artifacts - name: Upload Code Coverage Test Run Artifacts if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled' @@ -395,8 +397,15 @@ jobs: if: ${{ inputs.skip-code-coverage == false }} id: download-coverage-artifacts with: - name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} + ## DGM name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} path: artifacts/coverage/ + pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* + merge-multiple: true + + # DGM + - name: DGM Show Downloaded Test Run Artifacts + run: | + tree -a artifacts - name: Show Downloaded Test Run Artifacts if: ${{ inputs.skip-code-coverage == false }} diff --git a/.github/workflows/test-action-windows.yml b/.github/workflows/test-action-windows.yml index 192f73a1b5c4..7ee1aa17c48e 100644 --- a/.github/workflows/test-action-windows.yml +++ b/.github/workflows/test-action-windows.yml @@ -276,6 +276,8 @@ jobs: if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }}.grp${{ matrix.test-group || '1' }} fi + # DGM + tree -a artifacts - name: Destroy VM if: always() @@ -366,8 +368,15 @@ jobs: if: ${{ inputs.skip-code-coverage == false }} id: download-coverage-artifacts with: - name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} + ## DGM name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} path: artifacts/coverage/ + pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* + merge-multiple: true + + # DGM + - name: DGM Show Downloaded Test Run Artifacts + run: | + tree -a artifacts - name: Show Downloaded Test Run Artifacts if: ${{ inputs.skip-code-coverage == false }} From a6fff58e4a8f80987c0de8b9427c3ad4f12e026f Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 16 Sep 2024 16:58:34 -0600 Subject: [PATCH 82/97] Remove comments --- .github/workflows/ci.yml | 3 +-- .github/workflows/nightly.yml | 3 +-- .github/workflows/scheduled.yml | 3 +-- .github/workflows/templates/ci.yml.jinja | 3 +-- .github/workflows/test-action-linux.yml | 20 +++---------------- .github/workflows/test-action-macos.yml | 18 +++-------------- .github/workflows/test-action-windows.yml | 18 +++-------------- .../workflows/test-packages-action-linux.yml | 3 +-- .../workflows/test-packages-action-macos.yml | 3 +-- .../test-packages-action-windows.yml | 3 +-- 10 files changed, 16 insertions(+), 61 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 392d13294311..4e88f05b4a24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2019,8 +2019,7 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: all-testrun-coverage-artifacts - ## DGM pattern: all-testrun-coverage-artifacts-* - pattern: all-testrun-coverage-artifacts* + pattern: all-testrun-coverage-artifacts-* separate-directories: false delete-merged: true diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5033f37a7940..d7bb49730019 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2081,8 +2081,7 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: all-testrun-coverage-artifacts - ## DGM pattern: all-testrun-coverage-artifacts-* - pattern: all-testrun-coverage-artifacts* + pattern: all-testrun-coverage-artifacts-* separate-directories: false delete-merged: true diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 6fd4c1b15ee3..86e57fb71a42 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -2058,8 +2058,7 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: all-testrun-coverage-artifacts - ## DGM pattern: all-testrun-coverage-artifacts-* - pattern: all-testrun-coverage-artifacts* + pattern: all-testrun-coverage-artifacts-* separate-directories: false delete-merged: true diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index a035888a2430..91713863f187 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -362,8 +362,7 @@ uses: actions/upload-artifact/merge@v4 with: name: all-testrun-coverage-artifacts - ## DGM pattern: all-testrun-coverage-artifacts-* - pattern: all-testrun-coverage-artifacts* + pattern: all-testrun-coverage-artifacts-* separate-directories: false delete-merged: true diff --git a/.github/workflows/test-action-linux.yml b/.github/workflows/test-action-linux.yml index 6f1b2458b921..83f99a0b66d2 100644 --- a/.github/workflows/test-action-linux.yml +++ b/.github/workflows/test-action-linux.yml @@ -276,8 +276,6 @@ jobs: if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}${{ inputs.fips && '.fips' || '' }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }}.grp${{ matrix.test-group || '1' }} fi - # DGM - tree -a artifacts - name: Destroy VM if: always() @@ -291,7 +289,6 @@ jobs: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }} path: | artifacts/coverage/ - # DGM include-hidden-files: true - name: Upload JUnit XML Test Run Artifacts @@ -335,8 +332,7 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} - ## DGM pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* - pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}* + pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* separate-directories: false delete-merged: true @@ -346,8 +342,7 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} - ## DGM pattern: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* - pattern: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}* + pattern: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* separate-directories: false delete-merged: true @@ -357,28 +352,19 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} - ## DGM pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* - pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}* + pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* separate-directories: false delete-merged: true - # DGM - ## DGM delete this test # overwrite: true - name: Download Code Coverage Test Run Artifacts uses: actions/download-artifact@v4 if: ${{ inputs.skip-code-coverage == false }} id: download-coverage-artifacts with: - ## DGM name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} path: artifacts/coverage/ pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}* merge-multiple: true - # DGM - - name: DGM Show Downloaded Test Run Artifacts - run: | - tree -a artifacts - - name: Show Downloaded Test Run Artifacts if: ${{ inputs.skip-code-coverage == false }} run: | diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index a7217f270840..a3f0e75a7434 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -311,8 +311,6 @@ jobs: if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }} fi - # DGM - tree -a artifacts - name: Upload Code Coverage Test Run Artifacts if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled' @@ -321,7 +319,6 @@ jobs: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ env.TIMESTAMP }} path: | artifacts/coverage/ - # DGM include-hidden-files: true - name: Upload JUnit XML Test Run Artifacts @@ -365,8 +362,7 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - ## DGM pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* - pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* + pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* separate-directories: false delete-merged: true @@ -376,8 +372,7 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - ## DGM pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* - pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* + pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* separate-directories: false delete-merged: true @@ -387,8 +382,7 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - ## DGM pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* - pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* + pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* separate-directories: false delete-merged: true @@ -397,16 +391,10 @@ jobs: if: ${{ inputs.skip-code-coverage == false }} id: download-coverage-artifacts with: - ## DGM name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} path: artifacts/coverage/ pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* merge-multiple: true - # DGM - - name: DGM Show Downloaded Test Run Artifacts - run: | - tree -a artifacts - - name: Show Downloaded Test Run Artifacts if: ${{ inputs.skip-code-coverage == false }} run: | diff --git a/.github/workflows/test-action-windows.yml b/.github/workflows/test-action-windows.yml index 7ee1aa17c48e..e8944451b076 100644 --- a/.github/workflows/test-action-windows.yml +++ b/.github/workflows/test-action-windows.yml @@ -276,8 +276,6 @@ jobs: if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }}.grp${{ matrix.test-group || '1' }} fi - # DGM - tree -a artifacts - name: Destroy VM if: always() @@ -291,7 +289,6 @@ jobs: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }} path: | artifacts/coverage/ - # DGM include-hidden-files: true - name: Upload JUnit XML Test Run Artifacts @@ -336,8 +333,7 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - ## DGM pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* - pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* + pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* separate-directories: false delete-merged: true @@ -347,8 +343,7 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - ## DGM pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* - pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* + pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* separate-directories: false delete-merged: true @@ -358,8 +353,7 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - ## DGM pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* - pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* + pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* separate-directories: false delete-merged: true @@ -368,16 +362,10 @@ jobs: if: ${{ inputs.skip-code-coverage == false }} id: download-coverage-artifacts with: - ## DGM name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} path: artifacts/coverage/ pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* merge-multiple: true - # DGM - - name: DGM Show Downloaded Test Run Artifacts - run: | - tree -a artifacts - - name: Show Downloaded Test Run Artifacts if: ${{ inputs.skip-code-coverage == false }} run: | diff --git a/.github/workflows/test-packages-action-linux.yml b/.github/workflows/test-packages-action-linux.yml index dfde25f51866..36b4d7818d47 100644 --- a/.github/workflows/test-packages-action-linux.yml +++ b/.github/workflows/test-packages-action-linux.yml @@ -251,8 +251,7 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: pkg-testrun-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.pkg-type }} - ## DGM pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.pkg-type }}-* - pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.pkg-type }}* + pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.pkg-type }}-* separate-directories: true delete-merged: true diff --git a/.github/workflows/test-packages-action-macos.yml b/.github/workflows/test-packages-action-macos.yml index df3696cce17f..aca0b4cc244c 100644 --- a/.github/workflows/test-packages-action-macos.yml +++ b/.github/workflows/test-packages-action-macos.yml @@ -242,8 +242,7 @@ jobs: uses: actions/upload-artifact/merge@v4 with: name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }} - ## DGM pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-* - pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}* + pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-* separate-directories: true delete-merged: true diff --git a/.github/workflows/test-packages-action-windows.yml b/.github/workflows/test-packages-action-windows.yml index c91dc5973a3c..985ff96de820 100644 --- a/.github/workflows/test-packages-action-windows.yml +++ b/.github/workflows/test-packages-action-windows.yml @@ -250,8 +250,7 @@ jobs: continue-on-error: true with: name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }} - ## DGM pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-* - pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}* + pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-* separate-directories: true delete-merged: true From 2db4de4170da9bb98738125385453f6d60fc062d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 19 Sep 2024 13:44:18 -0600 Subject: [PATCH 83/97] Solve the lack of combined coverage reports --- .github/workflows/ci.yml | 4 +++- .github/workflows/nightly.yml | 4 +++- .github/workflows/scheduled.yml | 4 +++- .github/workflows/templates/ci.yml.jinja | 4 +++- .github/workflows/test-action-linux.yml | 1 + .github/workflows/test-action-macos.yml | 1 + .github/workflows/test-action-windows.yml | 1 + 7 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e88f05b4a24..75b6e8aa1964 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2027,8 +2027,10 @@ jobs: id: get-coverage-reports uses: actions/download-artifact@v4 with: - name: all-testrun-coverage-artifacts + ## name: all-testrun-coverage-artifacts path: artifacts/coverage/ + pattern: all-testrun-coverage-artifacts* + merge-multiple: true - name: Display structure of downloaded files run: tree -a artifacts/ diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d7bb49730019..afa043b118aa 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2089,8 +2089,10 @@ jobs: id: get-coverage-reports uses: actions/download-artifact@v4 with: - name: all-testrun-coverage-artifacts + ## name: all-testrun-coverage-artifacts path: artifacts/coverage/ + pattern: all-testrun-coverage-artifacts* + merge-multiple: true - name: Display structure of downloaded files run: tree -a artifacts/ diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 86e57fb71a42..caec0e546a90 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -2066,8 +2066,10 @@ jobs: id: get-coverage-reports uses: actions/download-artifact@v4 with: - name: all-testrun-coverage-artifacts + ## name: all-testrun-coverage-artifacts path: artifacts/coverage/ + pattern: all-testrun-coverage-artifacts* + merge-multiple: true - name: Display structure of downloaded files run: tree -a artifacts/ diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index 91713863f187..f105c30ca45c 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -370,8 +370,10 @@ id: get-coverage-reports uses: actions/download-artifact@v4 with: - name: all-testrun-coverage-artifacts + ## name: all-testrun-coverage-artifacts path: artifacts/coverage/ + pattern: all-testrun-coverage-artifacts* + merge-multiple: true - name: Display structure of downloaded files run: tree -a artifacts/ diff --git a/.github/workflows/test-action-linux.yml b/.github/workflows/test-action-linux.yml index 83f99a0b66d2..cb3ee37c3b31 100644 --- a/.github/workflows/test-action-linux.yml +++ b/.github/workflows/test-action-linux.yml @@ -405,3 +405,4 @@ jobs: with: name: all-testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} path: artifacts/coverage + include-hidden-files: true diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index a3f0e75a7434..0e429f2fcae1 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -440,3 +440,4 @@ jobs: with: name: all-testrun-coverage-artifacts-${{ inputs.distro-slug }}.${{ inputs.nox-session }} path: artifacts/coverage + include-hidden-files: true diff --git a/.github/workflows/test-action-windows.yml b/.github/workflows/test-action-windows.yml index e8944451b076..ac013a2869ad 100644 --- a/.github/workflows/test-action-windows.yml +++ b/.github/workflows/test-action-windows.yml @@ -406,3 +406,4 @@ jobs: with: name: all-testrun-coverage-artifacts-${{ inputs.distro-slug }}.${{ inputs.nox-session }} path: artifacts/coverage + include-hidden-files: true From 393dad1790e7e91d467a5a459cff0cd33421c5b9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 20 Sep 2024 09:20:34 -0600 Subject: [PATCH 84/97] Removed commented out name --- .github/workflows/ci.yml | 1 - .github/workflows/nightly.yml | 1 - .github/workflows/scheduled.yml | 1 - .github/workflows/templates/ci.yml.jinja | 1 - 4 files changed, 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75b6e8aa1964..d69a3c0de02a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2027,7 +2027,6 @@ jobs: id: get-coverage-reports uses: actions/download-artifact@v4 with: - ## name: all-testrun-coverage-artifacts path: artifacts/coverage/ pattern: all-testrun-coverage-artifacts* merge-multiple: true diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index afa043b118aa..c6627b25610b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2089,7 +2089,6 @@ jobs: id: get-coverage-reports uses: actions/download-artifact@v4 with: - ## name: all-testrun-coverage-artifacts path: artifacts/coverage/ pattern: all-testrun-coverage-artifacts* merge-multiple: true diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index caec0e546a90..b26b5267b695 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -2066,7 +2066,6 @@ jobs: id: get-coverage-reports uses: actions/download-artifact@v4 with: - ## name: all-testrun-coverage-artifacts path: artifacts/coverage/ pattern: all-testrun-coverage-artifacts* merge-multiple: true diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index f105c30ca45c..073c03ae1707 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -370,7 +370,6 @@ id: get-coverage-reports uses: actions/download-artifact@v4 with: - ## name: all-testrun-coverage-artifacts path: artifacts/coverage/ pattern: all-testrun-coverage-artifacts* merge-multiple: true From b912817240266153ae7d74eccb2ec4b489a2011b Mon Sep 17 00:00:00 2001 From: Shane Lee Date: Wed, 17 Jul 2024 14:47:14 -0600 Subject: [PATCH 85/97] Fixes cmd.run with requirements on Windows Formats the command properly for powershell Adds changelog and tests --- changelog/66596.fixed.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog/66596.fixed.md diff --git a/changelog/66596.fixed.md b/changelog/66596.fixed.md new file mode 100644 index 000000000000..a4a27151f2c1 --- /dev/null +++ b/changelog/66596.fixed.md @@ -0,0 +1,2 @@ +Fixed an issue with cmd.run with requirements when the shell is not the +default From bf580e66989601313434fe4e00589b2e02f353ac Mon Sep 17 00:00:00 2001 From: Marek Czernek Date: Mon, 8 Jul 2024 12:41:14 +0200 Subject: [PATCH 86/97] Make tests compatible with venv bundle --- tests/pytests/unit/modules/test_pip.py | 63 ++++++++++++++++++-------- tests/pytests/unit/test_fileserver.py | 4 +- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/tests/pytests/unit/modules/test_pip.py b/tests/pytests/unit/modules/test_pip.py index 87ffe29a8572..2ef71b592189 100644 --- a/tests/pytests/unit/modules/test_pip.py +++ b/tests/pytests/unit/modules/test_pip.py @@ -10,6 +10,10 @@ from salt.exceptions import CommandExecutionError from tests.support.mock import MagicMock, patch +TARGET = [] +if os.environ.get('VENV_PIP_TARGET'): + TARGET = ["--target", os.environ.get('VENV_PIP_TARGET')] + class FakeFopen: def __init__(self, filename): @@ -97,6 +101,7 @@ def test_install_frozen_app(python_binary): expected = [ *python_binary, "install", + *TARGET, pkg, ] mock.assert_called_with( @@ -118,6 +123,7 @@ def test_install_source_app(python_binary): expected = [ *python_binary, "install", + *TARGET, pkg, ] mock.assert_called_with( @@ -138,6 +144,7 @@ def test_fix4361(python_binary): "install", "--requirement", "requirements.txt", + *TARGET, ] mock.assert_called_with( expected_cmd, @@ -164,7 +171,7 @@ def test_install_multiple_editable(python_binary): "git+https://github.com/saltstack/salt-testing.git#egg=SaltTesting", ] - expected = [*python_binary, "install"] + expected = [*python_binary, "install", *TARGET] for item in editables: expected.extend(["--editable", item]) @@ -200,7 +207,7 @@ def test_install_multiple_pkgs_and_editables(python_binary): "git+https://github.com/saltstack/salt-testing.git#egg=SaltTesting", ] - expected = [*python_binary, "install"] + expected = [*python_binary, "install", *TARGET] expected.extend(pkgs) for item in editables: expected.extend(["--editable", item]) @@ -236,6 +243,7 @@ def test_install_multiple_pkgs_and_editables(python_binary): expected = [ *python_binary, "install", + *TARGET, pkgs[0], "--editable", editables[0], @@ -263,7 +271,7 @@ def test_issue5940_install_multiple_pip_mirrors(python_binary): expected = [*python_binary, "install", "--use-mirrors"] for item in mirrors: expected.extend(["--mirrors", item]) - expected.append("pep8") + expected = [*expected, *TARGET, "pep8"] # Passing mirrors as a list mock = MagicMock(return_value={"retcode": 0, "stdout": ""}) @@ -295,6 +303,7 @@ def test_issue5940_install_multiple_pip_mirrors(python_binary): "--use-mirrors", "--mirrors", mirrors[0], + *TARGET, "pep8", ] @@ -322,7 +331,7 @@ def test_install_with_multiple_find_links(python_binary): expected = [*python_binary, "install"] for item in find_links: expected.extend(["--find-links", item]) - expected.append(pkg) + expected = [*expected, *TARGET, pkg] # Passing mirrors as a list mock = MagicMock(return_value={"retcode": 0, "stdout": ""}) @@ -365,6 +374,7 @@ def test_install_with_multiple_find_links(python_binary): "install", "--find-links", find_links[0], + *TARGET, pkg, ] @@ -430,6 +440,7 @@ def test_install_cached_requirements_used(python_binary): "install", "--requirement", "my_cached_reqs", + *TARGET, ] mock.assert_called_with( expected, @@ -486,6 +497,7 @@ def test_install_log_argument_in_resulting_command(python_binary, tmp_path): "install", "--log", log_path, + *TARGET, pkg, ] mock.assert_called_with( @@ -516,7 +528,7 @@ def test_install_timeout_argument_in_resulting_command(python_binary): with patch.dict(pip.__salt__, {"cmd.run_all": mock}): pip.install(pkg, timeout=10) mock.assert_called_with( - expected + [10, pkg], + expected + [10, *TARGET, pkg], saltenv="base", runas=None, use_vt=False, @@ -528,7 +540,7 @@ def test_install_timeout_argument_in_resulting_command(python_binary): with patch.dict(pip.__salt__, {"cmd.run_all": mock}): pip.install(pkg, timeout="10") mock.assert_called_with( - expected + ["10", pkg], + expected + ["10", *TARGET, pkg], saltenv="base", runas=None, use_vt=False, @@ -552,6 +564,7 @@ def test_install_index_url_argument_in_resulting_command(python_binary): "install", "--index-url", index_url, + *TARGET, pkg, ] mock.assert_called_with( @@ -574,6 +587,7 @@ def test_install_extra_index_url_argument_in_resulting_command(python_binary): "install", "--extra-index-url", extra_index_url, + *TARGET, pkg, ] mock.assert_called_with( @@ -590,7 +604,7 @@ def test_install_no_index_argument_in_resulting_command(python_binary): mock = MagicMock(return_value={"retcode": 0, "stdout": ""}) with patch.dict(pip.__salt__, {"cmd.run_all": mock}): pip.install(pkg, no_index=True) - expected = [*python_binary, "install", "--no-index", pkg] + expected = [*python_binary, "install", "--no-index", *TARGET, pkg] mock.assert_called_with( expected, saltenv="base", @@ -606,7 +620,7 @@ def test_install_build_argument_in_resulting_command(python_binary): mock = MagicMock(return_value={"retcode": 0, "stdout": ""}) with patch.dict(pip.__salt__, {"cmd.run_all": mock}): pip.install(pkg, build=build) - expected = [*python_binary, "install", "--build", build, pkg] + expected = [*python_binary, "install", "--build", build, *TARGET, pkg] mock.assert_called_with( expected, saltenv="base", @@ -641,6 +655,7 @@ def test_install_download_argument_in_resulting_command(python_binary): expected = [ *python_binary, "install", + *TARGET, "--download", download, pkg, @@ -659,7 +674,7 @@ def test_install_no_download_argument_in_resulting_command(python_binary): mock = MagicMock(return_value={"retcode": 0, "stdout": ""}) with patch.dict(pip.__salt__, {"cmd.run_all": mock}): pip.install(pkg, no_download=True) - expected = [*python_binary, "install", "--no-download", pkg] + expected = [*python_binary, "install", *TARGET, "--no-download", pkg] mock.assert_called_with( expected, saltenv="base", @@ -686,6 +701,7 @@ def test_install_download_cache_dir_arguments_in_resulting_command(python_binary expected = [ *python_binary, "install", + *TARGET, cmd_arg, download_cache, pkg, @@ -715,7 +731,7 @@ def test_install_source_argument_in_resulting_command(python_binary): mock = MagicMock(return_value={"retcode": 0, "stdout": ""}) with patch.dict(pip.__salt__, {"cmd.run_all": mock}): pip.install(pkg, source=source) - expected = [*python_binary, "install", "--source", source, pkg] + expected = [*python_binary, "install", *TARGET, "--source", source, pkg] mock.assert_called_with( expected, saltenv="base", @@ -734,6 +750,7 @@ def test_install_exists_action_argument_in_resulting_command(python_binary): expected = [ *python_binary, "install", + *TARGET, "--exists-action", action, pkg, @@ -756,7 +773,7 @@ def test_install_install_options_argument_in_resulting_command(python_binary): install_options = ["--exec-prefix=/foo/bar", "--install-scripts=/foo/bar/bin"] pkg = "pep8" - expected = [*python_binary, "install"] + expected = [*python_binary, "install", *TARGET] for item in install_options: expected.extend(["--install-option", item]) expected.append(pkg) @@ -792,6 +809,7 @@ def test_install_install_options_argument_in_resulting_command(python_binary): expected = [ *python_binary, "install", + *TARGET, "--install-option", install_options[0], pkg, @@ -809,7 +827,7 @@ def test_install_global_options_argument_in_resulting_command(python_binary): global_options = ["--quiet", "--no-user-cfg"] pkg = "pep8" - expected = [*python_binary, "install"] + expected = [*python_binary, "install", *TARGET] for item in global_options: expected.extend(["--global-option", item]) expected.append(pkg) @@ -845,6 +863,7 @@ def test_install_global_options_argument_in_resulting_command(python_binary): expected = [ *python_binary, "install", + *TARGET, "--global-option", global_options[0], pkg, @@ -863,7 +882,7 @@ def test_install_upgrade_argument_in_resulting_command(python_binary): mock = MagicMock(return_value={"retcode": 0, "stdout": ""}) with patch.dict(pip.__salt__, {"cmd.run_all": mock}): pip.install(pkg, upgrade=True) - expected = [*python_binary, "install", "--upgrade", pkg] + expected = [*python_binary, "install", *TARGET, "--upgrade", pkg] mock.assert_called_with( expected, saltenv="base", @@ -881,6 +900,7 @@ def test_install_force_reinstall_argument_in_resulting_command(python_binary): expected = [ *python_binary, "install", + *TARGET, "--force-reinstall", pkg, ] @@ -901,6 +921,7 @@ def test_install_ignore_installed_argument_in_resulting_command(python_binary): expected = [ *python_binary, "install", + *TARGET, "--ignore-installed", pkg, ] @@ -918,7 +939,7 @@ def test_install_no_deps_argument_in_resulting_command(python_binary): mock = MagicMock(return_value={"retcode": 0, "stdout": ""}) with patch.dict(pip.__salt__, {"cmd.run_all": mock}): pip.install(pkg, no_deps=True) - expected = [*python_binary, "install", "--no-deps", pkg] + expected = [*python_binary, "install", *TARGET, "--no-deps", pkg] mock.assert_called_with( expected, saltenv="base", @@ -933,7 +954,7 @@ def test_install_no_install_argument_in_resulting_command(python_binary): mock = MagicMock(return_value={"retcode": 0, "stdout": ""}) with patch.dict(pip.__salt__, {"cmd.run_all": mock}): pip.install(pkg, no_install=True) - expected = [*python_binary, "install", "--no-install", pkg] + expected = [*python_binary, "install", *TARGET, "--no-install", pkg] mock.assert_called_with( expected, saltenv="base", @@ -949,7 +970,7 @@ def test_install_proxy_argument_in_resulting_command(python_binary): mock = MagicMock(return_value={"retcode": 0, "stdout": ""}) with patch.dict(pip.__salt__, {"cmd.run_all": mock}): pip.install(pkg, proxy=proxy) - expected = [*python_binary, "install", "--proxy", proxy, pkg] + expected = [*python_binary, "install", "--proxy", proxy, *TARGET, pkg] mock.assert_called_with( expected, saltenv="base", @@ -976,7 +997,7 @@ def test_install_proxy_false_argument_in_resulting_command(python_binary): with patch.dict(pip.__salt__, {"cmd.run_all": mock}): with patch.dict(pip.__opts__, config_mock): pip.install(pkg, proxy=proxy) - expected = [*python_binary, "install", pkg] + expected = [*python_binary, "install", *TARGET, pkg] mock.assert_called_with( expected, saltenv="base", @@ -1007,6 +1028,7 @@ def test_install_global_proxy_in_resulting_command(python_binary): "install", "--proxy", proxy, + *TARGET, pkg, ] mock.assert_called_with( @@ -1027,6 +1049,7 @@ def test_install_multiple_requirements_arguments_in_resulting_command(python_bin expected = [*python_binary, "install"] for item in cached_reqs: expected.extend(["--requirement", item]) + expected.extend(TARGET) # Passing option as a list mock = MagicMock(return_value={"retcode": 0, "stdout": ""}) @@ -1063,6 +1086,7 @@ def test_install_multiple_requirements_arguments_in_resulting_command(python_bin "install", "--requirement", cached_reqs[0], + *TARGET, ] mock.assert_called_with( expected, @@ -1083,6 +1107,7 @@ def test_install_extra_args_arguments_in_resulting_command(python_binary): expected = [ *python_binary, "install", + *TARGET, pkg, "--latest-pip-kwarg", "param", @@ -1599,7 +1624,7 @@ def test_install_pre_argument_in_resulting_command(python_binary): with patch.dict(pip.__salt__, {"cmd.run_all": mock}): with patch("salt.modules.pip.version", MagicMock(return_value="1.3")): pip.install(pkg, pre_releases=True) - expected = [*python_binary, "install", pkg] + expected = [*python_binary, "install", *TARGET, pkg] mock.assert_called_with( expected, saltenv="base", @@ -1615,7 +1640,7 @@ def test_install_pre_argument_in_resulting_command(python_binary): ): with patch("salt.modules.pip._get_pip_bin", MagicMock(return_value=["pip"])): pip.install(pkg, pre_releases=True) - expected = ["pip", "install", "--pre", pkg] + expected = ["pip", "install", *TARGET, "--pre", pkg] mock_run_all.assert_called_with( expected, saltenv="base", diff --git a/tests/pytests/unit/test_fileserver.py b/tests/pytests/unit/test_fileserver.py index 8dd3ea0a27d6..661a58bf5098 100644 --- a/tests/pytests/unit/test_fileserver.py +++ b/tests/pytests/unit/test_fileserver.py @@ -76,7 +76,7 @@ def test_file_server_url_escape(tmp_path): "fileserver_backend": ["roots"], "extension_modules": "", "optimization_order": [ - 0, + 0, 1 ], "file_roots": { "base": [fileroot], @@ -103,7 +103,7 @@ def test_file_server_serve_url_escape(tmp_path): "fileserver_backend": ["roots"], "extension_modules": "", "optimization_order": [ - 0, + 0, 1 ], "file_roots": { "base": [fileroot], From db31e2bc94bd7104d6d92a2c19eaa7b6e40301a0 Mon Sep 17 00:00:00 2001 From: Marek Czernek Date: Wed, 10 Jul 2024 10:37:33 +0200 Subject: [PATCH 87/97] Fix test_transactional_update tests --- .../unit/modules/test_transactional_update.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/pytests/unit/modules/test_transactional_update.py b/tests/pytests/unit/modules/test_transactional_update.py index eb6dfae28684..36bb1663f864 100644 --- a/tests/pytests/unit/modules/test_transactional_update.py +++ b/tests/pytests/unit/modules/test_transactional_update.py @@ -1,3 +1,4 @@ +import os import pytest import salt.loader.context @@ -10,6 +11,10 @@ pytest.mark.skip_on_windows(reason="Not supported on Windows"), ] +SALT_CALL_BINARY = "salt-call" +if os.environ.get('VIRTUAL_ENV'): + SALT_CALL_BINARY = f"{os.environ.get('VIRTUAL_ENV')}/bin/salt-call" + @pytest.fixture def configure_loader_modules(): @@ -381,7 +386,7 @@ def test_call_fails_function(): "--continue", "--quiet", "run", - "salt-call", + SALT_CALL_BINARY, "--out", "json", "-l", @@ -413,7 +418,7 @@ def test_call_success_no_reboot(): "--continue", "--quiet", "run", - "salt-call", + SALT_CALL_BINARY, "--out", "json", "-l", @@ -456,7 +461,7 @@ def test_call_success_reboot(): "--continue", "--quiet", "run", - "salt-call", + SALT_CALL_BINARY, "--out", "json", "-l", @@ -490,7 +495,7 @@ def test_call_success_parameters(): "--continue", "--quiet", "run", - "salt-call", + SALT_CALL_BINARY, "--out", "json", "-l", From 29d4137e0b2a462252f39b994e9e90da8dc5b2d9 Mon Sep 17 00:00:00 2001 From: Marek Czernek Date: Wed, 17 Jul 2024 11:35:37 +0200 Subject: [PATCH 88/97] We depend on msgpack < 1.0.0 --- tests/pytests/unit/utils/test_msgpack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytests/unit/utils/test_msgpack.py b/tests/pytests/unit/utils/test_msgpack.py index e15da262b00f..a87c5ed5a673 100644 --- a/tests/pytests/unit/utils/test_msgpack.py +++ b/tests/pytests/unit/utils/test_msgpack.py @@ -3,7 +3,7 @@ import salt.utils.msgpack from tests.support.mock import MagicMock, patch - +@pytest.mark.skipif(salt.utils.msgpack.version < (1, 0, 0), reason="Test requires msgpack version >= 1.0.0") def test_load_encoding(tmp_path): """ test when using msgpack version >= 1.0.0 we From 0675d1fec7bbc21e19d0abc8511e3766866114e3 Mon Sep 17 00:00:00 2001 From: Marek Czernek Date: Wed, 17 Jul 2024 16:21:11 +0200 Subject: [PATCH 89/97] Remove passlib assumption from pycrypto tests --- tests/pytests/unit/utils/test_pycrypto.py | 25 +++++++++++------------ 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/tests/pytests/unit/utils/test_pycrypto.py b/tests/pytests/unit/utils/test_pycrypto.py index 1dfcf9621c41..9dff08f883e1 100644 --- a/tests/pytests/unit/utils/test_pycrypto.py +++ b/tests/pytests/unit/utils/test_pycrypto.py @@ -57,21 +57,20 @@ def test_gen_hash_crypt(algorithm, expected): """ Test gen_hash with crypt library """ - with patch("salt.utils.pycrypto.methods", {}): - ret = salt.utils.pycrypto.gen_hash( - crypt_salt=expected["salt"], password=passwd, algorithm=algorithm - ) - assert ret == expected["hashed"] + ret = salt.utils.pycrypto.gen_hash( + crypt_salt=expected["salt"], password=passwd, algorithm=algorithm + ) + assert ret == expected["hashed"] - ret = salt.utils.pycrypto.gen_hash( - crypt_salt=expected["badsalt"], password=passwd, algorithm=algorithm - ) - assert ret != expected["hashed"] + ret = salt.utils.pycrypto.gen_hash( + crypt_salt=expected["badsalt"], password=passwd, algorithm=algorithm + ) + assert ret != expected["hashed"] - ret = salt.utils.pycrypto.gen_hash( - crypt_salt=None, password=passwd, algorithm=algorithm - ) - assert ret != expected["hashed"] + ret = salt.utils.pycrypto.gen_hash( + crypt_salt=None, password=passwd, algorithm=algorithm + ) + assert ret != expected["hashed"] @pytest.mark.skipif(not salt.utils.pycrypto.HAS_CRYPT, reason="crypt not available") From 101a773df176217da39f12759b4240f70d62ca26 Mon Sep 17 00:00:00 2001 From: Shane Lee Date: Fri, 19 Jul 2024 10:52:09 -0600 Subject: [PATCH 90/97] Fix pre-commit --- tests/pytests/unit/modules/test_pip.py | 4 ++-- tests/pytests/unit/modules/test_transactional_update.py | 3 ++- tests/pytests/unit/test_fileserver.py | 8 ++------ tests/pytests/unit/utils/test_msgpack.py | 6 +++++- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/pytests/unit/modules/test_pip.py b/tests/pytests/unit/modules/test_pip.py index 2ef71b592189..e30bf1e138d2 100644 --- a/tests/pytests/unit/modules/test_pip.py +++ b/tests/pytests/unit/modules/test_pip.py @@ -11,8 +11,8 @@ from tests.support.mock import MagicMock, patch TARGET = [] -if os.environ.get('VENV_PIP_TARGET'): - TARGET = ["--target", os.environ.get('VENV_PIP_TARGET')] +if os.environ.get("VENV_PIP_TARGET"): + TARGET = ["--target", os.environ.get("VENV_PIP_TARGET")] class FakeFopen: diff --git a/tests/pytests/unit/modules/test_transactional_update.py b/tests/pytests/unit/modules/test_transactional_update.py index 36bb1663f864..6d896d4cfbfd 100644 --- a/tests/pytests/unit/modules/test_transactional_update.py +++ b/tests/pytests/unit/modules/test_transactional_update.py @@ -1,4 +1,5 @@ import os + import pytest import salt.loader.context @@ -12,7 +13,7 @@ ] SALT_CALL_BINARY = "salt-call" -if os.environ.get('VIRTUAL_ENV'): +if os.environ.get("VIRTUAL_ENV"): SALT_CALL_BINARY = f"{os.environ.get('VIRTUAL_ENV')}/bin/salt-call" diff --git a/tests/pytests/unit/test_fileserver.py b/tests/pytests/unit/test_fileserver.py index 661a58bf5098..49be3967dc40 100644 --- a/tests/pytests/unit/test_fileserver.py +++ b/tests/pytests/unit/test_fileserver.py @@ -75,9 +75,7 @@ def test_file_server_url_escape(tmp_path): opts = { "fileserver_backend": ["roots"], "extension_modules": "", - "optimization_order": [ - 0, 1 - ], + "optimization_order": [0, 1], "file_roots": { "base": [fileroot], }, @@ -102,9 +100,7 @@ def test_file_server_serve_url_escape(tmp_path): opts = { "fileserver_backend": ["roots"], "extension_modules": "", - "optimization_order": [ - 0, 1 - ], + "optimization_order": [0, 1], "file_roots": { "base": [fileroot], }, diff --git a/tests/pytests/unit/utils/test_msgpack.py b/tests/pytests/unit/utils/test_msgpack.py index a87c5ed5a673..feebcf1f88d4 100644 --- a/tests/pytests/unit/utils/test_msgpack.py +++ b/tests/pytests/unit/utils/test_msgpack.py @@ -3,7 +3,11 @@ import salt.utils.msgpack from tests.support.mock import MagicMock, patch -@pytest.mark.skipif(salt.utils.msgpack.version < (1, 0, 0), reason="Test requires msgpack version >= 1.0.0") + +@pytest.mark.skipif( + salt.utils.msgpack.version < (1, 0, 0), + reason="Test requires msgpack version >= 1.0.0", +) def test_load_encoding(tmp_path): """ test when using msgpack version >= 1.0.0 we From c0354fc1618fa289f6851d419642f0c47779ddcf Mon Sep 17 00:00:00 2001 From: Marek Czernek Date: Mon, 22 Jul 2024 14:42:22 +0200 Subject: [PATCH 91/97] Ensure venv usage --- tests/pytests/unit/modules/test_transactional_update.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pytests/unit/modules/test_transactional_update.py b/tests/pytests/unit/modules/test_transactional_update.py index 6d896d4cfbfd..0f08e32ec59b 100644 --- a/tests/pytests/unit/modules/test_transactional_update.py +++ b/tests/pytests/unit/modules/test_transactional_update.py @@ -1,4 +1,5 @@ import os +import sys import pytest @@ -13,7 +14,7 @@ ] SALT_CALL_BINARY = "salt-call" -if os.environ.get("VIRTUAL_ENV"): +if os.environ.get("VIRTUAL_ENV") and os.environ.get("VIRTUAL_ENV") in sys.executable: SALT_CALL_BINARY = f"{os.environ.get('VIRTUAL_ENV')}/bin/salt-call" From 7a871376a269efc8acc21eb17598ed75efca55ed Mon Sep 17 00:00:00 2001 From: Marek Czernek Date: Mon, 22 Jul 2024 16:01:24 +0200 Subject: [PATCH 92/97] Reverts changes to test_transactional_update --- .../unit/modules/test_transactional_update.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/tests/pytests/unit/modules/test_transactional_update.py b/tests/pytests/unit/modules/test_transactional_update.py index 0f08e32ec59b..eb6dfae28684 100644 --- a/tests/pytests/unit/modules/test_transactional_update.py +++ b/tests/pytests/unit/modules/test_transactional_update.py @@ -1,6 +1,3 @@ -import os -import sys - import pytest import salt.loader.context @@ -13,10 +10,6 @@ pytest.mark.skip_on_windows(reason="Not supported on Windows"), ] -SALT_CALL_BINARY = "salt-call" -if os.environ.get("VIRTUAL_ENV") and os.environ.get("VIRTUAL_ENV") in sys.executable: - SALT_CALL_BINARY = f"{os.environ.get('VIRTUAL_ENV')}/bin/salt-call" - @pytest.fixture def configure_loader_modules(): @@ -388,7 +381,7 @@ def test_call_fails_function(): "--continue", "--quiet", "run", - SALT_CALL_BINARY, + "salt-call", "--out", "json", "-l", @@ -420,7 +413,7 @@ def test_call_success_no_reboot(): "--continue", "--quiet", "run", - SALT_CALL_BINARY, + "salt-call", "--out", "json", "-l", @@ -463,7 +456,7 @@ def test_call_success_reboot(): "--continue", "--quiet", "run", - SALT_CALL_BINARY, + "salt-call", "--out", "json", "-l", @@ -497,7 +490,7 @@ def test_call_success_parameters(): "--continue", "--quiet", "run", - SALT_CALL_BINARY, + "salt-call", "--out", "json", "-l", From 9922b1a1457ecbbc05d56ff59927898dfa2d4203 Mon Sep 17 00:00:00 2001 From: Marek Czernek Date: Fri, 26 Jul 2024 16:15:28 +0200 Subject: [PATCH 93/97] Fix yamllint tests --- tests/pytests/functional/utils/yamllint/test_yamllint.py | 2 +- tests/pytests/unit/modules/test_yaml.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytests/functional/utils/yamllint/test_yamllint.py b/tests/pytests/functional/utils/yamllint/test_yamllint.py index 403c6fc610ea..3c730523c4db 100644 --- a/tests/pytests/functional/utils/yamllint/test_yamllint.py +++ b/tests/pytests/functional/utils/yamllint/test_yamllint.py @@ -7,7 +7,7 @@ try: import salt.utils.yamllint as yamllint - YAMLLINT_AVAILABLE = True + YAMLLINT_AVAILABLE = yamllint.has_yamllint() except ImportError: YAMLLINT_AVAILABLE = False diff --git a/tests/pytests/unit/modules/test_yaml.py b/tests/pytests/unit/modules/test_yaml.py index 1f00af710c84..75bad8b5cf1a 100644 --- a/tests/pytests/unit/modules/test_yaml.py +++ b/tests/pytests/unit/modules/test_yaml.py @@ -13,7 +13,7 @@ import salt.modules.yaml import salt.utils.yamllint - YAMLLINT_AVAILABLE = True + YAMLLINT_AVAILABLE = salt.utils.yamllint.has_yamllint() except ImportError: YAMLLINT_AVAILABLE = False From caf62f97bd8f7b6931e462221bfbf662ea1001b6 Mon Sep 17 00:00:00 2001 From: Tom Doherty Date: Wed, 7 Aug 2024 15:18:41 +0100 Subject: [PATCH 94/97] fix yaml output In b9be2de, OrderedDict was replaced with HashableOrderedDict. Add logic to yamldumper to handle the new type. --- changelog/66783.fixed.md | 1 + salt/state.py | 7 +-- salt/utils/odict.py | 5 ++ salt/utils/yamldumper.py | 4 +- tests/unit/utils/test_yamldumper.py | 81 +++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 7 deletions(-) create mode 100644 changelog/66783.fixed.md diff --git a/changelog/66783.fixed.md b/changelog/66783.fixed.md new file mode 100644 index 000000000000..2bd08e3411d3 --- /dev/null +++ b/changelog/66783.fixed.md @@ -0,0 +1 @@ +fix yaml output diff --git a/salt/state.py b/salt/state.py index d60d78c334fe..e515e368b180 100644 --- a/salt/state.py +++ b/salt/state.py @@ -51,7 +51,7 @@ from salt.serializers.msgpack import deserialize as msgpack_deserialize from salt.serializers.msgpack import serialize as msgpack_serialize from salt.template import compile_template, compile_template_str -from salt.utils.odict import DefaultOrderedDict, OrderedDict +from salt.utils.odict import DefaultOrderedDict, HashableOrderedDict log = logging.getLogger(__name__) @@ -127,11 +127,6 @@ ).union(STATE_RUNTIME_KEYWORDS) -class HashableOrderedDict(OrderedDict): - def __hash__(self): - return id(self) - - def split_low_tag(tag): """ Take a low tag and split it back into the low dict that it came from diff --git a/salt/utils/odict.py b/salt/utils/odict.py index 2834f1d92469..11a3f3a30977 100644 --- a/salt/utils/odict.py +++ b/salt/utils/odict.py @@ -62,3 +62,8 @@ def __repr__(self, _repr_running={}): # pylint: disable=W0102 return "DefaultOrderedDict({}, {})".format( self.default_factory, super().__repr__() ) + + +class HashableOrderedDict(OrderedDict): + def __hash__(self): + return id(self) diff --git a/salt/utils/yamldumper.py b/salt/utils/yamldumper.py index 8c6e40394a35..8e694ab4a763 100644 --- a/salt/utils/yamldumper.py +++ b/salt/utils/yamldumper.py @@ -13,7 +13,7 @@ import yaml # pylint: disable=blacklisted-import import salt.utils.context -from salt.utils.odict import OrderedDict +from salt.utils.odict import HashableOrderedDict, OrderedDict try: from yaml import CDumper as Dumper @@ -71,7 +71,9 @@ def represent_undefined(dumper, data): OrderedDumper.add_representer(OrderedDict, represent_ordereddict) +OrderedDumper.add_representer(HashableOrderedDict, represent_ordereddict) SafeOrderedDumper.add_representer(OrderedDict, represent_ordereddict) +SafeOrderedDumper.add_representer(HashableOrderedDict, represent_ordereddict) SafeOrderedDumper.add_representer(None, represent_undefined) OrderedDumper.add_representer( diff --git a/tests/unit/utils/test_yamldumper.py b/tests/unit/utils/test_yamldumper.py index 9a1a6ab103ba..65c900ebb25e 100644 --- a/tests/unit/utils/test_yamldumper.py +++ b/tests/unit/utils/test_yamldumper.py @@ -2,7 +2,11 @@ Unit tests for salt.utils.yamldumper """ +from collections import OrderedDict, defaultdict + import salt.utils.yamldumper +from salt.utils.context import NamespacedDictWrapper +from salt.utils.odict import HashableOrderedDict from tests.support.unit import TestCase @@ -35,3 +39,80 @@ def test_yaml_safe_dump(self): salt.utils.yamldumper.safe_dump(data, default_flow_style=False) == "foo: bar\n" ) + + def test_yaml_ordered_dump(self): + """ + Test yaml.dump with OrderedDict + """ + data = OrderedDict([("foo", "bar"), ("baz", "qux")]) + exp_yaml = "{foo: bar, baz: qux}\n" + assert ( + salt.utils.yamldumper.dump(data, Dumper=salt.utils.yamldumper.OrderedDumper) + == exp_yaml + ) + + def test_yaml_safe_ordered_dump(self): + """ + Test yaml.safe_dump with OrderedDict + """ + data = OrderedDict([("foo", "bar"), ("baz", "qux")]) + exp_yaml = "{foo: bar, baz: qux}\n" + assert salt.utils.yamldumper.safe_dump(data) == exp_yaml + + def test_yaml_indent_safe_ordered_dump(self): + """ + Test yaml.dump with IndentedSafeOrderedDumper + """ + data = OrderedDict([("foo", ["bar", "baz"]), ("qux", "quux")]) + exp_yaml = "foo:\n- bar\n- baz\nqux: quux\n" + assert ( + salt.utils.yamldumper.dump( + data, + Dumper=salt.utils.yamldumper.IndentedSafeOrderedDumper, + default_flow_style=False, + ) + == exp_yaml + ) + + def test_yaml_defaultdict_dump(self): + """ + Test yaml.dump with defaultdict + """ + data = defaultdict(list) + data["foo"].append("bar") + exp_yaml = "foo: [bar]\n" + assert salt.utils.yamldumper.safe_dump(data) == exp_yaml + + def test_yaml_namespaced_dict_wrapper_dump(self): + """ + Test yaml.dump with NamespacedDictWrapper + """ + data = NamespacedDictWrapper({"test": {"foo": "bar"}}, "test") + exp_yaml = ( + "!!python/object/new:salt.utils.context.NamespacedDictWrapper\n" + "dictitems: {foo: bar}\n" + "state:\n" + " _NamespacedDictWrapper__dict:\n" + " test: {foo: bar}\n" + " pre_keys: !!python/tuple [test]\n" + ) + assert salt.utils.yamldumper.dump(data) == exp_yaml + + def test_yaml_undefined_dump(self): + """ + Test yaml.safe_dump with None + """ + data = {"foo": None} + exp_yaml = "{foo: null}\n" + assert salt.utils.yamldumper.safe_dump(data) == exp_yaml + + def test_yaml_hashable_ordered_dict_dump(self): + """ + Test yaml.dump with HashableOrderedDict + """ + data = HashableOrderedDict([("foo", "bar"), ("baz", "qux")]) + exp_yaml = "{foo: bar, baz: qux}\n" + assert ( + salt.utils.yamldumper.dump(data, Dumper=salt.utils.yamldumper.OrderedDumper) + == exp_yaml + ) From a9c9aa294cc9277744e844e35abad4be4f4d7a0a Mon Sep 17 00:00:00 2001 From: Sebastian Engel Date: Sat, 31 Aug 2024 12:36:17 +0200 Subject: [PATCH 95/97] doc: fix jinja.import_json module docstr --- salt/modules/jinja.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/jinja.py b/salt/modules/jinja.py index 86187c29f2bb..db8082e676fc 100644 --- a/salt/modules/jinja.py +++ b/salt/modules/jinja.py @@ -100,7 +100,7 @@ def import_json(path): .. code-block:: bash - salt myminion jinja.import_JSON myformula/foo.json + salt myminion jinja.import_json myformula/foo.json """ tmplstr = textwrap.dedent( """\ From 818dd9f8271dc5fa43aa842166877787bc7149de Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sun, 22 Sep 2024 15:02:26 -0700 Subject: [PATCH 96/97] Fix yamldumper test --- tests/pytests/unit/utils/test_yamldumper.py | 119 ++++++++++++++++++++ tests/unit/utils/test_yamldumper.py | 118 ------------------- 2 files changed, 119 insertions(+), 118 deletions(-) create mode 100644 tests/pytests/unit/utils/test_yamldumper.py delete mode 100644 tests/unit/utils/test_yamldumper.py diff --git a/tests/pytests/unit/utils/test_yamldumper.py b/tests/pytests/unit/utils/test_yamldumper.py new file mode 100644 index 000000000000..7f73c5d1e7ec --- /dev/null +++ b/tests/pytests/unit/utils/test_yamldumper.py @@ -0,0 +1,119 @@ +""" + Unit tests for salt.utils.yamldumper +""" + +from collections import OrderedDict, defaultdict + +import salt.utils.yamldumper +from salt.utils.context import NamespacedDictWrapper +from salt.utils.odict import HashableOrderedDict + + +def test_yaml_dump(): + """ + Test yaml.dump a dict + """ + data = {"foo": "bar"} + exp_yaml = "{foo: bar}\n" + + assert salt.utils.yamldumper.dump(data) == exp_yaml + + assert salt.utils.yamldumper.dump( + data, default_flow_style=False + ) == exp_yaml.replace("{", "").replace("}", "") + + +def test_yaml_safe_dump(): + """ + Test yaml.safe_dump a dict + """ + data = {"foo": "bar"} + assert salt.utils.yamldumper.safe_dump(data) == "{foo: bar}\n" + + assert ( + salt.utils.yamldumper.safe_dump(data, default_flow_style=False) == "foo: bar\n" + ) + + +def test_yaml_ordered_dump(): + """ + Test yaml.dump with OrderedDict + """ + data = OrderedDict([("foo", "bar"), ("baz", "qux")]) + exp_yaml = "{foo: bar, baz: qux}\n" + assert ( + salt.utils.yamldumper.dump(data, Dumper=salt.utils.yamldumper.OrderedDumper) + == exp_yaml + ) + + +def test_yaml_safe_ordered_dump(): + """ + Test yaml.safe_dump with OrderedDict + """ + data = OrderedDict([("foo", "bar"), ("baz", "qux")]) + exp_yaml = "{foo: bar, baz: qux}\n" + assert salt.utils.yamldumper.safe_dump(data) == exp_yaml + + +def test_yaml_indent_safe_ordered_dump(): + """ + Test yaml.dump with IndentedSafeOrderedDumper + """ + data = OrderedDict([("foo", ["bar", "baz"]), ("qux", "quux")]) + exp_yaml = "foo:\n- bar\n- baz\nqux: quux\n" + assert ( + salt.utils.yamldumper.dump( + data, + Dumper=salt.utils.yamldumper.IndentedSafeOrderedDumper, + default_flow_style=False, + ) + == exp_yaml + ) + + +def test_yaml_defaultdict_dump(): + """ + Test yaml.dump with defaultdict + """ + data = defaultdict(list) + data["foo"].append("bar") + exp_yaml = "foo: [bar]\n" + assert salt.utils.yamldumper.safe_dump(data) == exp_yaml + + +def test_yaml_namespaced_dict_wrapper_dump(): + """ + Test yaml.dump with NamespacedDictWrapper + """ + data = NamespacedDictWrapper({"test": {"foo": "bar"}}, "test") + exp_yaml = ( + "!!python/object/new:salt.utils.context.NamespacedDictWrapper\n" + "dictitems: {foo: bar}\n" + "state:\n" + " _NamespacedDictWrapper__dict:\n" + " test: {foo: bar}\n" + " pre_keys: !!python/tuple [test]\n" + ) + assert salt.utils.yamldumper.dump(data) == exp_yaml + + +def test_yaml_undefined_dump(): + """ + Test yaml.safe_dump with None + """ + data = {"foo": None} + exp_yaml = "{foo: null}\n" + assert salt.utils.yamldumper.safe_dump(data) == exp_yaml + + +def test_yaml_hashable_ordered_dict_dump(): + """ + Test yaml.dump with HashableOrderedDict + """ + data = HashableOrderedDict([("foo", "bar"), ("baz", "qux")]) + exp_yaml = "{foo: bar, baz: qux}\n" + assert ( + salt.utils.yamldumper.dump(data, Dumper=salt.utils.yamldumper.OrderedDumper) + == exp_yaml + ) diff --git a/tests/unit/utils/test_yamldumper.py b/tests/unit/utils/test_yamldumper.py deleted file mode 100644 index 65c900ebb25e..000000000000 --- a/tests/unit/utils/test_yamldumper.py +++ /dev/null @@ -1,118 +0,0 @@ -""" - Unit tests for salt.utils.yamldumper -""" - -from collections import OrderedDict, defaultdict - -import salt.utils.yamldumper -from salt.utils.context import NamespacedDictWrapper -from salt.utils.odict import HashableOrderedDict -from tests.support.unit import TestCase - - -class YamlDumperTestCase(TestCase): - """ - TestCase for salt.utils.yamldumper module - """ - - def test_yaml_dump(self): - """ - Test yaml.dump a dict - """ - data = {"foo": "bar"} - exp_yaml = "{foo: bar}\n" - - assert salt.utils.yamldumper.dump(data) == exp_yaml - - assert salt.utils.yamldumper.dump( - data, default_flow_style=False - ) == exp_yaml.replace("{", "").replace("}", "") - - def test_yaml_safe_dump(self): - """ - Test yaml.safe_dump a dict - """ - data = {"foo": "bar"} - assert salt.utils.yamldumper.safe_dump(data) == "{foo: bar}\n" - - assert ( - salt.utils.yamldumper.safe_dump(data, default_flow_style=False) - == "foo: bar\n" - ) - - def test_yaml_ordered_dump(self): - """ - Test yaml.dump with OrderedDict - """ - data = OrderedDict([("foo", "bar"), ("baz", "qux")]) - exp_yaml = "{foo: bar, baz: qux}\n" - assert ( - salt.utils.yamldumper.dump(data, Dumper=salt.utils.yamldumper.OrderedDumper) - == exp_yaml - ) - - def test_yaml_safe_ordered_dump(self): - """ - Test yaml.safe_dump with OrderedDict - """ - data = OrderedDict([("foo", "bar"), ("baz", "qux")]) - exp_yaml = "{foo: bar, baz: qux}\n" - assert salt.utils.yamldumper.safe_dump(data) == exp_yaml - - def test_yaml_indent_safe_ordered_dump(self): - """ - Test yaml.dump with IndentedSafeOrderedDumper - """ - data = OrderedDict([("foo", ["bar", "baz"]), ("qux", "quux")]) - exp_yaml = "foo:\n- bar\n- baz\nqux: quux\n" - assert ( - salt.utils.yamldumper.dump( - data, - Dumper=salt.utils.yamldumper.IndentedSafeOrderedDumper, - default_flow_style=False, - ) - == exp_yaml - ) - - def test_yaml_defaultdict_dump(self): - """ - Test yaml.dump with defaultdict - """ - data = defaultdict(list) - data["foo"].append("bar") - exp_yaml = "foo: [bar]\n" - assert salt.utils.yamldumper.safe_dump(data) == exp_yaml - - def test_yaml_namespaced_dict_wrapper_dump(self): - """ - Test yaml.dump with NamespacedDictWrapper - """ - data = NamespacedDictWrapper({"test": {"foo": "bar"}}, "test") - exp_yaml = ( - "!!python/object/new:salt.utils.context.NamespacedDictWrapper\n" - "dictitems: {foo: bar}\n" - "state:\n" - " _NamespacedDictWrapper__dict:\n" - " test: {foo: bar}\n" - " pre_keys: !!python/tuple [test]\n" - ) - assert salt.utils.yamldumper.dump(data) == exp_yaml - - def test_yaml_undefined_dump(self): - """ - Test yaml.safe_dump with None - """ - data = {"foo": None} - exp_yaml = "{foo: null}\n" - assert salt.utils.yamldumper.safe_dump(data) == exp_yaml - - def test_yaml_hashable_ordered_dict_dump(self): - """ - Test yaml.dump with HashableOrderedDict - """ - data = HashableOrderedDict([("foo", "bar"), ("baz", "qux")]) - exp_yaml = "{foo: bar, baz: qux}\n" - assert ( - salt.utils.yamldumper.dump(data, Dumper=salt.utils.yamldumper.OrderedDumper) - == exp_yaml - ) From b6e3f9b2c672fb89ab1efc6835e5a788526c7c8a Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sun, 22 Sep 2024 23:28:58 -0700 Subject: [PATCH 97/97] Fix test on platforms that do not have cdumper --- tests/pytests/unit/utils/test_yamldumper.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/pytests/unit/utils/test_yamldumper.py b/tests/pytests/unit/utils/test_yamldumper.py index 7f73c5d1e7ec..09a1106f5455 100644 --- a/tests/pytests/unit/utils/test_yamldumper.py +++ b/tests/pytests/unit/utils/test_yamldumper.py @@ -61,7 +61,11 @@ def test_yaml_indent_safe_ordered_dump(): Test yaml.dump with IndentedSafeOrderedDumper """ data = OrderedDict([("foo", ["bar", "baz"]), ("qux", "quux")]) - exp_yaml = "foo:\n- bar\n- baz\nqux: quux\n" + # Account for difference in SafeDumper vs CSafeDumper + if salt.utils.yamldumper.SafeDumper.__name__ == "SafeDumper": + exp_yaml = "foo:\n - bar\n - baz\nqux: quux\n" + else: + exp_yaml = "foo:\n- bar\n- baz\nqux: quux\n" assert ( salt.utils.yamldumper.dump( data,