From 8543b4d35c097acf8bc4ccf2cbd4ed6f3a10b065 Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Thu, 24 Aug 2023 12:30:14 +0200 Subject: [PATCH 01/11] tests: update dockerfile to rockylinux:8 --- Dockerfile | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 57fb842113..59a96c0622 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,35 @@ -# Use an official centos image as a parent image -FROM centos:7 +# Use an official RockyLinux image as a parent image +FROM rockylinux:8 # Set the working directory to install dependencies to /quattor WORKDIR /quattor # install library core in /quattor, tests need it -ADD https://codeload.github.com/quattor/template-library-core/tar.gz/master /quattor/template-library-core-master.tar.gz -RUN tar xvfz template-library-core-master.tar.gz +ADD https://codeload.github.com/quattor/template-library-core/tar.gz/master template-library-core-master.tar.gz +RUN tar -xzf template-library-core-master.tar.gz -# Install dependencies -RUN yum install -y maven epel-release -RUN rpm -U http://yum.quattor.org/devel/quattor-release-1-1.noarch.rpm +# point library core to where we downloaded it +ENV QUATTOR_TEST_TEMPLATE_LIBRARY_CORE /quattor/template-library-core-master + +# Prepare to install dependencies +RUN dnf -y install dnf-plugins-core && \ + dnf config-manager --set-enabled appstream && \ + dnf config-manager --set-enabled powertools && \ + dnf -y install epel-release http://yum.quattor.org/devel/quattor-yum-repo-2-1.noarch.rpm # The available version of perl-Test-Quattor is too old for mvnprove.pl to # work, but this is a quick way of pulling in a lot of required dependencies. # Surprisingly `which` is not installed by default and panc depends on it. # libselinux-utils is required for /usr/sbin/selinuxenabled -RUN yum install -y perl-Test-Quattor which panc aii-ks ncm-lib-blockdevices \ - ncm-ncd git libselinux-utils sudo perl-Crypt-OpenSSL-X509 \ - perl-Data-Compare perl-Date-Manip perl-File-Touch perl-JSON-Any \ - perl-Net-DNS perl-Net-FreeIPA perl-Net-OpenNebula \ - perl-Net-OpenStack-Client perl-NetAddr-IP perl-REST-Client \ - perl-Set-Scalar perl-Text-Glob -#perl-Git-Repository perl-Data-Structure-Util -# Hack around the two missing Perl rpms for ncm-ceph -RUN yum install -y cpanminus gcc -RUN cpanm install Git::Repository Data::Structure::Util - -# point library core to where we downloaded it -ENV QUATTOR_TEST_TEMPLATE_LIBRARY_CORE /quattor/template-library-core-master +RUN dnf install -y maven which rpm-build panc ncm-lib-blockdevices \ + ncm-ncd git libselinux-utils sudo perl-Crypt-OpenSSL-X509 \ + perl-Data-Compare perl-Date-Manip perl-File-Touch perl-JSON-Any \ + perl-Net-DNS perl-Net-FreeIPA perl-Net-OpenNebula \ + perl-Net-OpenStack-Client perl-NetAddr-IP perl-REST-Client \ + perl-Set-Scalar perl-Text-Glob cpanminus gcc wget \ + perl-Git-Repository perl-Data-Structure-Util \ + http://yum.quattor.org/devel/perl-Test-Quattor-18.3.0-SNAPSHOT20180406083650.noarch.rpm \ + http://yum.quattor.org/devel/aii-ks-21.12.1-SNAPSHOT20230627130118.noarch.rpm # set workdir to where we'll run the tests COPY --chown=99 . /quattor_test From 3a8818d7323406d2eae8cec76bd23d41c06b3eb7 Mon Sep 17 00:00:00 2001 From: Michael MELLIN Date: Wed, 21 Feb 2024 17:24:09 +0100 Subject: [PATCH 02/11] ncm-mysql: add an option to use encrypted user password --- .../src/main/pan/components/mysql/schema.pan | 2 ++ ncm-mysql/src/main/perl/mysql.pm | 17 +++++++++++++---- ncm-mysql/src/main/perl/mysql.pod | 4 +++- ncm-mysql/src/test/resources/basic_service.pan | 3 +++ 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/ncm-mysql/src/main/pan/components/mysql/schema.pan b/ncm-mysql/src/main/pan/components/mysql/schema.pan index f7ff7644f3..0361deb6ea 100755 --- a/ncm-mysql/src/main/pan/components/mysql/schema.pan +++ b/ncm-mysql/src/main/pan/components/mysql/schema.pan @@ -59,6 +59,7 @@ type component_mysql_db_user = { 'password' : string with (length(SELF) == 0) || component_mysql_password_valid(SELF) 'rights' : component_mysql_user_right[] = list('SELECT') 'shortPwd' : boolean = false + 'encrypted_pwd' : boolean = false }; @@ -84,6 +85,7 @@ type component_mysql_server_options = { 'host' ? string 'adminuser' : string 'adminpwd' : string with component_mysql_password_valid(SELF) + 'encrypted_adminpwd' : boolean = false 'options' ? string{} 'users' ? component_mysql_db_user{} }; diff --git a/ncm-mysql/src/main/perl/mysql.pm b/ncm-mysql/src/main/perl/mysql.pm index 42897c99dc..3af3e8f6d8 100644 --- a/ncm-mysql/src/main/perl/mysql.pm +++ b/ncm-mysql/src/main/perl/mysql.pm @@ -231,7 +231,7 @@ sub Configure my $user = unescape($user_e); $self->info("Granting user $user access to all databases on server $server_name..."); my $user_params = $server->{users}->{$user_e}; - if ( $self->mysqlAddUser(undef,$user,$user_params->{password},$user_params->{rights},$user_params->{shortPwd},$server) ) { + if ( $self->mysqlAddUser(undef,$user,$user_params->{password},$user_params->{rights},$user_params->{shortPwd},$user_params->{encrypted_pwd},$server) ) { $self->error("Error granting user $user access to all databases on server $server_name"); next; } @@ -302,7 +302,7 @@ sub Configure my $user = unescape($user_e); $self->info("Configuring user $user access to database $database..."); my $user_params = $databases->{$database}->{users}->{$user_e}; - if ( $self->mysqlAddUser($database,$user,$user_params->{password},$user_params->{rights},$user_params->{shortPwd}) ) { + if ( $self->mysqlAddUser($database,$user,$user_params->{password},$user_params->{rights},$user_params->{shortPwd},$user_params->{encrypted_pwd}) ) { $self->error("Error granting user $user access to database $database"); next; } @@ -404,6 +404,7 @@ sub mysqlCheckAdminPwd # First check if administrator account is working without password for either the specified server host or localhost my $admin_pwd_saved = $server->{adminpwd}; + my $admin_pwd_encrypted = $server->{encrypted_adminpwd}; my $server_host_saved = $server->{host}; $server->{adminpwd} = ''; my @db_hosts = ($server->{host}, 'localhost'); @@ -445,7 +446,7 @@ sub mysqlCheckAdminPwd # If it fails, try to change it assuming a password has not yet been set (even if previous test failed) if ( $status ) { $self->debug(1,"$function_name : trying to set administrator password on $server->{host}"); - $status = $self->mysqlAddUser(undef,$server->{adminuser},$admin_pwd_saved,'ALL',0,$server); + $status = $self->mysqlAddUser(undef,$server->{adminuser},$admin_pwd_saved,'ALL',0,$admin_pwd_encrypted,$server); if ( $status ) { if ( ($server->{host} ne $this_host_full) && ($server->{host} ne 'localhost') ) { $self->warn("Error setting administrator password on server $server->{host} ", @@ -461,6 +462,7 @@ sub mysqlCheckAdminPwd } $server->{adminpwd} = $admin_pwd_saved; + $server->{encrypted_adminpwd} = $admin_pwd_encrypted; $server->{adminhost} = undef; return $status; @@ -481,7 +483,7 @@ sub mysqlCheckAdminPwd sub mysqlAddUser { my $function_name = "mysqlAddUser"; - my ($self,$database,$db_user,$db_pwd,$db_rights,$short_pwd_hash,$server) = @_; + my ($self,$database,$db_user,$db_pwd,$db_rights,$short_pwd_hash,$encrypted_pwd,$server) = @_; if ( $database ) { if ( $server ) { @@ -509,6 +511,10 @@ sub mysqlAddUser return 0; } + unless ( defined($encrypted_pwd) ) { + $encrypted_pwd = 0; + } + unless ( $db_rights ) { $self->error("$function_name : 'db_rights' argument missing"); return 0; @@ -556,6 +562,9 @@ sub mysqlAddUser $self->debug(1, "$function_name : Adding MySQL connection account for user $userid on $host ", "(database=$database) using admin host $admin_server->{host}"); my $cmd = "grant $db_rights on $database to \"$userid\"\@\"$host\" identified by \"$db_pwd\" with grant option"; + if ( $encrypted_pwd ) { + $cmd = "grant $db_rights on $database to \"$userid\"\@\"$host\" identified by password \"$db_pwd\" with grant option"; + } $status = $self->mysqlExecCmd($admin_server, $cmd, undef, {$db_pwd => 'USERPASSWORD'}); if ( $status ) { # Error already signaled by caller diff --git a/ncm-mysql/src/main/perl/mysql.pod b/ncm-mysql/src/main/perl/mysql.pod index 2b07a7074b..4406152187 100755 --- a/ncm-mysql/src/main/perl/mysql.pod +++ b/ncm-mysql/src/main/perl/mysql.pod @@ -45,10 +45,12 @@ Value is a nlist with the following possible keys : =over -=item password : user MySQL password. Must be a cleartext password. +=item password : user MySQL password. Must be a cleartext password if encrypted_pwd is false (default value) or an encrypted password (using mysql command : select password('my_secret_password');). =item rights : list of MySQL privileges to grant to the user. +=item encrypted_pwd : boolean (optional). If true, password will be used as an encrypted value. + =back =head1 serviceName option diff --git a/ncm-mysql/src/test/resources/basic_service.pan b/ncm-mysql/src/test/resources/basic_service.pan index 6d72dcb591..f9f699d59a 100644 --- a/ncm-mysql/src/test/resources/basic_service.pan +++ b/ncm-mysql/src/test/resources/basic_service.pan @@ -11,6 +11,9 @@ prefix "/software/components/mysql/databases/opennebula"; "server" = "one"; "users/oneadmin/password" = 'p4ss'; "users/oneadmin/rights" = list("ALL PRIVILEGES"); +"users/oneuser/password" = '*2F01F3D078AE27EB3017F8F53DF9C31AEA6D90C5'; # clear password : plop +"users/oneuser/encrypted_adminpwd" = true; +"users/oneuser/rights" = list("ALL PRIVILEGES"); "createDb" = false; "initScript/file" = "/dev/null"; From a1ec2e508b265d6093349237561435ea490c42cb Mon Sep 17 00:00:00 2001 From: stdweird Date: Fri, 19 Jul 2024 14:58:06 +0200 Subject: [PATCH 03/11] ncm-systemd: systemctl show can doublequoted backslash values --- ncm-systemd/src/main/perl/Systemd/Systemctl.pm | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/ncm-systemd/src/main/perl/Systemd/Systemctl.pm b/ncm-systemd/src/main/perl/Systemd/Systemctl.pm index 4742e0776e..a82e6eae5a 100644 --- a/ncm-systemd/src/main/perl/Systemd/Systemctl.pm +++ b/ncm-systemd/src/main/perl/Systemd/Systemctl.pm @@ -148,8 +148,19 @@ sub systemctl_show while($output =~ m/^([^=\s]+)\s*=(.*)?$/mg) { my ($k,$v) = ($1,"$2"); if (grep {$_ eq $k} @PROPERTIES_ARRAY) { - my @values = split(/\s+/, $v); - $res->{$k} = \@values; + my $values = []; + foreach my $value (split(/\s+/, $v)) { + if ($value =~ m/^"(.*)"$/) { + $value = $1; + if ($value =~ m/\\{2}/) { + $value =~ s/\\{2}/\\/g; + } else { + $logger->error("Found double quoted value '$value' but no expected quoting pattern from '$k = $v'") + } + } + push(@$values, $value); + }; + $res->{$k} = $values; } else { $res->{$k} = $v; } From 0de697c129cd8ce3d892ea6fc68a2a47961b39d0 Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Mon, 19 Aug 2024 14:03:45 +0200 Subject: [PATCH 04/11] fix tests: add procps-ng for sysctl --- .github/workflows/continuous-integration.yaml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index 93f610e7e4..42ac29c34d 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -29,7 +29,7 @@ jobs: perl-Net-DNS perl-Net-FreeIPA perl-Net-OpenNebula \ perl-Net-OpenStack-Client perl-NetAddr-IP perl-REST-Client \ perl-Set-Scalar perl-Text-Glob cpanminus gcc wget \ - perl-Git-Repository perl-Data-Structure-Util \ + perl-Git-Repository perl-Data-Structure-Util procps-ng \ http://yum.quattor.org/devel/perl-Test-Quattor-18.3.0-SNAPSHOT20180406083650.noarch.rpm \ http://yum.quattor.org/devel/aii-ks-21.12.1-SNAPSHOT20230627130118.noarch.rpm - name: set up template library core from git master diff --git a/Dockerfile b/Dockerfile index 59a96c0622..e318437718 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN dnf install -y maven which rpm-build panc ncm-lib-blockdevices \ perl-Net-DNS perl-Net-FreeIPA perl-Net-OpenNebula \ perl-Net-OpenStack-Client perl-NetAddr-IP perl-REST-Client \ perl-Set-Scalar perl-Text-Glob cpanminus gcc wget \ - perl-Git-Repository perl-Data-Structure-Util \ + perl-Git-Repository perl-Data-Structure-Util procps-ng \ http://yum.quattor.org/devel/perl-Test-Quattor-18.3.0-SNAPSHOT20180406083650.noarch.rpm \ http://yum.quattor.org/devel/aii-ks-21.12.1-SNAPSHOT20230627130118.noarch.rpm From fa6cc17f6032553af6cfd1f8f40c1af8dfb14822 Mon Sep 17 00:00:00 2001 From: gregcorbett Date: Wed, 21 Aug 2024 16:43:13 +0100 Subject: [PATCH 05/11] ncm-metaconfig: Allow basic httpd remoteip config to be expressed - See https://httpd.apache.org/docs/2.4/mod/mod_remoteip.html - Added to the global section because, even though `remoteipproxyprotocol` can be defined at the vhost level, only one such definition will take affect, and do so globally. --- ncm-metaconfig/src/main/metaconfig/httpd/config/global.tt | 5 ++++- ncm-metaconfig/src/main/metaconfig/httpd/pan/schema.pan | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ncm-metaconfig/src/main/metaconfig/httpd/config/global.tt b/ncm-metaconfig/src/main/metaconfig/httpd/config/global.tt index 03ce6632ab..47b374d7b1 100644 --- a/ncm-metaconfig/src/main/metaconfig/httpd/config/global.tt +++ b/ncm-metaconfig/src/main/metaconfig/httpd/config/global.tt @@ -3,7 +3,10 @@ "hostnamelookups", ] -%] -[%- lists = ["directoryindex", "indexoptions", "indexignore"] -%] +[%- lists = ["directoryindex", "indexoptions", "indexignore", + "remoteiptrustedproxy", "remoteipproxyprotocolexceptions" + ] +-%] [%- quotes = ["serverroot"] -%] [%- FOREACH pair IN desc.pairs -%] [%- SWITCH pair.key -%] diff --git a/ncm-metaconfig/src/main/metaconfig/httpd/pan/schema.pan b/ncm-metaconfig/src/main/metaconfig/httpd/pan/schema.pan index 9940269cab..a6bf46e17e 100644 --- a/ncm-metaconfig/src/main/metaconfig/httpd/pan/schema.pan +++ b/ncm-metaconfig/src/main/metaconfig/httpd/pan/schema.pan @@ -680,6 +680,10 @@ type httpd_global_system = { "limitrequestfieldsize" ? long "traceenable" ? string with match(SELF, '^(on|off|extended)$') + + "remoteipproxyprotocol" ? choice('on', 'off') + "remoteipproxyprotocolexceptions" ? type_network_name[] + "remoteiptrustedproxy" ? type_network_name[] }; type httpd_ifmodule_parameters = { From ebbf0d7793f28a4ee07464cd3b06765547d181ac Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Thu, 22 Aug 2024 13:27:33 +0200 Subject: [PATCH 06/11] tests: update github actions --- .github/workflows/continuous-integration.yaml | 4 ++-- .github/workflows/panlint.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index 42ac29c34d..c8a41e60e4 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -8,7 +8,7 @@ jobs: container: rockylinux:8 steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Determine hash for caching key id: cachekeystep run: echo "pomcachekey=${{ hashFiles('**/pom.xml') }}" >> $GITHUB_ENV @@ -40,7 +40,7 @@ jobs: tar -xvzf template-library-core-master.tar.gz cd - - name: Cache Maven packages - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /tmp/m2 key: ${{ runner.os }}-m2-${{ env.pomcachekey }} diff --git a/.github/workflows/panlint.yaml b/.github/workflows/panlint.yaml index bbc1ea3a2c..83709f3f98 100644 --- a/.github/workflows/panlint.yaml +++ b/.github/workflows/panlint.yaml @@ -6,12 +6,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: # we need the full repo or HEAD^ does not work fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3 - name: Install dependencies From ecb95dfd5ade48b2c644348f6541b1a08ac89782 Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Wed, 14 Aug 2024 10:18:01 +0200 Subject: [PATCH 07/11] ncm-metaconfig: add schema for slurm 24.05 --- .../src/main/metaconfig/slurm/pan/schema.pan | 2 +- .../metaconfig/slurm/pan/schema_24.05.pan | 907 ++++++++++++++++++ .../slurm/tests/profiles/cgroups.pan | 2 - .../slurm/tests/profiles/config.pan | 10 +- .../metaconfig/slurm/tests/profiles/dbd.pan | 2 +- .../slurm/tests/regexps/cgroups/base | 1 - .../slurm/tests/regexps/config/base | 5 +- .../metaconfig/slurm/tests/regexps/dbd/base | 2 +- .../src/main/metaconfig/slurm/value.tt | 2 +- 9 files changed, 920 insertions(+), 13 deletions(-) create mode 100644 ncm-metaconfig/src/main/metaconfig/slurm/pan/schema_24.05.pan diff --git a/ncm-metaconfig/src/main/metaconfig/slurm/pan/schema.pan b/ncm-metaconfig/src/main/metaconfig/slurm/pan/schema.pan index 60993b735f..54fe00d382 100644 --- a/ncm-metaconfig/src/main/metaconfig/slurm/pan/schema.pan +++ b/ncm-metaconfig/src/main/metaconfig/slurm/pan/schema.pan @@ -1,6 +1,6 @@ declaration template metaconfig/slurm/schema; -variable METACONFIG_SLURM_VERSION ?= '23.02'; +variable METACONFIG_SLURM_VERSION ?= '24.05'; @{include version specific types at the end} include format('metaconfig/slurm/schema_%s', METACONFIG_SLURM_VERSION); diff --git a/ncm-metaconfig/src/main/metaconfig/slurm/pan/schema_24.05.pan b/ncm-metaconfig/src/main/metaconfig/slurm/pan/schema_24.05.pan new file mode 100644 index 0000000000..bf64944a52 --- /dev/null +++ b/ncm-metaconfig/src/main/metaconfig/slurm/pan/schema_24.05.pan @@ -0,0 +1,907 @@ +declaration template metaconfig/slurm/schema_24.05; + +@{ Schema for slurm configuration, see +https://slurm.schedmd.com +} + +include 'pan/types'; + +type slurm_debuglevel = choice('quiet', 'fatal', 'error', 'info', 'verbose', + 'debug', 'debug2', 'debug3', 'debug4', 'debug5'); + +type slurm_debugflags = choice('AuditRPCs', 'Backfill', 'BackfillMap', 'BGBlockAlgo', 'BGBlockAlgoDeep', 'BGBlockPick', + 'BGBlockWires', 'BurstBuffer', 'Cgroup', 'CPU_Bind', 'CpuFrequency', 'DB_ASSOC', 'DB_EVENT', 'DB_JOB', 'DB_QOS', + 'DB_QUERY', 'DB_RESERVATION', 'DB_RESOURCE', 'DB_STEP', 'DB_USAGE', 'DB_WCKEY', 'DBD_Agent', 'Elasticsearch', + 'Energy', 'Federation', 'FrontEnd', 'GLOB_SILENCE', 'Gres', 'HeteroJobs', 'Gang', 'JobAccountGather', + 'JobContainer', 'License', 'NodeFeatures', 'NO_CONF_HASH', 'Power', 'Priority', 'Profile', 'Protocol', + 'Reservation', 'Script', 'SelectType', 'Steps', 'Switch', 'TraceJobs', 'Triggers')[]; + + + +type slurm_gres = { + 'name' : string + 'type' ? string + 'consume' : boolean = true + 'number' : long(0..) +}; + + +@{all intervals in seconds} +type slurm_job_gather_frequency = { + 'energy' ? long(0..) + 'filesystem' ? long(0..) + 'network' ? long(0..) + 'task' ? long(0..) +}; + +type slurm_msg_aggregation = { + 'WindowMsgs' ? long(0..) + 'WindowTime' ? long(0..) +}; + +type slurm_sbcast_parameters = { + 'DestDir' ? absolute_file_path + 'Compression' ? choice("lz4", "none") + 'send_libs' ? boolean +}; + +type slurm_preempt_params = { + 'min_exempt_priority' ? long + 'reclaim_licenses' ? boolean + 'reorder_count' ? long + 'send_user_signal' ? long + 'strict_order' ? boolean + 'youngest_first' ? boolean +}; + +type slurm_dependency_parameters = { + 'disable_remote_singleton' ? boolean + '{kill_invalid_depend}' ? boolean + '{max_depend_depth}' ? long(0..) +}; + +type slurm_scheduler_parameters = { + 'assoc_limit_stop' ? boolean + @{in seconds} + '{batch_sched_delay}' ? long(0..) + 'bb_array_stage_cnt' ? long(0..) + 'bf_busy_nodes' ? boolean # should be set without argument + 'bf_continue' ? boolean # should be set without argument + @{in seconds} + 'bf_interval' ? long(0..) + 'bf_job_part_count_reserve' ? long(0..) + 'bf_licenses' ? boolean + 'bf_max_job_array_resv' ? long(0..) + 'bf_max_job_assoc' ? long(0..) + 'bf_max_job_part' ? long(0..) + 'bf_max_job_start' ? long(0..) + 'bf_max_job_test' ? long(0..) + 'bf_max_job_user' ? long(0..) + 'bf_max_job_user_part' ? long(0..) + 'bf_max_time' ? long(0..256) + 'bf_min_age_reserve' ? long(0..) + 'bf_min_prio_reserve' ? long(0..) + 'bf_node_space_size' ? long(2..2000000) + 'bf_resolution' ? long(0..) + 'bf_window' ? long(0..) + 'bf_window_linear' ? long(0..) + 'bf_yield_interval' ? long(0..) + 'bf_yield_sleep' ? long(0..) + 'build_queue_timeout' ? long(0..) + '{default_queue_depth}' ? long(0..) + 'defer' ? boolean + 'defer_batch' ? boolean + 'delay_boot' ? long(0..) + 'default_gbytes' ? boolean + 'disable_hetero_steps' ? boolean + 'enable_hetero_steps' ? boolean + 'enable_user_top' ? boolean + 'extra_constraints' ? boolean + 'Ignore_NUMA' ? boolean + 'ignore_prefer_validation' ? boolean + 'inventory_interval' ? long(0..) + '{kill_invalid_depend}' ? boolean with { + deprecated(0, 'kill_invalid_depend has moved to DependencyParameters'); true; + } + 'max_array_tasks' ? long(0..) # should be smaller than MaxArraySize + '{max_depend_depth}' ? long(0..) with {deprecated(0, 'max_depend_depth has moved to DependencyParameters'); true; } + 'max_rpc_cnt' ? long(0..) + 'max_sched_time' ? long(0..) + 'max_script_size' ? long(0..) + 'max_submit_line_size' ? long(0..) + 'max_switch_wait' ? long(0..) + '{no_backup_scheduling}' ? boolean + '{no_env_cache}' ? boolean + 'pack_serial_at_end' ? boolean + '{partition_job_depth}' ? long(0..) + 'nohold_on_prolog_fail' ? boolean + 'reduce_completing_frag' ? boolean + 'requeue_setup_env_fail' ? boolean + 'salloc_wait_nodes' ? boolean + 'sbatch_wait_nodes' ? boolean + 'sched_interval' ? long(0..) + 'sched_max_job_start' ? long(0..) + 'sched_min_interval' ? long(0..) + 'spec_cores_first' ? boolean + 'step_retry_count' ? long(0..) + 'step_retry_time' ? long(0..) + 'time_min_as_soft_limit' ? boolean + 'whole_hetjob' ? boolean +}; + +type slurm_select_type_parameters = { + 'NHC_ABSOLUTELY_NO' ? boolean + 'NHC_NO_STEPS' ? boolean + 'NHC_NO' ? boolean + 'CR_CPU' ? boolean + 'CR_CPU_Memory' ? boolean + 'CR_Core' ? boolean + 'CR_Core_Memory' ? boolean + 'CR_ONE_TASK_PER_CORE' ? boolean + 'CR_CORE_DEFAULT_DIST_BLOCK' ? boolean + 'CR_LLN' ? boolean + 'CR_Pack_Nodes' ? boolean + 'CR_Socket' ? boolean + 'CR_Socket_Memory' ? boolean + 'CR_Memory' ? boolean + 'ENFORCE_BINDING_GRES' ? boolean + 'LL_SHARED_GRES' ? boolean + 'MULTIPLE_SHARING_GRES_PJ' ? boolean + 'ONE_TASK_PER_SHARING_GRES' ? boolean +}; + +type slurm_task_plugin_param = { + 'Boards' ? boolean + 'Cores' ? boolean + 'None' ? boolean + 'Sockets' ? boolean + 'Threads' ? boolean + 'SlurmdOffSpec' ? boolean + 'Verbose' ? boolean + 'Autobind' ? boolean +}; + +type slurm_topology_param = { + 'Dragonfly' ? boolean + 'RoutePart' ? boolean + 'RouteTree' ? boolean + 'SwitchAsNodeRank' ? boolean + 'TopoOptional' ? boolean +}; + +type slurm_conf_health_check = { + 'HealthCheckInterval' ? long(0..) + 'HealthCheckNodeState' ? choice('ALLOC', 'ANY', 'CYCLE', 'IDLE', 'MIXED')[] + 'HealthCheckProgram' ? absolute_file_path +}; + + +type slurm_control_resourcelimits = choice( 'ALL', 'NONE', 'AS', 'CORE', 'CPU', 'DATA', 'FSIZE', + 'MEMLOCK', 'NOFILE', 'NPROC', 'RSS', 'STACK'); + +type slurm_mpi_params = { + @{port or port range} + 'ports' ? long(0..)[] with {length(SELF) == 1 || length(SELF) == 2} + 'disable_slurm_hydra_bootstrap' ? boolean +}; + +type slurm_launch_params = { + 'batch_step_set_cpu_freq' ? boolean + 'cray_net_exclusive' ? boolean + 'disable_send_gids' ? boolean + 'enable_nss_slurm' ? boolean + 'lustre_no_flush' ? boolean + 'mem_sort' ? boolean + 'mpir_use_nodeaddr' ? boolean + 'send_gids' ? boolean + 'slurmstepd_memlock' ? boolean + 'slurmstepd_memlock_all' ? boolean + 'test_exec' ? boolean + 'use_interactive_step' ? boolean +}; + +type slurm_job_comp_params = { + 'flush_timeout' ? long(0..) + 'poll_interval' ? long(0..) + 'requeue_on_msg_timeout' ? boolean + 'topic' ? string +}; + +type slurm_authalt_params = { + 'disable_token_creation' ? boolean + 'max_token_lifespan' ? long(0..) + 'jwt_key' ? absolute_file_path + 'jwks' ? absolute_file_path + 'userclaimfield' ? string +}; + +type slurm_communication_params = { + 'block_null_hash' ? boolean + 'CheckGhalQuiesce' ? boolean + 'DisableIPv4' ? boolean + 'EnableIPv6' ? boolean + 'getnameinfo_cache_timeout' ? long(0..) + 'keepaliveinterval' ? long(1..) + 'keepaliveprobes' ? long(1..) + 'keepalivetime' ? long(1..) + 'NoAddrCache' ? boolean + 'NoCtldInAddrAny' ? boolean + 'NoInAddrAny' ? boolean +}; + +type slurm_cron_parameters = { + 'enable' ? boolean +}; + +type slurm_auth_info = { + 'cred_expire' ? long(1..) + 'socket' ? absolute_file_path + 'ttl' ? long(1..) + 'use_client_ids' ? boolean +}; + +type slurm_conf_control = { + 'AllowSpecResourcesUsage' ? long(0..1) # actually a boolean, defaults to 0 for non-Cray systems + 'AuthAltParameters' ? slurm_authalt_params + 'AuthAltTypes' ? choice('jwt') + 'AuthInfo' ? slurm_auth_info + 'AuthType' ? choice('munge', 'slurm') + 'BackupController' ? string + 'BackupAddr' ? type_ipv4 + 'BurstBufferType' ? choice('none', 'datawarp', 'lua') + 'ChosLoc' ? absolute_file_path # see https://github.com/scanon/chos + 'CliFilterPlugins' ? string[] + 'ClusterName' : string + 'CommunicationParameters' ? slurm_communication_params + 'CompleteWait' ? long(0..65535) + 'ControlMachine' : string + 'ControlAddr' ? type_ipv4 + + 'CoreSpecPlugin' ? choice('cray', 'none') + 'CpuFreqDef' ? choice( 'Conservative', 'OnDemand', 'Performance', 'PowerSave', 'SchedUtil') + 'CpuFreqGovernors' ? choice('Conservative', 'OnDemand', 'Performance', 'PowerSave', 'UserSpace') + 'CredType' ? choice('munge', 'slurm') + 'CryptoType' ? choice("munge", "openssl") + 'DebugFlags' ? slurm_debugflags + 'DefaultStorageHost' ? string + 'DefaultStorageLoc' ? string + 'DefaultStoragePass' ? string + 'DefaultStoragePort' ? long(0..) + 'DefaultStorageType' ? choice("filetxt", "mysql", "none") + 'DefaultStorageUser' ? string + + 'DisableRootJobs' ? boolean # YES/NO + 'EnforcePartLimits' ? choice("ALL", "ANY", "NO", "YES") + 'FairShareDampeningFactor' ? long(1..) + 'FastSchedule' ? long(0..2) + 'FederationParameters' ? dict + 'FirstJobId' ? long(0..) + 'GresTypes' ? string[] + 'GroupUpdateForce' ? boolean # 0/1 + 'GroupUpdateTime' ? long(0..) + 'HashPlugin' ? choice("k12", "sha3") + 'JobContainerType' ? choice('cncu', 'tmpfs', 'none') + 'JobCredentialPrivateKey' ? absolute_file_path + 'JobCredentialPublicCertificate' ? absolute_file_path + 'JobFileAppend' ? boolean # 0/1 + 'JobRequeue' ? boolean # 0/1 + 'JobSubmitPlugins' ? choice('lua', 'pbs', 'all_partitions', 'require_timelimit', 'throttle', + 'defaults', 'logging', 'partition')[] + 'KillOnBadExit' ? boolean # 0/1 + 'LaunchType' ? choice( 'aprun', 'poe', 'runjob', 'slurm') + 'LaunchParameters' ? slurm_launch_params + 'Licenses' ? string[] + 'MailProg' ? absolute_file_path + @{0 disables array jobs, the value of MaxJobCount should be much larger than MaxArraySize} + 'MaxArraySize' ? long(0..4000001) + 'MaxBatchRequeue' ? long(1..) + 'MaxJobCount' ? long(0..200000) + 'MaxJobId' ? long(0..67108863) + 'MaxMemPerCPU' ? long(0..) + 'MaxMemPerNode' ? long(0..) + 'MaxNodeCount' ? long(0..) + 'MaxStepCount' ? long(0..) + 'MaxTasksPerNode' ? long(0..65533) + 'MpiDefault' ? choice("pmi2", "pmix", "none") + 'MpiParams' ? slurm_mpi_params + 'PluginDir' ? absolute_file_path[] # colon-separated + 'PlugStackConfig' ? absolute_file_path # defaults to plugstack.conf in the slurm conf dir + 'PreemptMode' ? choice('OFF', 'CANCEL', 'GANG', 'REQUEUE', 'SUSPEND', 'WITHIN') + 'PreemptType' ? choice( 'none', 'partition_prio', 'qos') + 'PreemptParameters' ? slurm_preempt_params + 'ProctrackType' ? choice( 'cgroup', 'cray', 'linuxproc', 'lua', 'sgi_job', 'pgid') + 'PropagatePrioProcess' ? long(0..2) + 'PropagateResourceLimits' ? slurm_control_resourcelimits + 'PropagateResourceLimitsExcept' ? slurm_control_resourcelimits + 'RebootProgram' ? absolute_file_path + 'ReconfigFlags' ? choice( 'KeepPartInfo', 'KeepPartState') + @{Separate multiple exit code, does not support ranges} + 'RequeueExit' ? long[] + @{Separate multiple exit code, does not support ranges} + 'RequeueExitHold' ? long[] + 'ReturnToService' : long(0..2) + + 'NodeFeaturesPlugins' ? choice('knl_cray', 'knl_generic') + 'MailDomain' ? string + 'MinJobAge' ? long(0..) + 'MsgAggregationParams' ? slurm_msg_aggregation + 'PrivateData' ? choice('accounts', 'cloud', 'events', 'jobs', 'nodes', 'partitions', + 'reservations', 'usage', 'users')[] + 'SallocDefaultCommand' ? string + 'SbcastParameters' ? slurm_sbcast_parameters + 'BcastExclude' ? absolute_file_path[] + 'ScronParameters' ? slurm_cron_parameters + 'SrunPortRange' ? string # ideally the range contains at least 1000 ports + 'TmpFS' ? absolute_file_path + 'TrackWCKey' ? boolean + 'TreeWidth' ? long(0..65533) + 'UnkillableStepProgram' ? absolute_file_path + 'UsePAM' ? boolean + 'VSizeFactor' ? long(0..65533) +}; + +type slurm_conf_prolog_epilog = { + 'Epilog' ? absolute_file_path + 'EpilogSlurmctld' ? absolute_file_path + 'Prolog' ? absolute_file_path + 'PrologEpilogTimeout' ? long(0..) # seconds + 'PrologFlags' ? choice( 'Alloc', 'Contain', 'DeferBatch', 'NoHold', 'ForceRequeueOnFail', + 'RunInJob', 'Serial', 'X11')[] + 'PrologSlurmctld' ? absolute_file_path + 'ResvEpilog' ? absolute_file_path + @{in minutes} + 'ResvOverRun' ? long(0..65533) # TODO: support UNLIMITED via -1 value + 'ResvProlog' ? absolute_file_path + 'SrunEpilog' ? absolute_file_path + 'SrunProlog' ? absolute_file_path + 'TaskEpilog' ? absolute_file_path + 'TaskProlog' ? absolute_file_path +}; + +type slurm_ctld_parameters = { + 'allow_user_triggers' ? boolean + 'cloud_dns' ? boolean + 'disable_triggers' ? boolean + 'enable_configless' ? boolean + 'enable_job_state_cache' ? boolean + 'enable_stepmgr' ? boolean + 'idle_on_node_suspend' ? boolean + 'power_save_interval' ? long(0..) + 'power_save_min_interval' ? long(0..) + '{max_dbd_msg_action}' ? choice('discard', 'exit') + 'max_powered_nodes' ? long(0..) + 'no_quick_restart' ? boolean + 'node_reg_mem_percent' ? long(0..100) + 'reboot_from_controller' ? boolean + 'rl_bucket_size' ? long(0..) + 'rl_enable' ? boolean + 'rl_log_freq' ? long(-1..) + 'rl_refill_period' ? long(1..) + 'rl_refill_rate' ? long(1..) + 'rl_table_size' ? long(1..) + '{user_resv_delete}' ? boolean + 'validate_nodeaddr_threads' ? long(1..) +}; + +type slurm_d_parameters = { + 'allow_ecores' ? boolean + 'config_overrides' ? boolean + 'contain_spank' ? boolean + 'l3cache_as_socket' ? boolean + 'numa_node_as_socket' ? boolean + 'shutdown_on_reboot' ? boolean +}; + +type slurm_conf_process = { + 'MCSParameters' ? dict # see https://slurm.schedmd.com/mcs.html + 'MCSPlugin' ? choice( 'none', 'account', 'group', 'user') + + 'SlurmUser' ? string + 'SlurmdUser' ? string + 'SlurmdParameters' ? slurm_d_parameters + 'SlurmctldParameters' ? slurm_ctld_parameters + 'SlurmctldPidFile' ? absolute_file_path + 'SlurmctldPlugstack' ? string[] + @{a port range} + 'SlurmctldPort' ? long(0..)[] with {length(SELF) == 1 || length(SELF) == 2} + 'SlurmdPidFile' ? absolute_file_path + 'SlurmdPort' ? long(0..) + 'SlurmdSpoolDir' ? absolute_file_path + + 'StateSaveLocation' ? absolute_file_path + 'SwitchType' ? choice("hpe_slingshot", "nvidia_imex") # prefix = "switch/" + 'SwitchParameters' ? string # Todo more specific if needed + 'TaskPlugin' ? choice('affinity', 'cgroup', 'none')[] + 'TaskPluginParam' ? slurm_task_plugin_param + 'TopologyParam' ? slurm_topology_param + 'TopologyPlugin' ? choice('3d_torus', 'block', 'default', 'tree') +}; + +type slurm_conf_timers = { + 'BatchStartTimeout' ? long(0..) + 'CompleteWait' ? long(0..) + 'EioTimeout' ? long(0..65533) + 'EpilogMsgTime' ? long(0..) + 'GetEnvTimeout' ? long(0..) + 'InactiveLimit' ? long(0..) + 'KeepAliveTime' ? long(0..65533) + 'KillWait' ? long(0..65533) + 'MessageTimeout' ? long(0..) + 'OverTimeLimit' ? long(0..) # minutes + 'ReturnToService' ? long(0..2) + 'SlurmctldTimeout' ? long(0..65533) + 'SlurmdTimeout' ? long(0..65533) + 'TCPTimeout' ? long(0..) + 'UnkillableStepTimeout' ? long(0..) + 'WaitTime' ? long(0..65533) +}; + +type slurm_conf_scheduling = { + 'DefMemPerCPU' ? long(0..) + 'DefMemPerNode' ? long(0..) + 'DefCpuPerGPU' ? long(0..) + 'FastSchedule' ? long + 'MaxMemPerNode' ? long(0..) + 'SchedulerTimeSlice' ? long(5..65533) + 'SchedulerParameters' ? slurm_scheduler_parameters + 'DependencyParameters' ? slurm_dependency_parameters + 'SchedulerType' ? choice('backfill', 'builtin', 'hold') # prefix="sched/" + 'SelectType' ? choice('bluegene', 'cray', 'linear', 'serial', 'cons_tres') # prefix="select/" + 'SelectTypeParameters' ? slurm_select_type_parameters +}; + + +type slurm_conf_job_priority = { + @{in minutes} + 'PriorityDecayHalfLife' ? long(0..) + @{in minutes} + 'PriorityCalcPeriod' ? long(0..) + 'PriorityFavorSmall' ? boolean + 'PriorityFlags' ? choice( 'ACCRUE_ALWAYS', 'CALCULATE_RUNNING', 'DEPTH_OBLIVIOUS', 'FAIR_TREE', + 'INCR_ONLY', 'MAX_TRES', 'SMALL_RELATIVE_TO_TIME' )[] + 'PriorityParameters' ? dict + @{in minutes} + 'PriorityMaxAge' ? long(0..) + 'PriorityUsageResetPeriod' ? choice( 'NONE', 'NOW', 'DAILY', 'WEEKLY', 'MONTHLY', 'QUARTERLY', 'YEARLY') + 'PriorityType' ? choice("basic", "multifactor") + 'PriorityWeightAge' ? long(0..) + 'PriorityWeightFairshare' ? long(0..) + 'PriorityWeightJobSize' ? long(0..) + 'PriorityWeightPartition' ? long(0..) + 'PriorityWeightQOS' ? long(0..) + 'PriorityWeightTRES' ? string[] # key-value pairs +}; + +type slurm_job_gather_params = { + 'NoShared' ? boolean + 'UsePss' ? boolean + 'NoOverMemoryKill' ? boolean +}; + +type slurm_accounting_storage_type = { + 'max_step_records' ? long(0..) +}; + +type slurm_conf_accounting = { + 'AccountingStorageBackupHost' ? string + 'AccountingStorageEnforce' ? choice('associations', 'limits', 'nojobs', 'nosteps', 'qos', 'safe', 'wckeys', 'all')[] + 'AccountingStorageHost' ? string + 'AccountingStorageLoc' ? string + 'AccountingStorageParameters' ? slurm_accounting_storage_type + 'AccountingStoragePass' ? string + 'AccountingStoragePort' ? long(0..) + 'AccountingStorageTRES' ? string[] + 'AccountingStorageType' ? choice("slurmdbd") + 'AccountingStorageUser' ? string + 'AccountingStoreFlags' ? choice('job_comment', 'job_env', 'job_script', 'job_extra', 'no_stdio')[] + + 'AcctGatherNodeFreq' ? long(0..) # for acct_gather_energy/rapl plugin set a value less than 300 + 'AcctGatherEnergyType' ? choice('ipmi', 'rapl', 'xcc', 'gpu', 'pm_counters') + 'AcctGatherInterconnectType' ? choice('ofed', 'sysfs') + 'AcctGatherFilesystemType' ? choice('lustre') + 'AcctGatherProfileType' ? choice('hdf5', 'influxdb') + + 'JobCompHost' ? string + 'JobCompLoc' ? string + 'JobCompPass' ? string + 'JobCompPort' ? long(0..) + 'JobCompType' ? choice("none", "elasticsearch", "filetxt", "kafka", "lua", "mysql", "script") + 'JobCompUser' ? string + 'JobCompParams' ? slurm_job_comp_params + + 'JobAcctGatherType' ? choice("linux", "none", "cgroup") + 'JobAcctGatherFrequency' ? slurm_job_gather_frequency + 'JobAcctGatherParams' ? slurm_job_gather_params +}; + +type slurm_conf_logging = { + 'LogTimeFormat' ? choice("iso8601", "iso8601_ms", "rfc5424", "rfc5424_ms", "rfc3339", + "clock", "short", "thread_id", "format_stderr")[] + 'SlurmctldDebug' ? slurm_debuglevel + 'SlurmctldLogFile' ? absolute_file_path + 'SlurmctldSyslogDebug' ? slurm_debuglevel + 'SlurmdDebug' ? slurm_debuglevel + 'SlurmdLogFile' ? absolute_file_path + 'SlurmdSyslogDebug' ? slurm_debuglevel + 'SlurmSchedLogFile' ? absolute_file_path + 'SlurmSchedLogLevel' ? long(0..1) +}; + +type slurm_conf_power = { + 'ResumeProgram' ? absolute_file_path + 'ResumeRate' ? long(0..) + 'ResumeTimeout' ? long(0..) + 'SuspendProgram' ? absolute_file_path + 'SuspendTimeout' ? long(0..) + 'SuspendExcNodes' ? string[] + 'SuspendExcParts' ? string[] + @{number of nodes per minute} + 'SuspendRate' ? long(0..) + @{in seconds} + 'SuspendTime' ? long(0..) +}; + +type slurm_conf_compute_nodes = { + 'NodeName' ? string[] + 'NodeHostname' ? string[] + 'NodeAddr' ? string[] + 'Boards' ? long(0..) + 'CoreSpecCount' ? long(0..) + 'CoresPerSocket' ? long(0..) + 'CpuBind' ? choice('none', 'board', 'socket', 'ldom', 'core', 'thread') + 'CPUs' ? long(0..) + 'CpuSpecList' ? long(0..)[] + 'Feature' ? string[] + 'Gres' ? slurm_gres[] + @{in megabytes} + 'MemSpecLimit' ? long(0..) + 'Port' ? long(0..) + 'Procs' ? long(0..) + @{in megabytes} + 'RealMemory' ? long(0..) + 'Reason' ? string # quotes is multiple words + 'RestrictedCoresPerGPU' ? long(0..) + 'Sockets' ? long(0..) + 'SocketsPerBoard' ? long(0..) + 'State' ? choice('CLOUD', 'DOWN', 'DRAIN', 'FAIL', 'FAILING', 'FUTURE', 'UNKNOWN') + 'ThreadsPerCore' ? long(0..) + @{in megabytes} + 'TmpDisk' ? long(0..) + 'TRESWeights' ? dict() + 'Weight' ? long(0..) +}; + +type slurm_conf_down_nodes = { + 'DownNodes' ? string[] + 'Reason' ? string # quota formultiple words + 'State' ? choice('DOWN', 'DRAIN', 'FAIL', 'FAILING', 'UNKNOWN') +}; + +type slurm_conf_frontend_nodes = { + 'AllowGroups' ? string[] + 'AllowUsers' ? string[] + 'DenyGroups' ? string[] + 'DenyUsers' ? string[] + 'FrontendName' ? string[] + 'FrontendAddr' ? string[] + 'Port' ? long(0..) + 'Reason' ? string + 'State' ? choice('DOWN', 'DRAIN', 'FAIL', 'FAILING', 'UNKNOWN') +}; + +type slurm_partition_select_type = { + 'CR_Core' ? boolean + 'CR_Core_Memory' ? boolean + 'CR_Socket' ? boolean + 'CR_Socket_Memory' ? boolean +}; + +type slurm_conf_partition = { + 'AllocNodes' ? string[] + 'AllowAccounts' ? string[] + 'AllowGroups' ? string[] + 'AllowQos' ? string[] + 'Alternate' ? string + 'CpuBind' ? choice('none', 'board', 'socket', 'ldom', 'core', 'thread') + 'Default' ? boolean + 'DefCpuPerGPU' ? long(0..) + @{in megabytes} + 'DefMemPerCPU' ? long(0..) + @{in megabytes} + 'DefMemPerGPU' ? long(0..) + @{in megabytes} + 'DefMemPerNode' ? long(0..) + 'DenyAccounts' ? string[] + 'DenyQos' ? string[] + 'DefaultTime' ? string + 'DisableRootJobs' ? boolean + 'ExclusiveTopo' ? boolean + 'ExclusiveUser' ? boolean + @{in seconds} + 'GraceTime' ? long(0..) + 'Hidden' ? boolean + 'LLN' ? boolean + 'MaxCPUsPerNode' ? long(0..) + @{in megabytes} + 'MaxMemPerCPU' ? long(0..) + @{in megabytes} + 'MaxMemPerNode' ? long(0..) + 'MaxNodes' ? long(0..) + @{in minutes} + 'MaxTime' ? long(0..) + 'MinNodes' ? long(0..) + 'Nodes' ? string[] + 'OverSubscribe' ? choice('EXCLUSIVE', 'FORCE', 'YES', 'NO') + 'PartitionName' ? string + 'PreemptMode' ? choice('OFF', 'CANCEL', 'CHECKPOINT', 'GANG', 'REQUEUE', 'SUSPEND') + 'PriorityJobFactor' ? long(0..65533) + 'PriorityTier' ? long(0..65533) + 'QOS' ? string + 'ReqResv' ? boolean + 'RootOnly' ? boolean + 'SelectTypeParameters' ? slurm_partition_select_type + 'State' ? choice('UP', 'DOWN', 'DRAIN', 'INACTIVE') + 'TRESBillingWeights' ? dict +}; + +type slurm_conf_nodes = { + @{key is used as nodename, unless NodeName attribute is set} + 'compute' : slurm_conf_compute_nodes{} + @{key is used as nodename, unless DownNodes attribute is set} + 'down'? slurm_conf_down_nodes{} + @{key is used as nodename, unless FrontendName attribute is set} + 'frontend' ? slurm_conf_frontend_nodes{} +}; + +type slurm_conf = { + 'control' : slurm_conf_control + 'process' : slurm_conf_process + 'health' ? slurm_conf_health_check + 'timers' ? slurm_conf_timers + 'prepilogue' ? slurm_conf_prolog_epilog + 'scheduling' : slurm_conf_scheduling + 'priority' : slurm_conf_job_priority + 'accounting' : slurm_conf_accounting + 'logging' : slurm_conf_logging + 'power' ? slurm_conf_power + 'nodes' ? slurm_conf_nodes + @{key is used as PartitionName, unless PartitionName attribute is set} + 'partitions' ? slurm_conf_partition{} +}; + +type slurm_cgroups_conf = { + 'AllowedDevicesFile' ? absolute_file_path + 'AllowedRAMSpace' ? long(0..) + 'AllowedSwapSpace' ? long(0..) + 'CgroupMountpoint' ? absolute_file_path + 'CgroupPlugin' ? choice('cgroup/v1', 'cgroup/v2', 'autodetect', 'disabled') + 'IgnoreSystemd' ? boolean + 'IgnoreSystemdOnFailure' ? boolean + 'EnableControllers' ? boolean + 'ConstrainCores' ? boolean + 'ConstrainDevices' ? boolean + 'ConstrainRAMSpace' ? boolean + 'ConstrainSwapSpace' ? boolean + 'MaxRAMPercent' ? double(0..) + 'MaxSwapPercent' ? double(0..) + 'MemorySwappiness' ? long(0..100) + 'MinRAMSpace' ? long(0..) + 'SystemdTimeout' ? long(0..) + 'SignalChildrenProcesses' ? boolean +}; + +type slurm_spank_plugin = { + @{plugin is optional (if not optional, it is required)} + 'optional' ? boolean + 'plugin' : absolute_file_path + 'arguments' ? dict() +}; + +type slurm_spank_includes = { + 'directory' : absolute_file_path +}; + +type slurm_spank_conf = { + 'plugins' ? slurm_spank_plugin[] + 'includes' ? slurm_spank_includes[] +}; + +type slurm_topology_leaf_switch = { + 'switch': string + 'nodes': type_fqdn[] +}; + +type slurm_topology_spine_switch = { + 'switch': string + 'switches': string[] +}; + +type slurm_topology_conf = { + 'leafswitch' : slurm_topology_leaf_switch[] + 'spineswitch' : slurm_topology_spine_switch[] +}; + +type slurm_acct_gather_conf = { + @{in seconds} + 'EnergyIPMIFrequency' ? long(0..) + 'EnergyIPMICalcAdjustment' ? boolean + 'EnergyIPMIPowerSensors' ? string + 'EnergyIPMIUsername' ? string + 'EnergyIPMIPassword' ? string + 'ProfileHDF5Dir' ? absolute_file_path + 'ProfileHDF5Default' ? choice('All', 'None', 'Energy', 'Filesystem', 'Network', 'Task')[] + 'InfinibandOFEDPort' ? long(0..) + 'ProfileInfluxDBDatabase' ? string + 'ProfileInfluxDBDefault' ? choice('All', 'None', 'Energy', 'Filesystem', 'Network', 'Task')[] + 'ProfileInfluxDBHost' ? string + 'ProfileInfluxDBUser' ? string + 'ProfileInfluxDBPass' ? string + 'ProfileInfluxDBRTPolicy' ? string + 'ProfileInfluxDBTimeout' ? long(0..) +}; + +type slurm_dbd_conf = { + 'AllResourcesAbsolute' ? boolean + 'AllowNoDefAcct' ? boolean + 'ArchiveDir' ? absolute_file_path + 'ArchiveEvents' ? boolean + 'ArchiveJobs' ? boolean + 'ArchiveResvs' ? boolean + 'ArchiveScript' ? absolute_file_path + 'ArchiveSteps' ? boolean + 'ArchiveSuspend' ? boolean + 'ArchiveTXN' ? boolean + 'ArchiveUsage' ? boolean + 'AuthAltParameters' ? slurm_authalt_params + 'AuthAltTypes' ? choice('jwt') + 'AuthInfo' ? slurm_auth_info + 'AuthType' ? choice('munge', 'slurm') + 'CommitDelay' ? long(1..) + 'DbdBackupHost' ? string + 'DbdAddr' ? string + 'DbdHost' ? string + # TODO: must be equal to the AccountingStoragePort parameter in the slurm.conf + 'DbdPort' ? long(0..) + 'DebugFlags' ? choice('DB_ARCHIVE', 'DB_ASSOC', 'DB_EVENT', 'DB_JOB', 'DB_QOS', 'DB_QUERY', 'DB_RESERVATION', + 'DB_RESOURCE', 'DB_STEP', 'DB_USAGE', 'DB_WCKEY', 'FEDERATION')[] + 'DebugLevel' ? slurm_debuglevel + 'DebugLevelSyslog' ? slurm_debuglevel + 'DefaultQOS' ? string + 'DisableCoordDBD' ? boolean + 'HashPlugin' ? choice("k12", "sha3") + 'keepaliveinterval' ? long(1..) + 'keepaliveprobes' ? long(1..) + 'keepalivetime' ? long(1..) + 'LogFile' ? absolute_file_path + 'LogTimeFormat' ? choice("iso8601", "iso8601_ms", "rfc5424", "rfc5424_ms", "rfc3339", + "clock", "short", "thread_id", "format_stderr")[] + 'MaxQueryTimeRange' ? long(0..) # unsure of this type + 'MessageTimeout' ? long(0..) + 'PidFile' ? absolute_file_path + 'PluginDir' ? absolute_file_path + 'PrivateData' ? choice( 'accounts', 'events', 'jobs', 'reservations', 'usage', 'users')[] + @{in hours} + 'PurgeEventAfter' ? long(1..) + @{in hours} + 'PurgeJobAfter' ? long(1..) + @{in hours} + 'PurgeResvAfter' ? long(1..) + @{in hours} + 'PurgeStepAfter' ? long(1..) + @{in hours} + 'PurgeSuspendAfter' ? long(1..) + @{in hours} + 'PurgeTXNAfter' ? long(1..) + @{in hours} + 'PurgeUsageAfter' ? long(1..) + 'SlurmUser' ? string + 'StorageHost' ? string + 'StorageBackupHost' ? string + 'StorageLoc' ? absolute_file_path + 'StoragePass' ? string + 'StoragePort' ? long(0..) + 'StorageType' ? choice("mysql") + 'StorageUser' ? string + 'TCPTimeout' ? long(0..) + 'TrackWCKey' ? boolean + 'TrackSlurmctldDown' ? boolean +}; + +type slurm_job_container_per_node_conf = { + 'AutoBasePath' ? boolean + 'Basepath' ? absolute_file_path + 'Dirs' ? absolute_file_path[] + 'InitScript' ? absolute_file_path + 'Shared' ? boolean +}; + +type slurm_job_container_node_conf = { + include slurm_job_container_per_node_conf + 'NodeName' : string[] +}; + +type slurm_job_container_conf = { + 'Default' ? slurm_job_container_per_node_conf + 'Nodes' ? slurm_job_container_node_conf[] +}; + +type slurm_gres_autodetect_conf = { + 'AutoDetect' ? choice('nvml', 'rsmi', 'oneapi', 'off') +}; + +type slurm_gres_per_node_conf = { + include slurm_gres_autodetect_conf + 'NodeName' : string[] + 'Cores' ? long[] + 'Count' ? long(0..) + 'File' ? absolute_file_path + 'Flags' ? choice('CountOnly', 'explicit', 'one_sharing', 'all_sharing', 'nvidia_gpu_env', + 'amd_gpu_env', 'intel_gpu_env', 'opencl_env', 'no_gpu_env')[] + 'Links' ? long(0..)[] + 'Name' ? choice('gpu', 'mps', 'nic', 'shard') + 'Type' ? string +}; + +type slurm_gres_conf = { + 'Default' ? slurm_gres_autodetect_conf + 'Nodes' ? slurm_gres_per_node_conf[] +}; + +type slurm_mpi_conf = { + 'PMIxCliTmpDirBase' ? absolute_file_path + 'PMIxCollFence' ? choice('mixed', 'tree', 'ring') + 'PMIxDebug' ? boolean + 'PMIxDirectConn' ? boolean + 'PMIxDirectConnEarly' ? boolean + 'PMIxDirectConnUCX' ? boolean + 'PMIxDirectSameArch' ? boolean + 'PMIxEnv' ? string[] + 'PMIxFenceBarrier' ? boolean + 'PMIxNetDevicesUCX' ? string + 'PMIxTimeout' ? long(1..) + 'PMIxTlsUCX' ? string[] +}; + +type slurm_oci_conf = { + 'ContainerPath' ? string + 'CreateEnvFile' ? choice('null', 'newline', 'disabled') + 'DebugFlags' ? slurm_debugflags + 'DisableCleanup' ? boolean + 'DisableHooks' ? string[] + 'EnvExclude' ? string + 'FileDebug' ? slurm_debuglevel + 'IgnoreFileConfigJson' ? boolean + 'MountSpoolDir' ? string + 'RunTimeCreate' ? string + 'RunTimeDelete' ? string + 'RunTimeEnvExclude' ? string + 'RunTimeKill' ? string + 'RunTimeQuery' ? string + 'RunTimeRun' ? string + 'RunTimeStart' ? string + 'SrunArgs' ? string[] + 'SrunPath' ? absolute_file_path + 'StdIODebug' ? slurm_debuglevel + 'SyslogDebug' ? slurm_debuglevel +}; + +type slurm_helpers_default_conf = { + 'Helper' ? absolute_file_path + 'Feature' ? string[] + 'Flags' ? choice('rebootless')[] +}; + +type slurm_helpers_per_node_conf = { + include slurm_helpers_default_conf + 'NodeName' : string[] +}; + +type slurm_helpers_conf = { + 'AllowUserBoot' ? string[] + 'BootTime' ? long(0..) + 'ExecTime' ? long(0..) + 'MutuallyExclusive' ? string[] + + 'Default' ? slurm_helpers_default_conf + 'Nodes' ? slurm_helpers_per_node_conf[] +}; diff --git a/ncm-metaconfig/src/main/metaconfig/slurm/tests/profiles/cgroups.pan b/ncm-metaconfig/src/main/metaconfig/slurm/tests/profiles/cgroups.pan index 4f2503e11c..50b95b0395 100644 --- a/ncm-metaconfig/src/main/metaconfig/slurm/tests/profiles/cgroups.pan +++ b/ncm-metaconfig/src/main/metaconfig/slurm/tests/profiles/cgroups.pan @@ -5,8 +5,6 @@ include 'metaconfig/slurm/cgroups'; '/software/components/metaconfig/dependencies' = null; prefix "/software/components/metaconfig/services/{/etc/slurm/cgroup.conf}/contents"; -'CgroupAutomount' = true; - 'ConstrainCores' = true; 'ConstrainRAMSpace' = true; 'ConstrainSwapSpace' = true; diff --git a/ncm-metaconfig/src/main/metaconfig/slurm/tests/profiles/config.pan b/ncm-metaconfig/src/main/metaconfig/slurm/tests/profiles/config.pan index bb3c028f91..5413ecaed8 100644 --- a/ncm-metaconfig/src/main/metaconfig/slurm/tests/profiles/config.pan +++ b/ncm-metaconfig/src/main/metaconfig/slurm/tests/profiles/config.pan @@ -61,9 +61,14 @@ prefix "/software/components/metaconfig/services/{/etc/slurm/slurm.conf}/content "SlurmctldParameters" = dict( "power_save_interval", 20, - "{cloud_reg_addrs}", false, "{user_resv_delete}", false, "{max_dbd_msg_action}", "discard", + "enable_configless", true, + "rl_enable", true, + "rl_bucket_size", 120, + "no_quick_restart", true, + "max_powered_nodes", 10, + "enable_stepmgr", true, ); "SlurmctldPidFile" = "/var/run/slurmctld.pid"; "SlurmctldPort" = list(6817); @@ -75,7 +80,6 @@ prefix "/software/components/metaconfig/services/{/etc/slurm/slurm.conf}/content #SrunEpilog= #SrunProlog= "StateSaveLocation" = "/var/spool/slurm"; -"SwitchType" = "none"; #TaskEpilog= "TaskPlugin" = list("affinity" , "cgroup"); #TaskProlog= @@ -126,7 +130,7 @@ prefix "/software/components/metaconfig/services/{/etc/slurm/slurm.conf}/content "{partition_job_depth}", 5, ); "DependencyParameters" = dict('{max_depend_depth}', 5); -"SelectType" = "cons_res"; +"SelectType" = "cons_tres"; "SelectTypeParameters" = dict("CR_Core_Memory", true); prefix "/software/components/metaconfig/services/{/etc/slurm/slurm.conf}/contents/priority"; diff --git a/ncm-metaconfig/src/main/metaconfig/slurm/tests/profiles/dbd.pan b/ncm-metaconfig/src/main/metaconfig/slurm/tests/profiles/dbd.pan index 264a6448c4..f0670bf8e2 100644 --- a/ncm-metaconfig/src/main/metaconfig/slurm/tests/profiles/dbd.pan +++ b/ncm-metaconfig/src/main/metaconfig/slurm/tests/profiles/dbd.pan @@ -14,7 +14,7 @@ prefix "/software/components/metaconfig/services/{/etc/slurm/slurmdbd.conf}/cont 'ArchiveSuspend' = false; 'ArchiveTXN' = false; 'ArchiveUsage' = false; -'AuthInfo' = '/var/run/munge/munge.socket.2'; +'AuthInfo' = dict('socket', '/var/run/munge/munge.socket.2'); 'AuthType' = 'munge'; 'DbdHost' = 'master23'; 'DebugLevel' = 'debug4'; diff --git a/ncm-metaconfig/src/main/metaconfig/slurm/tests/regexps/cgroups/base b/ncm-metaconfig/src/main/metaconfig/slurm/tests/regexps/cgroups/base index fbd9f1238a..f92792d0b0 100644 --- a/ncm-metaconfig/src/main/metaconfig/slurm/tests/regexps/cgroups/base +++ b/ncm-metaconfig/src/main/metaconfig/slurm/tests/regexps/cgroups/base @@ -5,7 +5,6 @@ quote --- AllowedSwapSpace=10 -CgroupAutomount=yes ConstrainCores=yes ConstrainRAMSpace=yes ConstrainSwapSpace=yes diff --git a/ncm-metaconfig/src/main/metaconfig/slurm/tests/regexps/config/base b/ncm-metaconfig/src/main/metaconfig/slurm/tests/regexps/config/base index a8731bd8af..0a6d9148cb 100644 --- a/ncm-metaconfig/src/main/metaconfig/slurm/tests/regexps/config/base +++ b/ncm-metaconfig/src/main/metaconfig/slurm/tests/regexps/config/base @@ -84,14 +84,13 @@ PriorityWeightJobSize=2500 # process SlurmUser=slurm -SlurmctldParameters=max_dbd_msg_action=discard,power_save_interval=20 +SlurmctldParameters=enable_configless,enable_stepmgr,max_dbd_msg_action=discard,max_powered_nodes=10,no_quick_restart,power_save_interval=20,rl_bucket_size=120,rl_enable SlurmctldPidFile=/var/run/slurmctld.pid SlurmctldPort=6817 SlurmdPidFile=/var/run/slurmd.pid SlurmdPort=6818 SlurmdSpoolDir=/var/spool/slurm/slurmd StateSaveLocation=/var/spool/slurm -SwitchType=switch/none TaskPlugin=task/affinity,task/cgroup # scheduling @@ -103,7 +102,7 @@ FastSchedule=1 MaxMemPerNode=345 SchedulerParameters=batch_sched_delay=5,bf_continue,bf_max_job_test=1024,bf_window=4320,default_queue_depth=128,partition_job_depth=5 SchedulerType=sched/backfill -SelectType=select/cons_res +SelectType=select/cons_tres SelectTypeParameters=CR_Core_Memory # timers diff --git a/ncm-metaconfig/src/main/metaconfig/slurm/tests/regexps/dbd/base b/ncm-metaconfig/src/main/metaconfig/slurm/tests/regexps/dbd/base index 288d27bf87..fd9943c62b 100644 --- a/ncm-metaconfig/src/main/metaconfig/slurm/tests/regexps/dbd/base +++ b/ncm-metaconfig/src/main/metaconfig/slurm/tests/regexps/dbd/base @@ -11,7 +11,7 @@ ArchiveSteps=no ArchiveSuspend=no ArchiveTXN=no ArchiveUsage=no -AuthInfo=/var/run/munge/munge.socket.2 +AuthInfo=socket=/var/run/munge/munge.socket.2 AuthType=auth/munge DbdHost=master23 DebugLevel=debug4 diff --git a/ncm-metaconfig/src/main/metaconfig/slurm/value.tt b/ncm-metaconfig/src/main/metaconfig/slurm/value.tt index e2a1ab84f3..4c02b566da 100644 --- a/ncm-metaconfig/src/main/metaconfig/slurm/value.tt +++ b/ncm-metaconfig/src/main/metaconfig/slurm/value.tt @@ -1,6 +1,6 @@ [%- IF key.match('(Types?|Plugin)$') -%] [%- INCLUDE 'metaconfig/slurm/type.tt' name=key data=value -%] -[%- ELSIF key.match('(Frequency|Param((eter)?s)?)$') -%] +[%- ELSIF key.match('(Frequency|Param((eter)?s)?)|Info$') -%] [%- INCLUDE 'metaconfig/slurm/params.tt' data=value -%] [%- ELSIF value.is_boolean -%] [%- IF flavour == 'config' -%] From 182ee5bbb2caddb15d62271b25568c655d24331b Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Mon, 19 Aug 2024 08:09:20 +0200 Subject: [PATCH 08/11] ncm-postfix: add support for smtpd_relay_restrictions --- ncm-postfix/src/main/pan/components/postfix/schema.pan | 1 + ncm-postfix/src/main/resources/main.tt | 3 +++ 2 files changed, 4 insertions(+) diff --git a/ncm-postfix/src/main/pan/components/postfix/schema.pan b/ncm-postfix/src/main/pan/components/postfix/schema.pan index e2e387be20..a6cea2c795 100644 --- a/ncm-postfix/src/main/pan/components/postfix/schema.pan +++ b/ncm-postfix/src/main/pan/components/postfix/schema.pan @@ -507,6 +507,7 @@ type postfix_main = { "smtpd_recipient_restrictions" ? string[] "smtpd_reject_udicted_recipient" ? boolean "smtpd_reject_udicted_sender" ? boolean + "smtpd_relay_restrictions" ? string[] "smtpd_restriction_classes" ? string "smtpd_sasl_auth_enable" ? boolean "smtpd_sasl_authenticated_header" ? boolean diff --git a/ncm-postfix/src/main/resources/main.tt b/ncm-postfix/src/main/resources/main.tt index 1cf0be11d8..d1e86e3cf6 100644 --- a/ncm-postfix/src/main/resources/main.tt +++ b/ncm-postfix/src/main/resources/main.tt @@ -1315,6 +1315,9 @@ smtpd_recipient_restrictions = [%- smtpd_recipient_restrictions.join(", ") %] [% IF smtpd_reject_unlisted_sender.defined %] [% INCLUDE postfix/key_value_boolean.tt key="smtpd_reject_unlisted_sender" value=smtpd_reject_unlisted_sender joiner="=" bool_handler="postfix/yes_no_boolean.tt" %] [%- END -%] +[%- IF smtpd_relay_restrictions.defined %] +smtpd_relay_restrictions = [%- smtpd_relay_restrictions.join(", ") %] +[%- END -%] [%- IF smtpd_restriction_classes.defined %] smtpd_restriction_classes = [% smtpd_restriction_classes %] [%- END -%] From 47c8c984376c103cbea31b52885e2bc29b4bb15c Mon Sep 17 00:00:00 2001 From: Donald Chung <37064034+DonaldChung-HK@users.noreply.github.com> Date: Fri, 23 Aug 2024 11:24:09 +0100 Subject: [PATCH 09/11] ncm-metaconfig beats: add seccomp to file beat schema to allow seccomp config for filebeat.yml (#1700) * ncm-metaconfig: add seccomp config to beats schema Amend the beats schema to support configuration seccomp default action. This is because in Linux >3.16 (such as Rocky 9+) filebeat may be prevented from starting. --- .../src/main/metaconfig/beats/pan/schema_6.3.pan | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ncm-metaconfig/src/main/metaconfig/beats/pan/schema_6.3.pan b/ncm-metaconfig/src/main/metaconfig/beats/pan/schema_6.3.pan index 80f6f7bfe3..fae06f487b 100644 --- a/ncm-metaconfig/src/main/metaconfig/beats/pan/schema_6.3.pan +++ b/ncm-metaconfig/src/main/metaconfig/beats/pan/schema_6.3.pan @@ -128,6 +128,14 @@ type beats_logging = { 'level' ? string with match(SELF, '^(critical|error|warning|info|debug)$') }; +@documentation{ + Secomp settings for RHEL with Linux >3.16 + (see https://www.elastic.co/guide/en/beats/filebeat/6.8/linux-seccomp.html) +} +type beats_seccomp = { + 'default_action' : choice('errno', 'trace', 'trap', 'kill_thread', 'kill_process', 'log', 'allow'); +}; + @documenation{ Shared components for each beats service } @@ -140,6 +148,7 @@ type beats_service = { 'refresh_topology_freq' ? long(0..) 'topology_expire' ? long(0..) 'geoip' ? beats_shipper_geoip + 'seccomp' ? beats_seccomp }; @documentation{ From d5b986b104df8da043645c59858a829f1ad65145 Mon Sep 17 00:00:00 2001 From: ulrich1919 Date: Wed, 28 Aug 2024 13:55:20 +0200 Subject: [PATCH 10/11] Fixes #1704: ncm-grub::pxeboot return SUCCESS on non-UEFI systems --- ncm-grub/src/main/perl/grub.pm | 6 ++++++ ncm-grub/src/test/perl/methods.t | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ncm-grub/src/main/perl/grub.pm b/ncm-grub/src/main/perl/grub.pm index 3c24ba42aa..8fcba8ae73 100755 --- a/ncm-grub/src/main/perl/grub.pm +++ b/ncm-grub/src/main/perl/grub.pm @@ -21,6 +21,7 @@ Readonly my $GRUB2_USER_CFG => "$GRUB2_DIR/user.cfg"; Readonly my $GRUBBY => '/sbin/grubby'; Readonly my $PREFIX => '/boot'; Readonly my $EFIBOOTMGR => '/sbin/efibootmgr'; +Readonly my $SYS_FIRMWARE_EFI => '/sys/firmware/efi'; Readonly::Hash my %SERIAL_CONSOLE_DEFAULTS => { unit => 0, speed => 9600, @@ -894,6 +895,11 @@ sub pxeboot return SUCCESS; } + if (!$self->directory_exists($SYS_FIRMWARE_EFI)) { + $self->info("pxeboot: no $SYS_FIRMWARE_EFI found. Not doing anything"); + return SUCCESS; + } + my $efi = CAF::Process->new([$EFIBOOTMGR, '-v'], log => $self, keeps_state => 1)->output(); if (!$efi) { $self->error("No output from $EFIBOOTMGR"); diff --git a/ncm-grub/src/test/perl/methods.t b/ncm-grub/src/test/perl/methods.t index e09aa1e584..8d0b614879 100644 --- a/ncm-grub/src/test/perl/methods.t +++ b/ncm-grub/src/test/perl/methods.t @@ -405,21 +405,26 @@ ok(command_history_ok([ =cut my $ebm = '/sbin/efibootmgr'; +my $sfe = '/sys/firmware/efi'; command_history_reset(); ok (!$cmp->file_exists($ebm), "efibootmgr does not exist"); ok($cmp->pxeboot(), "pxeboot returns success when efibootmgr is missing"); ok(command_history_ok(undef, ['']), "No commands were run when efibootmgr is missing"); +ok (!$cmp->directory_exists($sfe), "/sys/firmware/efi does not exist"); +ok($cmp->pxeboot(), "pxeboot returns success when /sys/firmware/efi is missing"); +ok(command_history_ok(undef, ['']), "No commands were run when /sys/firmware/efi is missing"); +$mock->mock('directory_exists', 1); set_file_contents($ebm, ''); set_desired_output("$ebm -v", "$EFIBOOTMGROUT"); ok ($cmp->file_exists($ebm), "efibootmgr does exist"); ok($cmp->pxeboot(), "pxeboot returns success"); +$mock->unmock('directory_exists'); ok(command_history_ok([ "$ebm -v", "$ebm -o 4,3,2", ]), "efibootmgr called, correct bootorder set"); - =head1 sanitize_arguments =cut From a3a43ae8bafead4692b7524481c157234e7a0bb6 Mon Sep 17 00:00:00 2001 From: James Adams Date: Tue, 3 Sep 2024 17:52:17 +0100 Subject: [PATCH 11/11] workflows: Switch CI action over to our own container image This is dramatically faster than having to install all the dependencies on every run. Switch from using nobody to the quattortest user provided by our image. --- .github/workflows/continuous-integration.yaml | 35 +++---------------- 1 file changed, 4 insertions(+), 31 deletions(-) diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index c8a41e60e4..7800d22d9f 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -5,40 +5,14 @@ on: [push, pull_request] jobs: runtests: runs-on: ubuntu-latest - container: rockylinux:8 + container: + image: ghcr.io/quattor/quattor-test-container:latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Determine hash for caching key id: cachekeystep run: echo "pomcachekey=${{ hashFiles('**/pom.xml') }}" >> $GITHUB_ENV - - name: set up dependencies - run: | - dnf -y install dnf-plugins-core - dnf config-manager --set-enabled appstream - dnf config-manager --set-enabled powertools - dnf -y install epel-release \ - http://yum.quattor.org/devel/quattor-yum-repo-2-1.noarch.rpm - # The available version of perl-Test-Quattor is too old for mvnprove.pl to - # work, but this is a quick way of pulling in a lot of required dependencies. - # Surprisingly `which` is not installed by default and panc depends on it. - # libselinux-utils is required for /usr/sbin/selinuxenabled - dnf install -y maven which rpm-build panc ncm-lib-blockdevices \ - ncm-ncd git libselinux-utils sudo perl-Crypt-OpenSSL-X509 \ - perl-Data-Compare perl-Date-Manip perl-File-Touch perl-JSON-Any \ - perl-Net-DNS perl-Net-FreeIPA perl-Net-OpenNebula \ - perl-Net-OpenStack-Client perl-NetAddr-IP perl-REST-Client \ - perl-Set-Scalar perl-Text-Glob cpanminus gcc wget \ - perl-Git-Repository perl-Data-Structure-Util procps-ng \ - http://yum.quattor.org/devel/perl-Test-Quattor-18.3.0-SNAPSHOT20180406083650.noarch.rpm \ - http://yum.quattor.org/devel/aii-ks-21.12.1-SNAPSHOT20230627130118.noarch.rpm - - name: set up template library core from git master - run: | - cd /tmp - # install library core in /tmp, tests need it - wget -O template-library-core-master.tar.gz https://codeload.github.com/quattor/template-library-core/tar.gz/master - tar -xvzf template-library-core-master.tar.gz - cd - - name: Cache Maven packages uses: actions/cache@v4 with: @@ -49,10 +23,9 @@ jobs: run: | # make sure it exists before chown mkdir -p /tmp/m2 - chown -R nobody:nobody . /tmp/m2 + chown -R quattortest:quattortest . /tmp/m2 # we have to run as a non-root user to pass the spma tests # secondly, we first download all maven dependencies and then run the tests because it fails with hanging downloads otherwise. - runuser --shell /bin/bash --preserve-environment --command "source /usr/bin/mvn_test.sh && mvn_run \"dependency:resolve-plugins dependency:go-offline $MVN_ARGS\" && mvn_test" nobody + runuser --shell /bin/bash --preserve-environment --command "source /usr/bin/mvn_test.sh && mvn_run \"dependency:resolve-plugins dependency:go-offline $MVN_ARGS\" && mvn_test" quattortest env: - QUATTOR_TEST_TEMPLATE_LIBRARY_CORE: /tmp/template-library-core-master MVN_ARGS: -Dmaven.repo.local=/tmp/m2