From a6563711952d44330c01eb39aab2d447516e9720 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Tue, 23 Apr 2024 22:42:37 -0500 Subject: [PATCH 1/2] (maint) tasks require path to inventory yaml provision tasks document the inventory parameter as the "location to inventory file", so make it behave as such. --- README.md | 20 ++++++++++---------- lib/task_helper.rb | 15 +++++++++++++-- plans/provisioner.pp | 6 +++--- plans/teardown.pp | 2 +- spec/tasks/abs_spec.rb | 4 ++-- spec/tasks/vagrant_spec.rb | 2 +- tasks/abs.rb | 12 +++++------- tasks/docker.rb | 5 ++--- tasks/docker_exp.rb | 5 ++--- tasks/lxd.rb | 2 +- tasks/provision_service.rb | 14 ++++++-------- tasks/vagrant.rb | 16 +++++++--------- 12 files changed, 53 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 993be9b8..e67c8177 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ $ cat ~/.fog ##### Setting up a new machine ```ruby -$ bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::abs --targets localhost action=provision platform=ubuntu-1604-x86_64 inventory=/Users/tp/workspace/git/provision +$ bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::abs --targets localhost action=provision platform=ubuntu-1604-x86_64 inventory=/Users/tp/workspace/git/provision/inventory.yaml Started on localhost... Finished on localhost: @@ -118,7 +118,7 @@ Ran on 1 node in 1.44 seconds ##### Tearing down a finished machine ```ruby -$ bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::abs --targets localhost action=tear_down inventory=/Users/tp/workspace/git/provision node_name=yh6f4djvz7o3te6.delivery.puppetlabs.net +$ bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::abs --targets localhost action=tear_down inventory=/Users/tp/workspace/git/provision/inventory.yaml node_name=yh6f4djvz7o3te6.delivery.puppetlabs.net Started on localhost... Finished on localhost: @@ -139,7 +139,7 @@ Containers by default will be managed in the current [docker context](https://do #### Provision ```ruby -$ bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::docker --targets localhost action=provision platform=ubuntu:14.04 inventory=/Users/tp/workspace/git/provision +$ bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::docker --targets localhost action=provision platform=ubuntu:14.04 inventory=/Users/tp/workspace/git/provision/inventory.yaml Started on localhost... Finished on localhost: @@ -166,13 +166,13 @@ These defaults can be overriden by passing the flags with different values i.e. ``` ```ruby -bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::docker --targets localhost action=provision platform=ubuntu:14.04 inventory=/Users/tp/workspace/git/provision vars='{ "docker_run_opts": ["-p 8086:8086", "-p 3000:3000"]}' +bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::docker --targets localhost action=provision platform=ubuntu:14.04 inventory=/Users/tp/workspace/git/provision/inventory.yaml vars='{ "docker_run_opts": ["-p 8086:8086", "-p 3000:3000"]}' ``` #### Tear down ```ruby -$ bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::docker --targets localhost action=tear_down inventory=/Users/tp/workspace/git/provision node_name=localhost:2222 +$ bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::docker --targets localhost action=tear_down inventory=/Users/tp/workspace/git/provision/inventory.yaml node_name=localhost:2222 Started on localhost... Finished on localhost: @@ -197,7 +197,7 @@ Tested with vagrant images: provision ```ruby -$ bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::vagrant --targets localhost action=provision platform=ubuntu/xenial64 inventory=/Users/tp/workspace/git/provision +$ bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::vagrant --targets localhost action=provision platform=ubuntu/xenial64 inventory=/Users/tp/workspace/git/provision/inventory.yaml Started on localhost... Finished on localhost: @@ -235,7 +235,7 @@ Ran on 1 target in 0.84 sec tear_down ```ruby -$ bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::vagrant --targets localhost action=tear_down inventory=/Users/tp/workspace/git/provision node_name=127.0.0.1:2222 +$ bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::vagrant --targets localhost action=tear_down inventory=/Users/tp/workspace/git/provision/inventory.yaml node_name=127.0.0.1:2222 Started on localhost... Finished on localhost: @@ -290,7 +290,7 @@ In the provision step you can invoke bundle exec rake 'litmus:provision_list[tes Manual invocation of the provision service task from a workflow can be done using: ```ruby -bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::provision_service --targets localhost action=provision platform=centos-7-v20200813 inventory=/Users/tp/workspace/git/provision vars='role: puppetserver' +bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::provision_service --targets localhost action=provision platform=centos-7-v20200813 inventory=/Users/tp/workspace/git/provision/inventory.yaml vars='role: puppetserver' ``` Or using Litmus: @@ -326,7 +326,7 @@ provision ```powershell PS> $env:LITMUS_HYPERV_VSWITCH = 'internal_nat' -PS> bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::vagrant --targets localhost action=provision platform=centos/7 inventory=/Users/tp/workspace/git/provision hyperv_smb_username=tp hyperv_smb_password=notMyrealPassword +PS> bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::vagrant --targets localhost action=provision platform=centos/7 inventory=/Users/tp/workspace/git/provision/inventory.yaml hyperv_smb_username=tp hyperv_smb_password=notMyrealPassword Started on localhost... Finished on localhost: @@ -359,7 +359,7 @@ Testing/development/debugging it is better to use ruby directly, you will need t Testing using bolt, the second step ```ruby -bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::docker --targets localhost action=provision platform=ubuntu:14.04 inventory=/Users/tp/workspace/git/provision +bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::docker --targets localhost action=provision platform=ubuntu:14.04 inventory=/Users/tp/workspace/git/provision/inventory.yaml ``` ## License diff --git a/lib/task_helper.rb b/lib/task_helper.rb index 3feeb127..d09d998a 100644 --- a/lib/task_helper.rb +++ b/lib/task_helper.rb @@ -1,8 +1,19 @@ # frozen_string_literal: true def sanitise_inventory_location(location) - # Inventory location is an optional task parameter. If not specified use the current directory - location.nil? ? Dir.pwd : location + # Inventory location is an optional task parameter. + location = location.nil? ? Dir.pwd : location + # If not specified use the current directory + inventory.yaml + if File.exist?(location) && File.directory?(location) + # DEPRECATED: puppet_litmus <= 1.4.0 support + if Gem.loaded_specs['puppet_litmus'].version <= Gem::Version.new('1.4.0') + File.join(location, 'spec', 'fixtures', 'litmus_inventory.yaml') + else + File.join(location, 'inventory.yaml') + end + else + location + end end def get_inventory_hash(inventory_full_path) diff --git a/plans/provisioner.pp b/plans/provisioner.pp index a0667d94..19296ac9 100644 --- a/plans/provisioner.pp +++ b/plans/provisioner.pp @@ -2,8 +2,8 @@ ) { # bolt command run 'touch ./inventory.yaml' # provision server machine, set role - run_task('provision::abs', 'localhost', action => 'provision', platform => 'centos-7-x86_64', inventory => './', vars => 'role: pe') + run_task('provision::abs', 'localhost', action => 'provision', platform => 'centos-7-x86_64', vars => 'role: pe') # provision agents agent_linux agent_windows - run_task('provision::abs', 'localhost', action => 'provision', platform => 'centos-6-x86_64', inventory => './', vars => 'role: agent_linux') - run_task('provision::abs', 'localhost', action => 'provision', platform => 'win-2016-x86_64', inventory => './', vars => 'role: agent_windows') + run_task('provision::abs', 'localhost', action => 'provision', platform => 'centos-6-x86_64', vars => 'role: agent_linux') + run_task('provision::abs', 'localhost', action => 'provision', platform => 'win-2016-x86_64', vars => 'role: agent_windows') } diff --git a/plans/teardown.pp b/plans/teardown.pp index 5dde856e..446cb173 100644 --- a/plans/teardown.pp +++ b/plans/teardown.pp @@ -3,6 +3,6 @@ $all_nodes = get_targets('*') $all_node_names = $all_nodes.map |$n| { $n.name } $all_node_names.each |$node_name| { - run_task('provision::abs', 'localhost', action=> 'tear_down', node_name=> $node_name, inventory => './') + run_task('provision::abs', 'localhost', action=> 'tear_down', node_name=> $node_name) } } diff --git a/spec/tasks/abs_spec.rb b/spec/tasks/abs_spec.rb index a191aabc..b4d37425 100644 --- a/spec/tasks/abs_spec.rb +++ b/spec/tasks/abs_spec.rb @@ -89,7 +89,7 @@ def with_env(env_vars) { action: 'provision', platform: 'redhat-8-x86_64', - inventory: tmpdir + inventory: inventory_file, } end let(:response_body) do @@ -139,7 +139,7 @@ def with_env(env_vars) { action: 'tear_down', node_name: 'foo-bar.test', - inventory: tmpdir + inventory: inventory_file } end let(:inventory_yaml) do diff --git a/spec/tasks/vagrant_spec.rb b/spec/tasks/vagrant_spec.rb index 5972ed5b..f339b5e9 100644 --- a/spec/tasks/vagrant_spec.rb +++ b/spec/tasks/vagrant_spec.rb @@ -20,7 +20,7 @@ hyperv_smb_username: 'hyperv_smb_username' }.to_json) allow(Open3).to receive(:capture3).with(%r{vagrant up --provider #{provider}}, any_args).and_return(['', '', 0]).once - allow(File).to receive(:read).with(%r{#{Dir.pwd}/.vagrant}).and_return('some_unique_id') + allow(File).to receive(:read).with(%r{#{Dir.pwd}/spec/fixtures/.vagrant}).and_return('some_unique_id') allow(Open3).to receive(:capture3).with(%r{vagrant ssh-config}, any_args).and_return(['', '', 0]).once allow(Net::SSH).to receive(:start).and_return(true) require_relative '../../tasks/vagrant' diff --git a/tasks/abs.rb b/tasks/abs.rb index 3562f84e..122e8955 100755 --- a/tasks/abs.rb +++ b/tasks/abs.rb @@ -85,8 +85,7 @@ def provision(platform, inventory_location, vars) raise "Timeout: unable to get a 200 response in #{poll_duration} seconds" if reply.code != '200' - inventory_full_path = File.join(inventory_location, '/spec/fixtures/litmus_inventory.yaml') - inventory_hash = get_inventory_hash(inventory_full_path) + inventory_hash = get_inventory_hash(inventory_location) data = JSON.parse(reply.body) data.each do |host| if platform_uses_ssh(host['type']) @@ -113,14 +112,13 @@ def provision(platform, inventory_location, vars) add_node_to_group(inventory_hash, node, group_name) end - File.open(inventory_full_path, 'w') { |f| f.write inventory_hash.to_yaml } + File.open(inventory_location, 'w') { |f| f.write inventory_hash.to_yaml } { status: 'ok', nodes: data.length } end def tear_down(node_name, inventory_location) - inventory_full_path = File.join(inventory_location, '/spec/fixtures/litmus_inventory.yaml') - if File.file?(inventory_full_path) - inventory_hash = inventory_hash_from_inventory_file(inventory_full_path) + if File.file?(inventory_location) + inventory_hash = inventory_hash_from_inventory_file(inventory_location) facts = facts_from_node(inventory_hash, node_name) platform = facts['platform'] job_id = facts['job_id'] @@ -147,7 +145,7 @@ def tear_down(node_name, inventory_location) targets_to_remove.each do |target| remove_node(inventory_hash, target) end - File.open(inventory_full_path, 'w') { |f| f.write inventory_hash.to_yaml } + File.open(inventory_location, 'w') { |f| f.write inventory_hash.to_yaml } { status: 'ok', removed: targets_to_remove } end diff --git a/tasks/docker.rb b/tasks/docker.rb index 19d1f325..b4c5e862 100755 --- a/tasks/docker.rb +++ b/tasks/docker.rb @@ -118,8 +118,7 @@ def random_ssh_forwarding_port(start_port = 52_222, end_port = 52_999) def provision(docker_platform, inventory_location, vars) include PuppetLitmus::InventoryManipulation - inventory_full_path = File.join(inventory_location, '/spec/fixtures/litmus_inventory.yaml') - inventory_hash = get_inventory_hash(inventory_full_path) + inventory_hash = get_inventory_hash(inventory_location) os_release_facts = docker_image_os_release_facts(docker_platform) distro = os_release_facts['ID'] version = os_release_facts['VERSION_ID'] @@ -183,7 +182,7 @@ def provision(docker_platform, inventory_location, vars) inventory_node['facts']['container_id'] = container_id add_node_to_group(inventory_hash, inventory_node, 'ssh_nodes') - File.open(inventory_full_path, 'w') { |f| f.write inventory_hash.to_yaml } + File.open(inventory_location, 'w') { |f| f.write inventory_hash.to_yaml } { status: 'ok', node_name: inventory_node['name'], node: inventory_node } end diff --git a/tasks/docker_exp.rb b/tasks/docker_exp.rb index b37aa25a..f7888fc0 100755 --- a/tasks/docker_exp.rb +++ b/tasks/docker_exp.rb @@ -12,8 +12,7 @@ def provision(docker_platform, inventory_location, vars) include PuppetLitmus::InventoryManipulation - inventory_full_path = File.join(inventory_location, '/spec/fixtures/litmus_inventory.yaml') - inventory_hash = get_inventory_hash(inventory_full_path) + inventory_hash = get_inventory_hash(inventory_location) os_release_facts = docker_image_os_release_facts(docker_platform) inventory_node = { @@ -55,7 +54,7 @@ def provision(docker_platform, inventory_location, vars) inventory_node['facts']['container_id'] = container_id add_node_to_group(inventory_hash, inventory_node, 'docker_nodes') - File.open(inventory_full_path, 'w') { |f| f.write inventory_hash.to_yaml } + File.open(inventory_location, 'w') { |f| f.write inventory_hash.to_yaml } { status: 'ok', node_name: inventory_node['name'], node: inventory_node } end diff --git a/tasks/lxd.rb b/tasks/lxd.rb index cd762b4f..58325892 100755 --- a/tasks/lxd.rb +++ b/tasks/lxd.rb @@ -109,7 +109,7 @@ def task(**params) @node_name = params.delete(:node_name) @vars = YAML.safe_load(params.delete(:vars) || '~') - @inventory_full_path = File.join(sanitise_inventory_location(params.delete(:inventory)), 'spec/fixtures/litmus_inventory.yaml') + @inventory_full_path = sanitise_inventory_location(params.delete(:inventory)) raise "Unable to find '#{@inventory_full_path}'" unless (action == 'provision') || File.file?(@inventory_full_path) @inventory = get_inventory_hash(@inventory_full_path) diff --git a/tasks/provision_service.rb b/tasks/provision_service.rb index b0df15ba..b5690dae 100755 --- a/tasks/provision_service.rb +++ b/tasks/provision_service.rb @@ -101,7 +101,6 @@ def provision(platform, inventory_location, vars, retry_attempts) data = JSON.parse(vars.tr(';', ',')) job_url = data['job_url'] end - inventory_full_path = File.join(inventory_location, '/spec/fixtures/litmus_inventory.yaml') currnet_retry_count = 0 begin params = platform_to_cloud_request_parameters(platform, cloud, region, zone) @@ -127,17 +126,17 @@ def provision(platform, inventory_location, vars, retry_attempts) end end - if File.file?(inventory_full_path) - inventory_hash = inventory_hash_from_inventory_file(inventory_full_path) + if File.file?(inventory_location) + inventory_hash = inventory_hash_from_inventory_file(inventory_location) inventory_hash['groups'].each do |g| response_hash['groups'].each do |bg| g['targets'] = g['targets'] + bg['targets'] if g['name'] == bg['name'] end end - File.open(inventory_full_path, 'w') { |f| f.write inventory_hash.to_yaml } + File.open(inventory_location, 'w') { |f| f.write inventory_hash.to_yaml } else FileUtils.mkdir_p(File.join(Dir.pwd, '/spec/fixtures')) - File.open(inventory_full_path, 'wb') do |f| + File.open(inventory_location, 'wb') do |f| f.write(YAML.dump(response_hash)) end end @@ -153,10 +152,9 @@ def tear_down(platform, inventory_location, _vars, retry_attempts) # remove all provisioned resources uri = URI.parse(ENV['SERVICE_URL'] || default_uri) - inventory_full_path = File.join(inventory_location, '/spec/fixtures/litmus_inventory.yaml') # rubocop:disable Style/GuardClause - if File.file?(inventory_full_path) - inventory_hash = inventory_hash_from_inventory_file(inventory_full_path) + if File.file?(inventory_location) + inventory_hash = inventory_hash_from_inventory_file(inventory_location) facts = facts_from_node(inventory_hash, platform) job_id = facts['uuid'] response = invoke_cloud_request(job_id, uri, '', 'delete', retry_attempts) diff --git a/tasks/vagrant.rb b/tasks/vagrant.rb index 9594a863..566f9ba7 100755 --- a/tasks/vagrant.rb +++ b/tasks/vagrant.rb @@ -125,10 +125,9 @@ def provision(platform, inventory_location, enable_synced_folder, provider, cpus end include PuppetLitmus - inventory_full_path = File.join(inventory_location, '/spec/fixtures/litmus_inventory.yaml') - inventory_hash = get_inventory_hash(inventory_full_path) - vagrant_dirs = Dir.glob("#{File.join(inventory_location, '.vagrant')}/*/").map { |d| File.basename(d) } - @vagrant_env = File.expand_path(File.join(inventory_location, '.vagrant', get_vagrant_dir(platform, vagrant_dirs))) + inventory_hash = get_inventory_hash(inventory_location) + vagrant_dirs = Dir.glob("#{File.join(File.dirname(inventory_location), '.vagrant')}/*/").map { |d| File.basename(d) } + @vagrant_env = File.expand_path(File.join(File.dirname(inventory_location), '.vagrant', get_vagrant_dir(platform, vagrant_dirs))) FileUtils.mkdir_p @vagrant_env generate_vagrantfile(File.join(@vagrant_env, 'Vagrantfile'), platform, enable_synced_folder, provider, cpus, memory, hyperv_vswitch, hyperv_smb_username, hyperv_smb_password, box_url) command = "vagrant up --provider #{provider}" @@ -192,23 +191,22 @@ def provision(platform, inventory_location, enable_synced_folder, provider, cpus node['vars'] = var_hash end add_node_to_group(inventory_hash, node, group_name) - File.open(inventory_full_path, 'w') { |f| f.write inventory_hash.to_yaml } + File.open(inventory_location, 'w') { |f| f.write inventory_hash.to_yaml } { status: 'ok', node_name: node_name, node: node } end def tear_down(node_name, inventory_location) include PuppetLitmus command = 'vagrant destroy -f' - inventory_full_path = File.join(inventory_location, '/spec/fixtures/litmus_inventory.yaml') - if File.file?(inventory_full_path) - inventory_hash = inventory_hash_from_inventory_file(inventory_full_path) + if File.file?(inventory_location) + inventory_hash = inventory_hash_from_inventory_file(inventory_location) vagrant_env = facts_from_node(inventory_hash, node_name)['vagrant_env'] run_local_command(command, vagrant_env) remove_node(inventory_hash, node_name) FileUtils.rm_r(vagrant_env) end warn "Removed #{node_name}" - File.open(inventory_full_path, 'w') { |f| f.write inventory_hash.to_yaml } + File.open(inventory_location, 'w') { |f| f.write inventory_hash.to_yaml } { status: 'ok' } end From 1d81c9719dab17a4504d64cf408e0e830ec26dac Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Tue, 9 Jul 2024 03:36:36 -0500 Subject: [PATCH 2/2] (maint) fix ci yaml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c528a9eb..8f8f121f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: reviewdog/action-shellcheck@v1 + - uses: reviewdog/action-shellcheck@v1 with: check_all_files_with_shebangs: "true"