From 0bf02262d41ccdb999064f7cdfe737ab882b8a49 Mon Sep 17 00:00:00 2001 From: Daniel Carabas Date: Thu, 15 Oct 2020 12:11:43 +0300 Subject: [PATCH 01/10] Prepare provision service task --- tasks/provision_service.json | 27 +++++++++++++++++++++++++++ tasks/provision_service.rb | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 tasks/provision_service.json create mode 100755 tasks/provision_service.rb diff --git a/tasks/provision_service.json b/tasks/provision_service.json new file mode 100644 index 0000000..9ce81d6 --- /dev/null +++ b/tasks/provision_service.json @@ -0,0 +1,27 @@ +{ + "puppet_task_version": 1, + "supports_noop": false, + "description": "Provision/Tear down a list of machines using the provisioning service", + "parameters": { + "action": { + "description": "Action to perform, tear_down or provision", + "type": "Enum[provision, tear_down]", + "default": "provision" + }, + "platform": { + "description": "Needed by litmus", + "type": "Optional[String[1]]" + }, + "inventory": { + "description": "Location of the inventory file", + "type": "Optional[String[1]]" + }, + "vars": { + "description": "The address of the provisioning service", + "type": "Optional[String[1]]" + } + }, + "files": [ + "provision/lib/task_helper.rb" + ] +} diff --git a/tasks/provision_service.rb b/tasks/provision_service.rb new file mode 100755 index 0000000..aa9b25d --- /dev/null +++ b/tasks/provision_service.rb @@ -0,0 +1,34 @@ +#!/usr/bin/env ruby +require 'json' +require 'net/http' +require 'yaml' +require 'puppet_litmus' +require 'etc' +require_relative '../lib/task_helper' + +def provision(platform, inventory_location, vars) + #Call the provision service with the information necessary and write the inventory file locally + {status: 'ok', node_name: vars, node: platform, invloc: inventory_location, v: vars} +end + +def tear_down(platform, inventory_location, vars) + #remove all provisioned resources +end + +params = JSON.parse(STDIN.read) +platform = params['platform'] +action = params['action'] +vars = params['vars'] +inventory_location = sanitise_inventory_location(params['inventory']) +raise 'specify a node_name when tearing down' if action == 'tear_down' && node_name.nil? +raise 'specify a platform when provisioning' if action == 'provision' && platform.nil? + +begin + result = provision(platform, inventory_location, vars) if action == 'provision' + result = tear_down(node_name, inventory_location, vars) if action == 'tear_down' + puts result.to_json + exit 0 +rescue => e + puts({ _error: { kind: 'facter_task/failure', msg: e.message } }.to_json) + exit 1 +end From 3a28154e16d6c44e72ec58dec81cb7dc869eac78 Mon Sep 17 00:00:00 2001 From: Daniel Carabas Date: Mon, 26 Oct 2020 17:44:33 +0200 Subject: [PATCH 02/10] Add task for provision service usage --- tasks/provision_service.json | 4 ++ tasks/provision_service.rb | 101 +++++++++++++++++++++++++++++++++-- 2 files changed, 101 insertions(+), 4 deletions(-) diff --git a/tasks/provision_service.json b/tasks/provision_service.json index 9ce81d6..3fcbd0e 100644 --- a/tasks/provision_service.json +++ b/tasks/provision_service.json @@ -12,6 +12,10 @@ "description": "Needed by litmus", "type": "Optional[String[1]]" }, + "node_name": { + "description": "Needed by litmus", + "type": "Optional[String[1]]" + }, "inventory": { "description": "Location of the inventory file", "type": "Optional[String[1]]" diff --git a/tasks/provision_service.rb b/tasks/provision_service.rb index aa9b25d..f5f67a4 100755 --- a/tasks/provision_service.rb +++ b/tasks/provision_service.rb @@ -5,20 +5,113 @@ require 'puppet_litmus' require 'etc' require_relative '../lib/task_helper' +include PuppetLitmus::InventoryManipulation + +def default_uri + URI.parse('https://facade-main-6f3kfepqcq-ew.a.run.app/v1/provision') +end + +def platform_to_cloud_request_parameters(platform, _job_url) + params = case platform + when String + { cloud: 'gcp', images: [platform] } + when Array + { cloud: 'gcp', images: platform } + else + platform[:cloud] = 'gcp' if platform[:cloud].nil? + platform[:images] = [platform[:images]] if platform[:images].is_a?(String) + platform + end + params +end + +# curl -X POST https://facade-validation-6f3kfepqcq-ew.a.run.app/v1/provision --data @test_machines.json +def invoke_cloud_request(params, uri, job_url, verb) + case verb.downcase + when 'post' + request = Net::HTTP::Post.new(uri, 'Content-Type' => 'application/json') + machines = [] + machines << params + request.body = { url: job_url, VMs: machines }.to_json + when 'delete' + request = Net::HTTP::Delete.new(uri) + request.body = { uuid: params }.to_json + else + raise StandardError 'Unknown verb' + end + + File.open('request.json', 'wb') do |f| + f.write(request.body) + end + + req_options = { + use_ssl: uri.scheme == 'https', + read_timeout: 500, + } + + response = Net::HTTP.start(uri.hostname, uri.port, req_options) do |http| + http.request(request) + end + response.body +end def provision(platform, inventory_location, vars) - #Call the provision service with the information necessary and write the inventory file locally - {status: 'ok', node_name: vars, node: platform, invloc: inventory_location, v: vars} + # Call the provision service with the information necessary and write the inventory file locally + + job_url = ENV['GITHUB_URL'] + uri = ENV['SERVICE_URL'] + uri = default_uri if uri.nil? + if job_url.nil? + data = JSON.parse(vars.tr(';', ',')) + job_url = data['job_url'] + end + inventory_full_path = File.join(inventory_location, 'inventory.yaml') + + params = platform_to_cloud_request_parameters(platform, job_url) + response = invoke_cloud_request(params, uri, job_url, 'post') + if File.file?(inventory_full_path) + inventory_hash = inventory_hash_from_inventory_file(inventory_full_path) + response_hash = YAML.safe_load(response) + + inventory_hash['groups'].each do |g| + response_hash['groups'].each do |bg| + if g['name'] == bg['name'] + g['targets'] = g['targets'] + bg['targets'] + end + end + end + + File.open(inventory_full_path, 'w') { |f| f.write inventory_hash.to_yaml } + else + File.open('inventory.yaml', 'wb') do |f| + f.write(response) + end + end + { status: 'ok', node_name: platform } end -def tear_down(platform, inventory_location, vars) - #remove all provisioned resources +def tear_down(platform, inventory_location, _vars) + # remove all provisioned resources + uri = ENV['SERVICE_URL'] + uri = default_uri if uri.nil? + + inventory_full_path = File.join(inventory_location, 'inventory.yaml') + # rubocop:disable Style/GuardClause + if File.file?(inventory_full_path) + inventory_hash = inventory_hash_from_inventory_file(inventory_full_path) + facts = facts_from_node(inventory_hash, platform) + job_id = facts['uuid'] + response = invoke_cloud_request(job_id, uri, '', 'delete') + return response.to_json + end + # rubocop:enable Style/GuardClause end params = JSON.parse(STDIN.read) platform = params['platform'] action = params['action'] vars = params['vars'] +node_name = params['node_name'] inventory_location = sanitise_inventory_location(params['inventory']) raise 'specify a node_name when tearing down' if action == 'tear_down' && node_name.nil? raise 'specify a platform when provisioning' if action == 'provision' && platform.nil? From 29ca209fb4761de88258cea21653fb67e18441c7 Mon Sep 17 00:00:00 2001 From: Daniel Carabas Date: Mon, 26 Oct 2020 18:20:06 +0200 Subject: [PATCH 03/10] Allow setting cloud, region and zone --- tasks/provision_service.rb | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tasks/provision_service.rb b/tasks/provision_service.rb index f5f67a4..eaa6dd7 100755 --- a/tasks/provision_service.rb +++ b/tasks/provision_service.rb @@ -11,14 +11,14 @@ def default_uri URI.parse('https://facade-main-6f3kfepqcq-ew.a.run.app/v1/provision') end -def platform_to_cloud_request_parameters(platform, _job_url) +def platform_to_cloud_request_parameters(platform, cloud, region, zone) params = case platform when String - { cloud: 'gcp', images: [platform] } + { cloud: cloud, region: region, zone: zone, images: [platform] } when Array - { cloud: 'gcp', images: platform } + { cloud: cloud, region: region, zone: zone, images: platform } else - platform[:cloud] = 'gcp' if platform[:cloud].nil? + platform[:cloud] = cloud unless cloud.nil? platform[:images] = [platform[:images]] if platform[:images].is_a?(String) platform end @@ -60,6 +60,9 @@ def provision(platform, inventory_location, vars) job_url = ENV['GITHUB_URL'] uri = ENV['SERVICE_URL'] + cloud = ENV['CLOUD'] + region = ENV['REGION'] + zone = ENV['ZONE'] uri = default_uri if uri.nil? if job_url.nil? data = JSON.parse(vars.tr(';', ',')) @@ -67,7 +70,7 @@ def provision(platform, inventory_location, vars) end inventory_full_path = File.join(inventory_location, 'inventory.yaml') - params = platform_to_cloud_request_parameters(platform, job_url) + params = platform_to_cloud_request_parameters(platform, cloud, region, zone) response = invoke_cloud_request(params, uri, job_url, 'post') if File.file?(inventory_full_path) inventory_hash = inventory_hash_from_inventory_file(inventory_full_path) From 440f1541033c5ff8ab3546c0d3dc7ea5d2f4bde2 Mon Sep 17 00:00:00 2001 From: carabasdaniel Date: Tue, 27 Oct 2020 13:37:12 +0200 Subject: [PATCH 04/10] Show unknown verb in the error Co-authored-by: David Schmitt --- tasks/provision_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/provision_service.rb b/tasks/provision_service.rb index eaa6dd7..7b8176b 100755 --- a/tasks/provision_service.rb +++ b/tasks/provision_service.rb @@ -37,7 +37,7 @@ def invoke_cloud_request(params, uri, job_url, verb) request = Net::HTTP::Delete.new(uri) request.body = { uuid: params }.to_json else - raise StandardError 'Unknown verb' + raise StandardError "Unknown verb: '#{verb}'" end File.open('request.json', 'wb') do |f| From e6258ba64c95f9d5b165fabf7e1f04fb32764fa6 Mon Sep 17 00:00:00 2001 From: carabasdaniel Date: Tue, 27 Oct 2020 13:37:32 +0200 Subject: [PATCH 05/10] Use default for GITHUB_URL Co-authored-by: David Schmitt --- tasks/provision_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/provision_service.rb b/tasks/provision_service.rb index 7b8176b..2e21cc7 100755 --- a/tasks/provision_service.rb +++ b/tasks/provision_service.rb @@ -58,7 +58,7 @@ def invoke_cloud_request(params, uri, job_url, verb) def provision(platform, inventory_location, vars) # Call the provision service with the information necessary and write the inventory file locally - job_url = ENV['GITHUB_URL'] + job_url = ENV['GITHUB_URL'] || "https://api.github.com/repos/#{ENV['GITHUB_REPOSITORY']}/actions/runs/#{ENV['GITHUB_RUN_ID'}]" uri = ENV['SERVICE_URL'] cloud = ENV['CLOUD'] region = ENV['REGION'] From ea53d4387bb428f6e351033b574f492ab1ef9d17 Mon Sep 17 00:00:00 2001 From: Daniel Carabas Date: Tue, 27 Oct 2020 13:49:35 +0200 Subject: [PATCH 06/10] Update README.md --- README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/README.md b/README.md index 8bf3c23..9164a5b 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Simple tasks to provision and tear_down containers / instances and virtual machi * [Docker](#docker) * [Vagrant](#vagrant) * [Vmpooler](#vmpooler) + * [Provision Service](#provision_service) 4. [Limitations - OS compatibility, etc.](#limitations) 5. [Development - Guide for contributing to the module](#development) @@ -214,6 +215,35 @@ Successful on 1 node: localhost Ran on 1 node in 4.52 seconds ``` +### Provision_service + +The provision service task is meant to be used from a Github Action workflow. + +Example usage: +Using the following provision.yaml file: + +``` +test_serv: + provisioner: provision::provision_service + params: + cloud: gcp + region: europe-west1 + zone: europe-west1-d + images: ['centos-7-v20200618', 'windows-server-2016-dc-v20200813'] +``` + +In the provision step you can invoke bundle exec rake 'litmus:provision_list[test_serv]' and this will ensure the creation of two VMs in GCP. + +Manual invokation of the provision service task from a workflow can be done using: +``` +bundle exec bolt --modulepath /Users/tp/workspace/git/ task run provision::provision_service --nodes localhost action=provision platform=centos-7-v20200813 inventory=/Users/tp/workspace/git/provision +``` +Or using Litmus: + +``` +bundle exec rake 'litmus:provision[provision::provision_service, centos-7-v20200813]' +``` + #### Synced Folders By default the task will provision a Vagrant box with the [synced folder]() **disabled**. From a25d6943bbe93a533bf1ec3b46d187dd60fff9cb Mon Sep 17 00:00:00 2001 From: Daniel Carabas Date: Tue, 27 Oct 2020 13:54:45 +0200 Subject: [PATCH 07/10] Fix typo in GITHUB_RUN_ID ENV usage --- tasks/provision_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/provision_service.rb b/tasks/provision_service.rb index 2e21cc7..eb2108e 100755 --- a/tasks/provision_service.rb +++ b/tasks/provision_service.rb @@ -58,7 +58,7 @@ def invoke_cloud_request(params, uri, job_url, verb) def provision(platform, inventory_location, vars) # Call the provision service with the information necessary and write the inventory file locally - job_url = ENV['GITHUB_URL'] || "https://api.github.com/repos/#{ENV['GITHUB_REPOSITORY']}/actions/runs/#{ENV['GITHUB_RUN_ID'}]" + job_url = ENV['GITHUB_URL'] || "https://api.github.com/repos/#{ENV['GITHUB_REPOSITORY']}/actions/runs/#{ENV['GITHUB_RUN_ID']}" uri = ENV['SERVICE_URL'] cloud = ENV['CLOUD'] region = ENV['REGION'] From 4ef0e67099fdb396e3811016b5d24adf8dfc5e57 Mon Sep 17 00:00:00 2001 From: Daniel Carabas Date: Tue, 27 Oct 2020 16:05:01 +0200 Subject: [PATCH 08/10] Show provision errors --- tasks/provision_service.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tasks/provision_service.rb b/tasks/provision_service.rb index eb2108e..30c995c 100755 --- a/tasks/provision_service.rb +++ b/tasks/provision_service.rb @@ -8,7 +8,7 @@ include PuppetLitmus::InventoryManipulation def default_uri - URI.parse('https://facade-main-6f3kfepqcq-ew.a.run.app/v1/provision') + URI.parse('https://facade-release-6f3kfepqcq-ew.a.run.app/v1/provision') end def platform_to_cloud_request_parameters(platform, cloud, region, zone) @@ -52,7 +52,14 @@ def invoke_cloud_request(params, uri, job_url, verb) response = Net::HTTP.start(uri.hostname, uri.port, req_options) do |http| http.request(request) end - response.body + # rubocop:disable Style/GuardClause + if response.code == '200' + return response.body + else + puts "ERROR CODE: #{response.code} - BODY: #{response.body}" + exit 1 + end + # rubocop:enable Style/GuardClause end def provision(platform, inventory_location, vars) From 929ab717ef8b2be9eda4629f60de71f2fd3bb835 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 27 Oct 2020 14:25:37 +0000 Subject: [PATCH 09/10] Fix URI parse handling for SERVICE_URI --- tasks/provision_service.rb | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tasks/provision_service.rb b/tasks/provision_service.rb index 30c995c..d0d2242 100755 --- a/tasks/provision_service.rb +++ b/tasks/provision_service.rb @@ -8,7 +8,7 @@ include PuppetLitmus::InventoryManipulation def default_uri - URI.parse('https://facade-release-6f3kfepqcq-ew.a.run.app/v1/provision') + 'https://facade-release-6f3kfepqcq-ew.a.run.app/v1/provision' end def platform_to_cloud_request_parameters(platform, cloud, region, zone) @@ -66,11 +66,10 @@ def provision(platform, inventory_location, vars) # Call the provision service with the information necessary and write the inventory file locally job_url = ENV['GITHUB_URL'] || "https://api.github.com/repos/#{ENV['GITHUB_REPOSITORY']}/actions/runs/#{ENV['GITHUB_RUN_ID']}" - uri = ENV['SERVICE_URL'] + uri = URI.parse(ENV['SERVICE_URL'] || default_uri) cloud = ENV['CLOUD'] region = ENV['REGION'] zone = ENV['ZONE'] - uri = default_uri if uri.nil? if job_url.nil? data = JSON.parse(vars.tr(';', ',')) job_url = data['job_url'] @@ -102,8 +101,7 @@ def provision(platform, inventory_location, vars) def tear_down(platform, inventory_location, _vars) # remove all provisioned resources - uri = ENV['SERVICE_URL'] - uri = default_uri if uri.nil? + uri = URI.parse(ENV['SERVICE_URL'] || default_uri) inventory_full_path = File.join(inventory_location, 'inventory.yaml') # rubocop:disable Style/GuardClause From f9aab5ac4c5220a21ec6a52b59fb7dd3b811c966 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Wed, 28 Oct 2020 16:42:43 +0000 Subject: [PATCH 10/10] Adjust read_timeout to be approximately as long as the backend timeout --- tasks/provision_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/provision_service.rb b/tasks/provision_service.rb index d0d2242..3c43bc5 100755 --- a/tasks/provision_service.rb +++ b/tasks/provision_service.rb @@ -46,7 +46,7 @@ def invoke_cloud_request(params, uri, job_url, verb) req_options = { use_ssl: uri.scheme == 'https', - read_timeout: 500, + read_timeout: 60 * 5, # timeout reads after 5 minutes - that's longer than the backend service would keep the request open } response = Net::HTTP.start(uri.hostname, uri.port, req_options) do |http|