Skip to content

Commit

Permalink
Fix gcp vm updates, check for error on operations
Browse files Browse the repository at this point in the history
  • Loading branch information
var77 committed May 19, 2024
1 parent fe8ddf7 commit 9462c45
Show file tree
Hide file tree
Showing 8 changed files with 49 additions and 18 deletions.
1 change: 1 addition & 0 deletions lib/hosting/gcp_apis.rb
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ def wait_for_operation(zone, operation)

loop do
response = connection.post(path: "/compute/v1/projects/#{@project}/#{(zone == "global") ? "" : "zones/"}#{zone}/operations/#{operation}/wait", expects: [200])
Hosting::GcpApis.check_errors(response)
body = JSON.parse(response.body)
break unless body["status"] != "DONE"
rescue Excon::Error::Timeout
Expand Down
6 changes: 6 additions & 0 deletions model/gcp_vm.rb
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,12 @@ def inhost_name
self.class.ubid_to_name(UBID.from_uuidish(id))
end

def is_stopped?
gcp_client = Hosting::GcpApis.new
vm = gcp_client.get_vm(name, "#{location}-a")
vm["status"] == "TERMINATED"
end

def self.redacted_columns
super + [:public_key]
end
Expand Down
3 changes: 1 addition & 2 deletions model/lantern/lantern_server.rb
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,7 @@ def display_state
return "failed" if vm.display_state == "failed"
return "domain setup" if strand.label.include?("domain")
return "ssl setup" if strand.label == "setup_ssl"
return "updating" if strand.label.include?("update")
return "updating" if strand.label == "init_sql"
return "updating" if vm.display_state == "updating" || strand.label.include?("update") || strand.label == "init_sql"
return "unavailable" if strand.label == "wait_db_available"
return "running" if strand.label == "wait"
"creating"
Expand Down
10 changes: 3 additions & 7 deletions prog/gcp_vm/nexus.rb
Original file line number Diff line number Diff line change
Expand Up @@ -165,9 +165,7 @@ def host
end

label def wait_vm_stopped
gcp_client = Hosting::GcpApis.new
vm = gcp_client.get_vm(gcp_vm.name, "#{gcp_vm.location}-a")
if vm["status"] == "TERMINATED"
if gcp_vm.is_stopped?
gcp_vm.update(display_state: "stopped")
hop_wait
end
Expand All @@ -193,11 +191,10 @@ def host
end

label def update_storage
if gcp_vm.display_state != "stopped"
if !gcp_vm.is_stopped?
hop_stop_vm
end
decr_update_storage
gcp_vm.update(display_state: "updating")
gcp_client = Hosting::GcpApis.new
zone = "#{gcp_vm.location}-a"
vm = gcp_client.get_vm(gcp_vm.name, zone)
Expand All @@ -212,11 +209,10 @@ def host
end

label def update_size
if gcp_vm.display_state != "stopped"
if !gcp_vm.is_stopped?
hop_stop_vm
end
decr_update_size
gcp_vm.update(display_state: "updating")
gcp_client = Hosting::GcpApis.new
gcp_client.update_vm_type(gcp_vm.name, "#{gcp_vm.location}-a", gcp_vm.display_size)

Expand Down
10 changes: 10 additions & 0 deletions spec/lib/hosting/gcp_apis_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,16 @@
api = described_class.new
expect { api.wait_for_operation("us-central1-a", "test-op") }.not_to raise_error
end

it "throws error" do
stub_request(:post, "https://oauth2.googleapis.com/token").to_return(status: 200, body: JSON.dump({}), headers: {"Content-Type" => "application/json"})
stub_request(:post, "https://compute.googleapis.com/compute/v1/projects/test-project/zones/us-central1-a/operations/test-op/wait")
.to_return(status: 200, body: JSON.dump({"status" => "RUNNING"}), headers: {"Content-Type" => "application/json"})
.to_return(status: 200, body: JSON.dump({"status" => "DONE", "error" => {"errors" => [{"message" => "test"}]}}), headers: {"Content-Type" => "application/json"})

api = described_class.new
expect { api.wait_for_operation("us-central1-a", "test-op") }.to raise_error "test"
end
end

describe "#create_image" do
Expand Down
16 changes: 16 additions & 0 deletions spec/model/gcp_vm_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -54,4 +54,20 @@
expect(gcp_vm.inhost_name).to eq("pvr1mcnh")
end
end

describe "is_stopped" do
it "returns false" do
api = instance_double(Hosting::GcpApis)
expect(Hosting::GcpApis).to receive(:new).and_return(api)
expect(api).to receive(:get_vm).with(gcp_vm.name, "#{gcp_vm.location}-a").and_return({"status" => "RUNNING"})
expect(gcp_vm.is_stopped?).to be(false)
end

it "returns true" do
api = instance_double(Hosting::GcpApis)
expect(Hosting::GcpApis).to receive(:new).and_return(api)
expect(api).to receive(:get_vm).with(gcp_vm.name, "#{gcp_vm.location}-a").and_return({"status" => "TERMINATED"})
expect(gcp_vm.is_stopped?).to be(true)
end
end
end
6 changes: 6 additions & 0 deletions spec/model/lantern/lantern_server_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@
expect(lantern_server.display_state).to eq("updating")
end

it "shows updating from vm status" do
expect(lantern_server.vm).to receive(:display_state).and_return("updating").at_least(:once)
expect(lantern_server).to receive(:strand).and_return(instance_double(Strand, label: "wait")).at_least(:once)
expect(lantern_server.display_state).to eq("updating")
end

it "shows updating if init_sql" do
expect(lantern_server.vm).to receive(:display_state).and_return("running").at_least(:once)
expect(lantern_server).to receive(:strand).and_return(instance_double(Strand, label: "init_sql")).at_least(:once)
Expand Down
15 changes: 6 additions & 9 deletions spec/prog/gcp_vm/nexus_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -221,13 +221,12 @@

describe "#update_storage" do
it "stops vm before updating" do
expect(gcp_vm).to receive(:display_state).and_return("running")
expect(gcp_vm).to receive(:is_stopped?).and_return(false)
expect { nx.update_storage }.to hop("stop_vm")
end

it "resizes vm disk" do
expect(gcp_vm).to receive(:display_state).and_return("stopped")
expect(gcp_vm).to receive(:update).with({display_state: "updating"})
expect(gcp_vm).to receive(:is_stopped?).and_return(true)
gcp_api = instance_double(Hosting::GcpApis)
expect(Hosting::GcpApis).to receive(:new).and_return(gcp_api)
expect(gcp_api).to receive(:get_vm).with("dummy-vm", "us-central1-a").and_return({"disks" => [{"source" => "https://compute.googleapis.com/compute/v1/projects/test/zones/us-central1-a/disks/test-disk"}]})
Expand All @@ -236,8 +235,7 @@
end

it "resizes vm disk and hop to vm size update" do
expect(gcp_vm).to receive(:display_state).and_return("stopped")
expect(gcp_vm).to receive(:update).with({display_state: "updating"})
expect(gcp_vm).to receive(:is_stopped?).and_return(true)
expect(nx).to receive(:when_update_size_set?).and_yield
gcp_api = instance_double(Hosting::GcpApis)
expect(Hosting::GcpApis).to receive(:new).and_return(gcp_api)
Expand All @@ -249,12 +247,12 @@

describe "#update_size" do
it "hops to stop_vm" do
expect(gcp_vm).to receive(:is_stopped?).and_return(false)
expect { nx.update_size }.to hop("stop_vm")
end

it "updates vm size" do
expect(gcp_vm).to receive(:display_state).and_return("stopped")
expect(gcp_vm).to receive(:update).with({display_state: "updating"})
expect(gcp_vm).to receive(:is_stopped?).and_return(true)
gcp_api = instance_double(Hosting::GcpApis)
expect(Hosting::GcpApis).to receive(:new).and_return(gcp_api)
expect(gcp_api).to receive(:update_vm_type).with("dummy-vm", "us-central1-a", "standard-1")
Expand All @@ -263,8 +261,7 @@
end

it "hops to update_storage after vm size update" do
expect(gcp_vm).to receive(:display_state).and_return("stopped")
expect(gcp_vm).to receive(:update).with({display_state: "updating"})
expect(gcp_vm).to receive(:is_stopped?).and_return(true)
expect(nx).to receive(:when_update_storage_set?).and_yield
gcp_api = instance_double(Hosting::GcpApis)
expect(Hosting::GcpApis).to receive(:new).and_return(gcp_api)
Expand Down

0 comments on commit 9462c45

Please sign in to comment.