Fix gcp vm updates, check for error on operations

lib/hosting/gcp_apis.rb

@@ -43,6 +43,7 @@ def wait_for_operation(zone, operation)
 
     loop do
       response = connection.post(path: "/compute/v1/projects/#{@project}/#{(zone == "global") ? "" : "zones/"}#{zone}/operations/#{operation}/wait", expects: [200])
+      Hosting::GcpApis.check_errors(response)
       body = JSON.parse(response.body)
       break unless body["status"] != "DONE"
     rescue Excon::Error::Timeout

model/gcp_vm.rb

@@ -57,6 +57,12 @@ def inhost_name
     self.class.ubid_to_name(UBID.from_uuidish(id))
   end
 
+  def is_stopped?
+    gcp_client = Hosting::GcpApis.new
+    vm = gcp_client.get_vm(name, "#{location}-a")
+    vm["status"] == "TERMINATED"
+  end
+
   def self.redacted_columns
     super + [:public_key]
   end

model/lantern/lantern_server.rb

@@ -63,8 +63,7 @@ def display_state
     return "failed" if vm.display_state == "failed"
     return "domain setup" if strand.label.include?("domain")
     return "ssl setup" if strand.label == "setup_ssl"
-    return "updating" if strand.label.include?("update")
-    return "updating" if strand.label == "init_sql"
+    return "updating" if vm.display_state == "updating" || strand.label.include?("update") || strand.label == "init_sql"
     return "unavailable" if strand.label == "wait_db_available"
     return "running" if strand.label == "wait"
     "creating"

prog/gcp_vm/nexus.rb

@@ -165,9 +165,7 @@ def host
   end
 
   label def wait_vm_stopped
-    gcp_client = Hosting::GcpApis.new
-    vm = gcp_client.get_vm(gcp_vm.name, "#{gcp_vm.location}-a")
-    if vm["status"] == "TERMINATED"
+    if gcp_vm.is_stopped?
       gcp_vm.update(display_state: "stopped")
       hop_wait
     end
@@ -193,11 +191,10 @@ def host
   end
 
   label def update_storage
-    if gcp_vm.display_state != "stopped"
+    if !gcp_vm.is_stopped?
       hop_stop_vm
     end
     decr_update_storage
-    gcp_vm.update(display_state: "updating")
     gcp_client = Hosting::GcpApis.new
     zone = "#{gcp_vm.location}-a"
     vm = gcp_client.get_vm(gcp_vm.name, zone)
@@ -212,11 +209,10 @@ def host
   end
 
   label def update_size
-    if gcp_vm.display_state != "stopped"
+    if !gcp_vm.is_stopped?
       hop_stop_vm
     end
     decr_update_size
-    gcp_vm.update(display_state: "updating")
     gcp_client = Hosting::GcpApis.new
     gcp_client.update_vm_type(gcp_vm.name, "#{gcp_vm.location}-a", gcp_vm.display_size)
 

spec/lib/hosting/gcp_apis_spec.rb

@@ -323,6 +323,16 @@
         api = described_class.new
         expect { api.wait_for_operation("us-central1-a", "test-op") }.not_to raise_error
       end
+
+      it "throws error" do
+        stub_request(:post, "https://oauth2.googleapis.com/token").to_return(status: 200, body: JSON.dump({}), headers: {"Content-Type" => "application/json"})
+        stub_request(:post, "https://compute.googleapis.com/compute/v1/projects/test-project/zones/us-central1-a/operations/test-op/wait")
+          .to_return(status: 200, body: JSON.dump({"status" => "RUNNING"}), headers: {"Content-Type" => "application/json"})
+          .to_return(status: 200, body: JSON.dump({"status" => "DONE", "error" => {"errors" => [{"message" => "test"}]}}), headers: {"Content-Type" => "application/json"})
+
+        api = described_class.new
+        expect { api.wait_for_operation("us-central1-a", "test-op") }.to raise_error "test"
+      end
     end
 
     describe "#create_image" do

spec/model/gcp_vm_spec.rb

@@ -54,4 +54,20 @@
       expect(gcp_vm.inhost_name).to eq("pvr1mcnh")
     end
   end
+
+  describe "is_stopped" do
+    it "returns false" do
+      api = instance_double(Hosting::GcpApis)
+      expect(Hosting::GcpApis).to receive(:new).and_return(api)
+      expect(api).to receive(:get_vm).with(gcp_vm.name, "#{gcp_vm.location}-a").and_return({"status" => "RUNNING"})
+      expect(gcp_vm.is_stopped?).to be(false)
+    end
+
+    it "returns true" do
+      api = instance_double(Hosting::GcpApis)
+      expect(Hosting::GcpApis).to receive(:new).and_return(api)
+      expect(api).to receive(:get_vm).with(gcp_vm.name, "#{gcp_vm.location}-a").and_return({"status" => "TERMINATED"})
+      expect(gcp_vm.is_stopped?).to be(true)
+    end
+  end
 end

spec/model/lantern/lantern_server_spec.rb

@@ -51,6 +51,12 @@
       expect(lantern_server.display_state).to eq("updating")
     end
 
+    it "shows updating from vm status" do
+      expect(lantern_server.vm).to receive(:display_state).and_return("updating").at_least(:once)
+      expect(lantern_server).to receive(:strand).and_return(instance_double(Strand, label: "wait")).at_least(:once)
+      expect(lantern_server.display_state).to eq("updating")
+    end
+
     it "shows updating if init_sql" do
       expect(lantern_server.vm).to receive(:display_state).and_return("running").at_least(:once)
       expect(lantern_server).to receive(:strand).and_return(instance_double(Strand, label: "init_sql")).at_least(:once)

spec/prog/gcp_vm/nexus_spec.rb

@@ -221,13 +221,12 @@
 
   describe "#update_storage" do
     it "stops vm before updating" do
-      expect(gcp_vm).to receive(:display_state).and_return("running")
+      expect(gcp_vm).to receive(:is_stopped?).and_return(false)
       expect { nx.update_storage }.to hop("stop_vm")
     end
 
     it "resizes vm disk" do
-      expect(gcp_vm).to receive(:display_state).and_return("stopped")
-      expect(gcp_vm).to receive(:update).with({display_state: "updating"})
+      expect(gcp_vm).to receive(:is_stopped?).and_return(true)
       gcp_api = instance_double(Hosting::GcpApis)
       expect(Hosting::GcpApis).to receive(:new).and_return(gcp_api)
       expect(gcp_api).to receive(:get_vm).with("dummy-vm", "us-central1-a").and_return({"disks" => [{"source" => "https://compute.googleapis.com/compute/v1/projects/test/zones/us-central1-a/disks/test-disk"}]})
@@ -236,8 +235,7 @@
     end
 
     it "resizes vm disk and hop to vm size update" do
-      expect(gcp_vm).to receive(:display_state).and_return("stopped")
-      expect(gcp_vm).to receive(:update).with({display_state: "updating"})
+      expect(gcp_vm).to receive(:is_stopped?).and_return(true)
       expect(nx).to receive(:when_update_size_set?).and_yield
       gcp_api = instance_double(Hosting::GcpApis)
       expect(Hosting::GcpApis).to receive(:new).and_return(gcp_api)
@@ -249,12 +247,12 @@
 
   describe "#update_size" do
     it "hops to stop_vm" do
+      expect(gcp_vm).to receive(:is_stopped?).and_return(false)
       expect { nx.update_size }.to hop("stop_vm")
     end
 
     it "updates vm size" do
-      expect(gcp_vm).to receive(:display_state).and_return("stopped")
-      expect(gcp_vm).to receive(:update).with({display_state: "updating"})
+      expect(gcp_vm).to receive(:is_stopped?).and_return(true)
       gcp_api = instance_double(Hosting::GcpApis)
       expect(Hosting::GcpApis).to receive(:new).and_return(gcp_api)
       expect(gcp_api).to receive(:update_vm_type).with("dummy-vm", "us-central1-a", "standard-1")
@@ -263,8 +261,7 @@
     end
 
     it "hops to update_storage after vm size update" do
-      expect(gcp_vm).to receive(:display_state).and_return("stopped")
-      expect(gcp_vm).to receive(:update).with({display_state: "updating"})
+      expect(gcp_vm).to receive(:is_stopped?).and_return(true)
       expect(nx).to receive(:when_update_storage_set?).and_yield
       gcp_api = instance_double(Hosting::GcpApis)
       expect(Hosting::GcpApis).to receive(:new).and_return(gcp_api)