improve dns switchover

model/lantern/lantern_resource.rb

@@ -183,7 +183,7 @@ def sync_sequences_with_parent
         "SELECT setval('#{values[0]}.#{values[1]}', #{values[2]});"
       end
 
-      representative_server.run_query(statements, db: db)
+      representative_server.run_query(statements.join("\n"), db: db)
     end
   end
 

model/lantern/lantern_server.rb

@@ -257,16 +257,27 @@ def autoresize_disk
     incr_update_storage_size
   end
 
+  def destroy_domain
+    cf_client = Dns::Cloudflare.new
+    cf_client.delete_dns_record(domain)
+  end
+
   def swap_dns(other_server)
     strand.stack.first["domain"] = other_server.domain
     strand.modified!(:stack)
     strand.save_changes
     other_server.update(domain: nil)
+
+    if domain
+      destroy_domain
+      update(domain: nil)
+    end
+
     incr_add_domain
   end
 
   def is_dns_correct?
-    Resolv.getaddress(domain) == vm.sshable.host
+    domain && Resolv.getaddress(domain) == vm.sshable.host
   end
 
   def stop_container

prog/lantern/lantern_resource_nexus.rb

@@ -230,14 +230,7 @@ def before_run
     nap 30
   end
 
-  label def update_hosts
-    current_master = lantern_resource.parent.representative_server
-    current_master_domain = current_master.domain
-    new_master_domain = lantern_resource.representative_server.domain
-
-    lantern_resource.representative_server.update(domain: current_master_domain)
-    current_master.update(domain: new_master_domain)
-
+  label def finish_take_over
     # update display_states
     lantern_resource.update(display_state: nil)
     lantern_resource.parent.update(display_state: nil)
@@ -248,6 +241,17 @@ def before_run
     hop_wait
   end
 
+  label def update_hosts
+    current_master = lantern_resource.parent.representative_server
+    current_master_domain = current_master.domain
+    new_master_domain = lantern_resource.representative_server.domain
+
+    lantern_resource.representative_server.update(domain: current_master_domain)
+    current_master.update(domain: new_master_domain)
+
+    hop_finish_take_over
+  end
+
   label def wait_swap_ip
     ready = false
     begin
@@ -290,13 +294,19 @@ def before_run
 
   label def switch_dns_with_parent
     lantern_resource.parent.representative_server.stop_container
+    lantern_resource.update(logical_replication: false)
 
     if lantern_resource.parent.representative_server.domain.nil?
-      hop_wait_servers
+      hop_finish_take_over
     end
 
     lantern_resource.representative_server.swap_dns(lantern_resource.parent.representative_server)
-    hop_wait_servers
+    hop_wait_switch_dns
+  end
+
+  label def wait_switch_dns
+    nap 10 if !lantern_resource.representative_server.is_dns_correct?
+    hop_finish_take_over
   end
 
   label def destroy

prog/lantern/lantern_server_nexus.rb

@@ -391,11 +391,6 @@ def before_run
     hop_setup_ssl
   end
 
-  def destroy_domain
-    cf_client = Dns::Cloudflare.new
-    cf_client.delete_dns_record(lantern_server.domain)
-  end
-
   def add_domain_to_stack(domain)
     current_frame = strand.stack.first
     current_frame["domain"] = domain
@@ -659,7 +654,7 @@ def remove_domain_from_stack
       strand.children.each { _1.destroy }
 
       if !lantern_server.domain.nil?
-        destroy_domain
+        lantern_server.destroy_domain
       end
 
       if lantern_server.primary?

spec/model/lantern/lantern_resource_spec.rb

@@ -312,8 +312,8 @@
       ]
       statements_db2 = statements_db1 # identical statements for the test
 
-      expect(representative_server).to receive(:run_query).with(statements_db1, db: "db1")
-      expect(representative_server).to receive(:run_query).with(statements_db2, db: "db2")
+      expect(representative_server).to receive(:run_query).with(statements_db1.join("\n"), db: "db1")
+      expect(representative_server).to receive(:run_query).with(statements_db2.join("\n"), db: "db2")
 
       expect { lantern_resource.sync_sequences_with_parent }.not_to raise_error
     end

spec/model/lantern/lantern_server_spec.rb

@@ -785,6 +785,23 @@
   end
 
   describe "#swap_dns" do
+    it "swaps domains with another server and removes domain" do
+      frame = {}
+      other = instance_double(described_class)
+      strand = instance_double(Strand)
+      expect(lantern_server).to receive(:strand).and_return(strand).at_least(:once)
+      expect(lantern_server).to receive(:domain).and_return("old-domain").at_least(:once)
+      expect(lantern_server).to receive(:update).with(domain: nil)
+      expect(lantern_server.strand).to receive(:stack).and_return([frame]).at_least(:once)
+      expect(lantern_server.strand).to receive(:modified!).with(:stack)
+      expect(lantern_server.strand).to receive(:save_changes)
+      expect(lantern_server).to receive(:incr_add_domain)
+      expect(lantern_server).to receive(:destroy_domain)
+      expect(other).to receive(:domain).and_return("test")
+      expect(other).to receive(:update).with(domain: nil)
+      expect { lantern_server.swap_dns(other) }.not_to raise_error
+    end
+
     it "swaps domains with another server" do
       frame = {}
       other = instance_double(described_class)
@@ -829,4 +846,14 @@
       expect { lantern_server.start_container }.not_to raise_error
     end
   end
+
+  describe "#destroy_domain" do
+    it "destroys domain" do
+      cf_client = instance_double(Dns::Cloudflare)
+      expect(Dns::Cloudflare).to receive(:new).and_return(cf_client)
+      expect(lantern_server).to receive(:domain).and_return("example.com")
+      expect(cf_client).to receive(:delete_dns_record).with("example.com")
+      lantern_server.destroy_domain
+    end
+  end
 end

spec/prog/lantern/lantern_resource_nexus_spec.rb

@@ -359,26 +359,36 @@
   end
 
   describe "#update_hosts" do
-    it "updates the domains of the current and new master, updates display states, and removes fork association" do
+    it "updates the domains of the current and new master" do
       parent = instance_double(LanternResource)
       current_master = instance_double(LanternServer, domain: "current-master-domain.com")
       new_master = instance_double(LanternServer, domain: "new-master-domain.com")
-      timeline = instance_double(LanternTimeline)
 
       expect(lantern_resource).to receive(:parent).and_return(parent).at_least(:once)
       expect(parent).to receive(:representative_server).and_return(current_master).at_least(:once)
       expect(lantern_resource).to receive(:representative_server).and_return(new_master).at_least(:once)
       expect(new_master).to receive(:update).with(domain: "current-master-domain.com")
       expect(current_master).to receive(:update).with(domain: "new-master-domain.com")
 
+      expect { nx.update_hosts }.to hop("finish_take_over")
+    end
+  end
+
+  describe "#finish_take_over" do
+    it "updates display states, and removes fork association" do
+      parent = instance_double(LanternResource)
+      timeline = instance_double(LanternTimeline)
+
+      expect(lantern_resource).to receive(:parent).and_return(parent).at_least(:once)
+
       expect(lantern_resource).to receive(:update).with(display_state: nil)
       expect(parent).to receive(:update).with(display_state: nil)
 
       expect(lantern_resource).to receive(:update).with(parent_id: nil)
       expect(lantern_resource).to receive(:timeline).and_return(timeline)
       expect(timeline).to receive(:update).with(parent_id: nil)
 
-      expect { nx.update_hosts }.to hop("wait")
+      expect { nx.finish_take_over }.to hop("wait")
     end
   end
 
@@ -408,19 +418,36 @@
   end
 
   describe "#switch_dns_with_parent" do
-    it "hops to wait_servers" do
+    it "hops to finish_take_over" do
       parent = instance_double(LanternResource, representative_server: instance_double(LanternServer, domain: nil))
       expect(lantern_resource).to receive(:parent).and_return(parent).at_least(:once)
       expect(lantern_resource.parent.representative_server).to receive(:stop_container)
-      expect { nx.switch_dns_with_parent }.to hop("wait_servers")
+      expect { nx.switch_dns_with_parent }.to hop("finish_take_over")
     end
 
-    it "switches dns with parent and hop to wait_servers" do
+    it "switches dns with parent and hop to wait_switch_dns" do
       parent = instance_double(LanternResource, representative_server: instance_double(LanternServer, domain: "test-domain"))
       expect(lantern_resource).to receive(:parent).and_return(parent).at_least(:once)
       expect(lantern_resource.parent.representative_server).to receive(:stop_container)
       expect(lantern_resource.representative_server).to receive(:swap_dns).with(parent.representative_server)
-      expect { nx.switch_dns_with_parent }.to hop("wait_servers")
+      expect(lantern_resource).to receive(:update).with(logical_replication: false)
+      expect { nx.switch_dns_with_parent }.to hop("wait_switch_dns")
+    end
+  end
+
+  describe "#wait_switch_dns" do
+    it "naps if dns is not ready" do
+      representative_server = instance_double(LanternServer)
+      expect(lantern_resource).to receive(:representative_server).and_return(representative_server).at_least(:once)
+      expect(representative_server).to receive(:is_dns_correct?).and_return(false)
+      expect { nx.wait_switch_dns }.to nap 10
+    end
+
+    it "hops to finish_take_over" do
+      representative_server = instance_double(LanternServer)
+      expect(lantern_resource).to receive(:representative_server).and_return(representative_server).at_least(:once)
+      expect(representative_server).to receive(:is_dns_correct?).and_return(true)
+      expect { nx.wait_switch_dns }.to hop("finish_take_over")
     end
   end
 end

spec/prog/lantern/lantern_server_nexus_spec.rb

@@ -614,16 +614,6 @@
     end
   end
 
-  describe "#destroy_domain" do
-    it "destroys domain" do
-      cf_client = instance_double(Dns::Cloudflare)
-      expect(Dns::Cloudflare).to receive(:new).and_return(cf_client)
-      expect(lantern_server).to receive(:domain).and_return("example.com")
-      expect(cf_client).to receive(:delete_dns_record).with("example.com")
-      nx.destroy_domain
-    end
-  end
-
   describe "#add_domain_to_stack" do
     it "adds domain to current frame" do
       domain = "db.lantern.dev"
@@ -831,7 +821,7 @@
       expect(lantern_server).to receive(:primary?).and_return(true)
       expect(lantern_server.timeline).to receive(:incr_destroy).at_least(:once)
       expect(lantern_server).to receive(:domain).and_return("example.com")
-      expect(nx).to receive(:destroy_domain)
+      expect(lantern_server).to receive(:destroy_domain)
       expect(lantern_server).to receive(:destroy)
       expect { nx.destroy }.to exit({"msg" => "lantern server was deleted"})
     end