From 94281d6644d212e04ebfd649784645dd8b1f2f80 Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Fri, 8 Nov 2024 19:39:01 +0400 Subject: [PATCH] improve dns switchover --- model/lantern/lantern_resource.rb | 2 +- model/lantern/lantern_server.rb | 13 +++++- prog/lantern/lantern_resource_nexus.rb | 30 +++++++++----- prog/lantern/lantern_server_nexus.rb | 7 +--- spec/model/lantern/lantern_resource_spec.rb | 4 +- spec/model/lantern/lantern_server_spec.rb | 27 ++++++++++++ .../lantern/lantern_resource_nexus_spec.rb | 41 +++++++++++++++---- .../prog/lantern/lantern_server_nexus_spec.rb | 12 +----- 8 files changed, 98 insertions(+), 38 deletions(-) diff --git a/model/lantern/lantern_resource.rb b/model/lantern/lantern_resource.rb index 800196663..a6bf79f9a 100644 --- a/model/lantern/lantern_resource.rb +++ b/model/lantern/lantern_resource.rb @@ -183,7 +183,7 @@ def sync_sequences_with_parent "SELECT setval('#{values[0]}.#{values[1]}', #{values[2]});" end - representative_server.run_query(statements, db: db) + representative_server.run_query(statements.join("\n"), db: db) end end diff --git a/model/lantern/lantern_server.rb b/model/lantern/lantern_server.rb index 690d82cc9..0a2d5a9f4 100644 --- a/model/lantern/lantern_server.rb +++ b/model/lantern/lantern_server.rb @@ -257,16 +257,27 @@ def autoresize_disk incr_update_storage_size end + def destroy_domain + cf_client = Dns::Cloudflare.new + cf_client.delete_dns_record(domain) + end + def swap_dns(other_server) strand.stack.first["domain"] = other_server.domain strand.modified!(:stack) strand.save_changes other_server.update(domain: nil) + + if domain + destroy_domain + update(domain: nil) + end + incr_add_domain end def is_dns_correct? - Resolv.getaddress(domain) == vm.sshable.host + domain && Resolv.getaddress(domain) == vm.sshable.host end def stop_container diff --git a/prog/lantern/lantern_resource_nexus.rb b/prog/lantern/lantern_resource_nexus.rb index 44e8b124b..e36c80543 100644 --- a/prog/lantern/lantern_resource_nexus.rb +++ b/prog/lantern/lantern_resource_nexus.rb @@ -230,14 +230,7 @@ def before_run nap 30 end - label def update_hosts - current_master = lantern_resource.parent.representative_server - current_master_domain = current_master.domain - new_master_domain = lantern_resource.representative_server.domain - - lantern_resource.representative_server.update(domain: current_master_domain) - current_master.update(domain: new_master_domain) - + label def finish_take_over # update display_states lantern_resource.update(display_state: nil) lantern_resource.parent.update(display_state: nil) @@ -248,6 +241,17 @@ def before_run hop_wait end + label def update_hosts + current_master = lantern_resource.parent.representative_server + current_master_domain = current_master.domain + new_master_domain = lantern_resource.representative_server.domain + + lantern_resource.representative_server.update(domain: current_master_domain) + current_master.update(domain: new_master_domain) + + hop_finish_take_over + end + label def wait_swap_ip ready = false begin @@ -290,13 +294,19 @@ def before_run label def switch_dns_with_parent lantern_resource.parent.representative_server.stop_container + lantern_resource.update(logical_replication: false) if lantern_resource.parent.representative_server.domain.nil? - hop_wait_servers + hop_finish_take_over end lantern_resource.representative_server.swap_dns(lantern_resource.parent.representative_server) - hop_wait_servers + hop_wait_switch_dns + end + + label def wait_switch_dns + nap 10 if !lantern_resource.representative_server.is_dns_correct? + hop_finish_take_over end label def destroy diff --git a/prog/lantern/lantern_server_nexus.rb b/prog/lantern/lantern_server_nexus.rb index dea8c0b60..c693a7f84 100644 --- a/prog/lantern/lantern_server_nexus.rb +++ b/prog/lantern/lantern_server_nexus.rb @@ -391,11 +391,6 @@ def before_run hop_setup_ssl end - def destroy_domain - cf_client = Dns::Cloudflare.new - cf_client.delete_dns_record(lantern_server.domain) - end - def add_domain_to_stack(domain) current_frame = strand.stack.first current_frame["domain"] = domain @@ -659,7 +654,7 @@ def remove_domain_from_stack strand.children.each { _1.destroy } if !lantern_server.domain.nil? - destroy_domain + lantern_server.destroy_domain end if lantern_server.primary? diff --git a/spec/model/lantern/lantern_resource_spec.rb b/spec/model/lantern/lantern_resource_spec.rb index e2a2e2c4d..527fb3332 100644 --- a/spec/model/lantern/lantern_resource_spec.rb +++ b/spec/model/lantern/lantern_resource_spec.rb @@ -312,8 +312,8 @@ ] statements_db2 = statements_db1 # identical statements for the test - expect(representative_server).to receive(:run_query).with(statements_db1, db: "db1") - expect(representative_server).to receive(:run_query).with(statements_db2, db: "db2") + expect(representative_server).to receive(:run_query).with(statements_db1.join("\n"), db: "db1") + expect(representative_server).to receive(:run_query).with(statements_db2.join("\n"), db: "db2") expect { lantern_resource.sync_sequences_with_parent }.not_to raise_error end diff --git a/spec/model/lantern/lantern_server_spec.rb b/spec/model/lantern/lantern_server_spec.rb index 6d4702736..adc4e040d 100644 --- a/spec/model/lantern/lantern_server_spec.rb +++ b/spec/model/lantern/lantern_server_spec.rb @@ -785,6 +785,23 @@ end describe "#swap_dns" do + it "swaps domains with another server and removes domain" do + frame = {} + other = instance_double(described_class) + strand = instance_double(Strand) + expect(lantern_server).to receive(:strand).and_return(strand).at_least(:once) + expect(lantern_server).to receive(:domain).and_return("old-domain").at_least(:once) + expect(lantern_server).to receive(:update).with(domain: nil) + expect(lantern_server.strand).to receive(:stack).and_return([frame]).at_least(:once) + expect(lantern_server.strand).to receive(:modified!).with(:stack) + expect(lantern_server.strand).to receive(:save_changes) + expect(lantern_server).to receive(:incr_add_domain) + expect(lantern_server).to receive(:destroy_domain) + expect(other).to receive(:domain).and_return("test") + expect(other).to receive(:update).with(domain: nil) + expect { lantern_server.swap_dns(other) }.not_to raise_error + end + it "swaps domains with another server" do frame = {} other = instance_double(described_class) @@ -829,4 +846,14 @@ expect { lantern_server.start_container }.not_to raise_error end end + + describe "#destroy_domain" do + it "destroys domain" do + cf_client = instance_double(Dns::Cloudflare) + expect(Dns::Cloudflare).to receive(:new).and_return(cf_client) + expect(lantern_server).to receive(:domain).and_return("example.com") + expect(cf_client).to receive(:delete_dns_record).with("example.com") + lantern_server.destroy_domain + end + end end diff --git a/spec/prog/lantern/lantern_resource_nexus_spec.rb b/spec/prog/lantern/lantern_resource_nexus_spec.rb index 74f799e08..864ca3ecf 100644 --- a/spec/prog/lantern/lantern_resource_nexus_spec.rb +++ b/spec/prog/lantern/lantern_resource_nexus_spec.rb @@ -359,11 +359,10 @@ end describe "#update_hosts" do - it "updates the domains of the current and new master, updates display states, and removes fork association" do + it "updates the domains of the current and new master" do parent = instance_double(LanternResource) current_master = instance_double(LanternServer, domain: "current-master-domain.com") new_master = instance_double(LanternServer, domain: "new-master-domain.com") - timeline = instance_double(LanternTimeline) expect(lantern_resource).to receive(:parent).and_return(parent).at_least(:once) expect(parent).to receive(:representative_server).and_return(current_master).at_least(:once) @@ -371,6 +370,17 @@ expect(new_master).to receive(:update).with(domain: "current-master-domain.com") expect(current_master).to receive(:update).with(domain: "new-master-domain.com") + expect { nx.update_hosts }.to hop("finish_take_over") + end + end + + describe "#finish_take_over" do + it "updates display states, and removes fork association" do + parent = instance_double(LanternResource) + timeline = instance_double(LanternTimeline) + + expect(lantern_resource).to receive(:parent).and_return(parent).at_least(:once) + expect(lantern_resource).to receive(:update).with(display_state: nil) expect(parent).to receive(:update).with(display_state: nil) @@ -378,7 +388,7 @@ expect(lantern_resource).to receive(:timeline).and_return(timeline) expect(timeline).to receive(:update).with(parent_id: nil) - expect { nx.update_hosts }.to hop("wait") + expect { nx.finish_take_over }.to hop("wait") end end @@ -408,19 +418,36 @@ end describe "#switch_dns_with_parent" do - it "hops to wait_servers" do + it "hops to finish_take_over" do parent = instance_double(LanternResource, representative_server: instance_double(LanternServer, domain: nil)) expect(lantern_resource).to receive(:parent).and_return(parent).at_least(:once) expect(lantern_resource.parent.representative_server).to receive(:stop_container) - expect { nx.switch_dns_with_parent }.to hop("wait_servers") + expect { nx.switch_dns_with_parent }.to hop("finish_take_over") end - it "switches dns with parent and hop to wait_servers" do + it "switches dns with parent and hop to wait_switch_dns" do parent = instance_double(LanternResource, representative_server: instance_double(LanternServer, domain: "test-domain")) expect(lantern_resource).to receive(:parent).and_return(parent).at_least(:once) expect(lantern_resource.parent.representative_server).to receive(:stop_container) expect(lantern_resource.representative_server).to receive(:swap_dns).with(parent.representative_server) - expect { nx.switch_dns_with_parent }.to hop("wait_servers") + expect(lantern_resource).to receive(:update).with(logical_replication: false) + expect { nx.switch_dns_with_parent }.to hop("wait_switch_dns") + end + end + + describe "#wait_switch_dns" do + it "naps if dns is not ready" do + representative_server = instance_double(LanternServer) + expect(lantern_resource).to receive(:representative_server).and_return(representative_server).at_least(:once) + expect(representative_server).to receive(:is_dns_correct?).and_return(false) + expect { nx.wait_switch_dns }.to nap 10 + end + + it "hops to finish_take_over" do + representative_server = instance_double(LanternServer) + expect(lantern_resource).to receive(:representative_server).and_return(representative_server).at_least(:once) + expect(representative_server).to receive(:is_dns_correct?).and_return(true) + expect { nx.wait_switch_dns }.to hop("finish_take_over") end end end diff --git a/spec/prog/lantern/lantern_server_nexus_spec.rb b/spec/prog/lantern/lantern_server_nexus_spec.rb index 5d65eb600..e90b69ec5 100644 --- a/spec/prog/lantern/lantern_server_nexus_spec.rb +++ b/spec/prog/lantern/lantern_server_nexus_spec.rb @@ -614,16 +614,6 @@ end end - describe "#destroy_domain" do - it "destroys domain" do - cf_client = instance_double(Dns::Cloudflare) - expect(Dns::Cloudflare).to receive(:new).and_return(cf_client) - expect(lantern_server).to receive(:domain).and_return("example.com") - expect(cf_client).to receive(:delete_dns_record).with("example.com") - nx.destroy_domain - end - end - describe "#add_domain_to_stack" do it "adds domain to current frame" do domain = "db.lantern.dev" @@ -831,7 +821,7 @@ expect(lantern_server).to receive(:primary?).and_return(true) expect(lantern_server.timeline).to receive(:incr_destroy).at_least(:once) expect(lantern_server).to receive(:domain).and_return("example.com") - expect(nx).to receive(:destroy_domain) + expect(lantern_server).to receive(:destroy_domain) expect(lantern_server).to receive(:destroy) expect { nx.destroy }.to exit({"msg" => "lantern server was deleted"}) end