From 80ee4fead6c3cf8483d6b400c99ca8f1c015f5b5 Mon Sep 17 00:00:00 2001
From: Varik Matevosyan <varikmatevosyan@gmail.com>
Date: Wed, 13 Nov 2024 14:23:45 +0400
Subject: [PATCH 1/5] add api routes to initiate upgrades

---
 lib/validation.rb                             |  5 ++
 model/lantern/lantern_resource.rb             |  2 -
 rhizome/lantern/bin/configure                 |  1 +
 routes/api/project/location/lantern.rb        | 37 ++++++++++
 spec/lib/validation_spec.rb                   | 10 +++
 .../api/project/location/lantern_spec.rb      | 71 +++++++++++++++++++
 6 files changed, 124 insertions(+), 2 deletions(-)

diff --git a/lib/validation.rb b/lib/validation.rb
index 63dba3abe..87248bb40 100644
--- a/lib/validation.rb
+++ b/lib/validation.rb
@@ -91,6 +91,11 @@ def self.validate_version(version, field_name)
     fail ValidationFailed.new({version: msg}) unless version && !version.to_s.strip.empty?
   end
 
+  def self.validate_rollback_request(parent_id)
+    msg = "parent_id should not be empty"
+    fail ValidationFailed.new({parent_id: msg}) unless parent_id && !parent_id.to_s.strip.empty?
+  end
+
   def self.validate_storage_volumes(storage_volumes, boot_disk_index)
     allowed_keys = [:encrypted, :size_gib, :boot, :skip_sync]
     fail ValidationFailed.new({storage_volumes: "At least one storage volume is required."}) if storage_volumes.empty?
diff --git a/model/lantern/lantern_resource.rb b/model/lantern/lantern_resource.rb
index 9fb4fb15a..41f893f02 100644
--- a/model/lantern/lantern_resource.rb
+++ b/model/lantern/lantern_resource.rb
@@ -227,8 +227,6 @@ def delete_logical_subscription(name)
   def create_logical_replica(lantern_version: nil, extras_version: nil, minor_version: nil, pg_upgrade: nil)
     # TODO::
     # 1. If new database will be created during logical replication it won't be added automatically
-    # 2. New timeline will be generated for lantern resource
-    # 3. We need rollback mechanism (basically that will be ip swap again)
     ubid = LanternResource.generate_ubid
     create_ddl_log
     create_publication("pub_#{ubid}")
diff --git a/rhizome/lantern/bin/configure b/rhizome/lantern/bin/configure
index eaae037cf..009d0ee5b 100755
--- a/rhizome/lantern/bin/configure
+++ b/rhizome/lantern/bin/configure
@@ -82,6 +82,7 @@ def setup_env
       f.puts("POSTGRESQL_LOG_LINE_PREFIX=lantern-logline: app: %a user: %u time: %t proc_start: %s pid: %p linenumber: %l === ")
       f.puts("POSTGRESQL_LOG_DURATION=true")
       f.puts("POSTGRESQL_LOG_MIN_DURATION_STATEMENT=250ms")
+      f.puts("POSTGRESQL_WAL_LEVEL=logical")
       f.puts("GOOGLE_APPLICATION_CREDENTIALS_BIGQUERY_B64=#{$configure_hash["gcp_creds_big_query_b64"]}")
       f.puts("BIGQUERY_DATASET=#{$configure_hash["big_query_dataset"]}")
 
diff --git a/routes/api/project/location/lantern.rb b/routes/api/project/location/lantern.rb
index 233f1ed80..a15841501 100644
--- a/routes/api/project/location/lantern.rb
+++ b/routes/api/project/location/lantern.rb
@@ -130,12 +130,14 @@ class CloverApi
       end
 
       r.get "backups" do
+        Authorization.authorize(@current_user.id, "Postgres:view", pg.id)
         pg.timeline.backups_with_metadata
           .sort_by { |hsh| hsh[:last_modified] }
           .map { |hsh| {time: hsh[:last_modified], label: pg.timeline.get_backup_label(hsh[:key]), compressed_size: hsh[:compressed_size], uncompressed_size: hsh[:uncompressed_size]} }
       end
 
       r.post "push-backup" do
+        Authorization.authorize(@current_user.id, "Postgres:edit", pg.id)
         pg.timeline.take_manual_backup
         response.status = 200
         r.halt
@@ -150,10 +152,45 @@ class CloverApi
       end
 
       r.post "dissociate-forks" do
+        Authorization.authorize(@current_user.id, "Postgres:edit", pg.id)
         pg.dissociate_forks
         response.status = 200
         r.halt
       end
+
+      r.post "upgrade-with-replica" do
+        Authorization.authorize(@current_user.id, "Postgres:edit", pg.id)
+        Authorization.authorize(@current_user.id, "Postgres:create", @project.id)
+        lantern_version = r.params["lantern_version"].empty? ? nil : r.params["lantern_version"]
+        extras_version = r.params["extras_version"].empty? ? nil : r.params["extras_version"]
+        minor_version = r.params["minor_version"].empty? ? nil : r.params["minor_version"]
+        pg_upgrade = r.params["pg_upgrade"].empty? ? nil : r.params["pg_upgrade"]
+        st = pg.create_logical_replica(
+          lantern_version: lantern_version,
+          extras_version: extras_version,
+          minor_version: minor_version,
+          pg_upgrade: pg_upgrade
+        )
+        replica = LanternResource[st.id]
+        serialize(replica, :detailed)
+      end
+
+      r.post "switchover" do
+        Authorization.authorize(@current_user.id, "Postgres:edit", pg.id)
+        pg.incr_switchover_with_parent
+        response.status = 200
+        r.halt
+      end
+
+      r.post "rollback-switchover" do
+        Authorization.authorize(@current_user.id, "Postgres:edit", pg.id)
+        parent = LanternResource[r.params["parent_id"]]
+        Authorization.authorize(@current_user.id, "Postgres:edit", parent.id)
+        Validation.validate_rollback_request(r.params["parent_id"])
+        MiscOperations.rollback_switchover(pg, parent)
+        response.status = 200
+        r.halt
+      end
     end
 
     r.get true do
diff --git a/spec/lib/validation_spec.rb b/spec/lib/validation_spec.rb
index bc79c1f53..3e7408edc 100644
--- a/spec/lib/validation_spec.rb
+++ b/spec/lib/validation_spec.rb
@@ -170,6 +170,16 @@
       end
     end
 
+    describe "#validate_rollback_request" do
+      it "valid request" do
+        expect(described_class.validate_rollback_request("test-id")).to be_nil
+      end
+
+      it "invalid version" do
+        expect { described_class.validate_rollback_request("") }.to raise_error described_class::ValidationFailed
+      end
+    end
+
     describe "#validate_lantern_size" do
       it "valid lantern size" do
         expect(described_class.validate_lantern_size("n1-standard-2").name).to eq("n1-standard-2")
diff --git a/spec/routes/api/project/location/lantern_spec.rb b/spec/routes/api/project/location/lantern_spec.rb
index 2a1a009d0..55e768202 100644
--- a/spec/routes/api/project/location/lantern_spec.rb
+++ b/spec/routes/api/project/location/lantern_spec.rb
@@ -322,5 +322,76 @@
         expect(last_response.status).to eq(200)
       end
     end
+
+    describe "upgrade-with-replica" do
+      it "creates a new replica" do
+        expect(Project).to receive(:from_ubid).and_return(project).at_least(:once)
+        query_res = class_double(LanternResource, first: pg)
+        allow(query_res).to receive(:where).and_return(query_res)
+        expect(project).to receive(:lantern_resources_dataset).and_return(query_res)
+        expect(pg).to receive(:create_logical_replica).with(
+          lantern_version: nil,
+          extras_version: nil,
+          minor_version: nil,
+          pg_upgrade: nil
+        ).and_return(instance_double(Strand, id: pg.id))
+
+        post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/upgrade-with-replica", {lantern_version: "", extras_version: "", minor_version: "", pg_upgrade: ""}
+        expect(last_response.status).to eq(200)
+      end
+
+      it "creates a new replica with upgrade request" do
+        expect(Project).to receive(:from_ubid).and_return(project).at_least(:once)
+        query_res = class_double(LanternResource, first: pg)
+        allow(query_res).to receive(:where).and_return(query_res)
+        expect(project).to receive(:lantern_resources_dataset).and_return(query_res)
+        expect(pg).to receive(:create_logical_replica).with(
+          lantern_version: "0.5.0",
+          extras_version: "0.5.0",
+          minor_version: "1",
+          pg_upgrade: {"lantern_version" => "0.6.0", "extras_version" => "0.6.0", "minor_version" => "1", "pg_version" => "17"}
+        ).and_return(instance_double(Strand, id: pg.id))
+
+        post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/upgrade-with-replica", {
+          lantern_version: "0.5.0",
+          extras_version: "0.5.0",
+          minor_version: "1",
+          pg_upgrade: {"lantern_version" => "0.6.0", "extras_version" => "0.6.0", "minor_version" => "1", "pg_version" => 17}
+        }
+        expect(last_response.status).to eq(200)
+      end
+    end
+
+    describe "switchover" do
+      it "performs a switchover" do
+        expect(Project).to receive(:from_ubid).and_return(project).at_least(:once)
+        query_res = class_double(LanternResource, first: pg)
+        allow(query_res).to receive(:where).and_return(query_res)
+        expect(project).to receive(:lantern_resources_dataset).and_return(query_res)
+        expect(pg).to receive(:incr_switchover_with_parent)
+
+        post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/switchover"
+        expect(last_response.status).to eq(200)
+      end
+    end
+
+    describe "rollback-switchover" do
+      it "rolls back a switchover" do
+        expect(Project).to receive(:from_ubid).and_return(project).at_least(:once)
+        query_res = class_double(LanternResource, first: pg)
+        allow(query_res).to receive(:where).and_return(query_res)
+        expect(project).to receive(:lantern_resources_dataset).and_return(query_res)
+        parent_id = LanternResource.generate_uuid
+        parent_resource = instance_double(LanternResource, id: parent_id)
+        expect(Authorization).to receive(:authorize).with(user.id, "Postgres:edit", pg.id)
+        expect(Authorization).to receive(:authorize).with(user.id, "Postgres:edit", parent_resource.id)
+        allow(LanternResource).to receive(:[]).with(parent_id).and_return(parent_resource)
+        expect(Validation).to receive(:validate_rollback_request).with(parent_id)
+        expect(MiscOperations).to receive(:rollback_switchover).with(pg, parent_resource)
+
+        post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/rollback-switchover", {parent_id: parent_id}
+        expect(last_response.status).to eq(200)
+      end
+    end
   end
 end

From 8c0aa082764faaf4d5137a34df62b7dfa48f5ebe Mon Sep 17 00:00:00 2001
From: Varik Matevosyan <varikmatevosyan@gmail.com>
Date: Wed, 13 Nov 2024 16:29:49 +0400
Subject: [PATCH 2/5] update wal-g binary path in delete command

---
 prog/lantern/lantern_timeline_nexus.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/prog/lantern/lantern_timeline_nexus.rb b/prog/lantern/lantern_timeline_nexus.rb
index 298959d4f..d4d169ee5 100644
--- a/prog/lantern/lantern_timeline_nexus.rb
+++ b/prog/lantern/lantern_timeline_nexus.rb
@@ -50,7 +50,7 @@ def before_run
 
     if lantern_timeline.need_cleanup?
       retain_after = (Time.new - (24 * 60 * 60 * Config.backup_retention_days)).strftime("%Y-%m-%dT%H:%M:%S.%LZ")
-      cmd = "docker compose -f /var/lib/lantern/docker-compose.yaml exec -T -u root postgresql bash -c \"GOOGLE_APPLICATION_CREDENTIALS=/tmp/google-application-credentials-wal-g.json /opt/bitnami/postgresql/bin/wal-g delete retain FULL 7 --after #{retain_after} --confirm\""
+      cmd = "docker compose -f /var/lib/lantern/docker-compose.yaml exec -T -u root postgresql bash -c \"GOOGLE_APPLICATION_CREDENTIALS=/tmp/google-application-credentials-wal-g.json /usr/local/go/bin/wal-g delete retain FULL 7 --after #{retain_after} --confirm\""
       lantern_timeline.leader.vm.sshable.cmd("common/bin/daemonizer '#{cmd}' delete_old_backups")
     end
 

From a9844226a56e40ae0bc721269fe339c781ea0f1c Mon Sep 17 00:00:00 2001
From: Varik Matevosyan <varikmatevosyan@gmail.com>
Date: Wed, 13 Nov 2024 18:53:36 +0400
Subject: [PATCH 3/5] move rollback functionality to lantern_resource class

---
 lib/validation.rb                             |  6 +-
 ...241113_lantern_resource_rollback_target.rb |  9 +++
 misc/misc_operations.rb                       | 25 -------
 model/lantern/lantern_resource.rb             | 21 +++++-
 prog/lantern/lantern_resource_nexus.rb        | 20 +++++-
 routes/api/project/location/lantern.rb        | 18 +++--
 serializers/api/lantern.rb                    |  3 +-
 spec/lib/validation_spec.rb                   |  4 +-
 spec/model/lantern/lantern_resource_spec.rb   | 29 ++++++++
 .../lantern/lantern_resource_nexus_spec.rb    | 39 ++++++++++-
 .../api/project/location/lantern_spec.rb      | 67 +++++++++++++++----
 11 files changed, 190 insertions(+), 51 deletions(-)
 create mode 100644 migrate/20241113_lantern_resource_rollback_target.rb

diff --git a/lib/validation.rb b/lib/validation.rb
index 87248bb40..4aab9f02f 100644
--- a/lib/validation.rb
+++ b/lib/validation.rb
@@ -91,9 +91,9 @@ def self.validate_version(version, field_name)
     fail ValidationFailed.new({version: msg}) unless version && !version.to_s.strip.empty?
   end
 
-  def self.validate_rollback_request(parent_id)
-    msg = "parent_id should not be empty"
-    fail ValidationFailed.new({parent_id: msg}) unless parent_id && !parent_id.to_s.strip.empty?
+  def self.validate_rollback_request(pg)
+    msg = "database does not have rollback_target"
+    fail ValidationFailed.new({rollback_target: msg}) unless !pg.rollback_target.nil?
   end
 
   def self.validate_storage_volumes(storage_volumes, boot_disk_index)
diff --git a/migrate/20241113_lantern_resource_rollback_target.rb b/migrate/20241113_lantern_resource_rollback_target.rb
new file mode 100644
index 000000000..c0bfc651a
--- /dev/null
+++ b/migrate/20241113_lantern_resource_rollback_target.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:lantern_resource) do
+      add_column :rollback_target, :uuid
+    end
+  end
+end
diff --git a/misc/misc_operations.rb b/misc/misc_operations.rb
index 4cebf2eaf..5b17ef44c 100644
--- a/misc/misc_operations.rb
+++ b/misc/misc_operations.rb
@@ -240,29 +240,4 @@ def self.create_image(lantern_version: "0.2.7", extras_version: "0.1.5", minor_v
     puts "Image created"
     vm.incr_destroy
   end
-
-  def self.rollback_switchover(current_resource, old_resource)
-    # stop current one and start old one
-    begin
-      current_resource.representative_server.stop_container(1)
-    rescue
-    end
-
-    old_resource.representative_server.start_container
-
-    # update dns
-    cf_client = Dns::Cloudflare.new
-    cf_client.upsert_dns_record(current_resource.representative_server.domain, old_resource.representative_server.vm.sshable.host)
-    old_resource.representative_server.update(domain: current_resource.representative_server.domain)
-    current_resource.representative_server.update(domain: nil)
-
-    # disable readonly as soon as it is started
-    loop do
-      old_resource.representative_server.run_query("SELECT 1")
-      old_resource.set_to_readonly(status: "off")
-      break
-    rescue
-      sleep 10
-    end
-  end
 end
diff --git a/model/lantern/lantern_resource.rb b/model/lantern/lantern_resource.rb
index 41f893f02..0698ffa77 100644
--- a/model/lantern/lantern_resource.rb
+++ b/model/lantern/lantern_resource.rb
@@ -22,7 +22,7 @@ class LanternResource < Sequel::Model
   include Authorization::HyperTagMethods
   include Authorization::TaggableMethods
 
-  semaphore :destroy, :swap_leaders_with_parent, :switchover_with_parent
+  semaphore :destroy, :swap_leaders_with_parent, :switchover_with_parent, :rollback_switchover
 
   plugin :column_encryption do |enc|
     enc.column :superuser_password
@@ -293,4 +293,23 @@ module HaType
     ASYNC = "async"
     SYNC = "sync"
   end
+
+  def rollback_switchover
+    current_resource = LanternResource[rollback_target]
+    # stop current one and start old one
+    begin
+      current_resource.representative_server.stop_container(1)
+    rescue
+    end
+
+    representative_server.start_container
+
+    # update dns
+    cf_client = Dns::Cloudflare.new
+    cf_client.upsert_dns_record(current_resource.representative_server.domain, representative_server.vm.sshable.host)
+    representative_server.update(domain: current_resource.representative_server.domain)
+    current_resource.representative_server.update(domain: nil)
+
+    update(rollback_target: nil)
+  end
 end
diff --git a/prog/lantern/lantern_resource_nexus.rb b/prog/lantern/lantern_resource_nexus.rb
index 5f00e93df..b84394a53 100644
--- a/prog/lantern/lantern_resource_nexus.rb
+++ b/prog/lantern/lantern_resource_nexus.rb
@@ -233,11 +233,12 @@ def before_run
   label def finish_take_over
     # update display_states
     lantern_resource.update(display_state: nil)
-    lantern_resource.parent.update(display_state: nil)
+    lantern_resource.parent.update(display_state: nil, rollback_target: lantern_resource.id)
 
     # remove fork association so parent can be deleted
     lantern_resource.update(parent_id: nil)
     lantern_resource.timeline.update(parent_id: nil)
+
     hop_wait
   end
 
@@ -267,6 +268,23 @@ def before_run
     end
   end
 
+  label def rollback_switchover
+    lantern_resource.rollback_switchover
+    hop_wait_rollback_switchover
+  end
+
+  label def wait_rollback_switchover
+    nap 10 if !lantern_resource.representative_server.is_dns_correct?
+    begin
+      connection = Sequel.connect(lantern_resource.connection_string)
+      connection["SELECT 1"].first
+      lantern_resource.set_to_readonly(status: "off")
+    rescue
+      nap 10
+    end
+    hop_wait_servers
+  end
+
   label def swap_leaders_with_parent
     decr_swap_leaders_with_parent
     lantern_resource.parent.set_to_readonly
diff --git a/routes/api/project/location/lantern.rb b/routes/api/project/location/lantern.rb
index a15841501..c53a47abe 100644
--- a/routes/api/project/location/lantern.rb
+++ b/routes/api/project/location/lantern.rb
@@ -158,7 +158,7 @@ class CloverApi
         r.halt
       end
 
-      r.post "upgrade-with-replica" do
+      r.post "logical-replica" do
         Authorization.authorize(@current_user.id, "Postgres:edit", pg.id)
         Authorization.authorize(@current_user.id, "Postgres:create", @project.id)
         lantern_version = r.params["lantern_version"].empty? ? nil : r.params["lantern_version"]
@@ -177,6 +177,11 @@ class CloverApi
 
       r.post "switchover" do
         Authorization.authorize(@current_user.id, "Postgres:edit", pg.id)
+
+        if pg.parent.nil? || !pg.logical_replication
+          fail CloverError.new(400, "Invalid request", "Database does not have parent or is not in logical replication state")
+        end
+
         pg.incr_switchover_with_parent
         response.status = 200
         r.halt
@@ -184,10 +189,13 @@ class CloverApi
 
       r.post "rollback-switchover" do
         Authorization.authorize(@current_user.id, "Postgres:edit", pg.id)
-        parent = LanternResource[r.params["parent_id"]]
-        Authorization.authorize(@current_user.id, "Postgres:edit", parent.id)
-        Validation.validate_rollback_request(r.params["parent_id"])
-        MiscOperations.rollback_switchover(pg, parent)
+        Validation.validate_rollback_request(pg)
+        current_resource = LanternResource[pg.rollback_target]
+        if current_resource.nil?
+          fail CloverError.new(404, "Not Found", "rollback_target not found")
+        end
+
+        pg.incr_rollback_switchover
         response.status = 200
         r.halt
       end
diff --git a/serializers/api/lantern.rb b/serializers/api/lantern.rb
index 732a71400..d995cfec3 100644
--- a/serializers/api/lantern.rb
+++ b/serializers/api/lantern.rb
@@ -28,7 +28,8 @@ def self.base(pg)
       db_user: pg.db_user,
       db_user_password: pg.db_user_password,
       repl_user: pg.repl_user,
-      repl_password: pg.repl_password
+      repl_password: pg.repl_password,
+      pg_version: pg.pg_version
     }
   end
 
diff --git a/spec/lib/validation_spec.rb b/spec/lib/validation_spec.rb
index 3e7408edc..d23c97cff 100644
--- a/spec/lib/validation_spec.rb
+++ b/spec/lib/validation_spec.rb
@@ -172,11 +172,11 @@
 
     describe "#validate_rollback_request" do
       it "valid request" do
-        expect(described_class.validate_rollback_request("test-id")).to be_nil
+        expect(described_class.validate_rollback_request(instance_double(LanternResource, rollback_target: LanternResource.generate_uuid))).to be_nil
       end
 
       it "invalid version" do
-        expect { described_class.validate_rollback_request("") }.to raise_error described_class::ValidationFailed
+        expect { described_class.validate_rollback_request(instance_double(LanternResource, rollback_target: nil)) }.to raise_error described_class::ValidationFailed
       end
     end
 
diff --git a/spec/model/lantern/lantern_resource_spec.rb b/spec/model/lantern/lantern_resource_spec.rb
index 8e3b8b522..33bb9e166 100644
--- a/spec/model/lantern/lantern_resource_spec.rb
+++ b/spec/model/lantern/lantern_resource_spec.rb
@@ -336,4 +336,33 @@
       expect(lantern_resource.get_logical_replication_lag("test_slot")).to be(0)
     end
   end
+
+  describe "#rollback_switchover" do
+    it "performs a rollback switchover successfully" do
+      current_representative_server = instance_double(LanternServer, domain: "example.com", vm: instance_double(GcpVm, sshable: instance_double(Sshable, host: "127.0.0.1")))
+      old_representative_server = instance_double(LanternServer, domain: nil, vm: instance_double(GcpVm, sshable: instance_double(Sshable, host: "127.0.0.2")))
+      current_resource = instance_double(described_class, representative_server: current_representative_server)
+      expect(lantern_resource).to receive(:rollback_target).and_return("test-target").at_least(:once)
+      expect(lantern_resource).to receive(:representative_server).and_return(old_representative_server).at_least(:once)
+      allow(described_class).to receive(:[]).with(lantern_resource.rollback_target).and_return(current_resource)
+
+      expect(current_resource.representative_server).to receive(:stop_container).with(1).and_return(true).at_least(:once)
+
+      expect(old_representative_server).to receive(:start_container)
+
+      cf_client = instance_double(Dns::Cloudflare)
+      allow(Dns::Cloudflare).to receive(:new).and_return(cf_client)
+      expect(cf_client).to receive(:upsert_dns_record).with(
+        current_resource.representative_server.domain,
+        old_representative_server.vm.sshable.host
+      )
+
+      expect(old_representative_server).to receive(:update).with(domain: current_resource.representative_server.domain)
+      expect(current_resource.representative_server).to receive(:update).with(domain: nil)
+
+      expect(lantern_resource).to receive(:update).with(rollback_target: nil)
+
+      expect { lantern_resource.rollback_switchover }.not_to raise_error
+    end
+  end
 end
diff --git a/spec/prog/lantern/lantern_resource_nexus_spec.rb b/spec/prog/lantern/lantern_resource_nexus_spec.rb
index edc63ae70..bf642c02c 100644
--- a/spec/prog/lantern/lantern_resource_nexus_spec.rb
+++ b/spec/prog/lantern/lantern_resource_nexus_spec.rb
@@ -401,7 +401,7 @@
       expect(lantern_resource).to receive(:parent).and_return(parent).at_least(:once)
 
       expect(lantern_resource).to receive(:update).with(display_state: nil)
-      expect(parent).to receive(:update).with(display_state: nil)
+      expect(parent).to receive(:update).with(display_state: nil, rollback_target: lantern_resource.id)
 
       expect(lantern_resource).to receive(:update).with(parent_id: nil)
       expect(lantern_resource).to receive(:timeline).and_return(timeline)
@@ -500,4 +500,41 @@
       expect { nx.wait_switch_dns }.to hop("finish_take_over")
     end
   end
+
+  describe "#rollback_switchover" do
+    it "rollbacks switchover and hops to wait_rollback_switchover" do
+      expect(lantern_resource).to receive(:rollback_switchover)
+      expect { nx.rollback_switchover }.to hop("wait_rollback_switchover")
+    end
+
+    it "waits for rollback switchover and naps if dns is not updated yet" do
+      representative_server = instance_double(LanternServer)
+      expect(lantern_resource).to receive(:representative_server).and_return(representative_server).at_least(:once)
+      expect(representative_server).to receive(:is_dns_correct?).and_return(false)
+      expect { nx.wait_rollback_switchover }.to nap 10
+    end
+
+    it "waits for rollback switchover and naps if can not connnect" do
+      representative_server = instance_double(LanternServer)
+      expect(lantern_resource).to receive(:representative_server).and_return(representative_server).at_least(:once)
+      expect(Sequel).to receive(:connect).and_return(DB)
+      expect(representative_server).to receive(:is_dns_correct?).and_return(true)
+      expect(DB).to receive(:[]).with("SELECT 1").and_raise
+      expect { nx.wait_rollback_switchover }.to nap 10
+    end
+
+    it "waits for rollback switchover and hops to wait server" do
+      representative_server = instance_double(LanternServer)
+      expect(lantern_resource).to receive(:representative_server).and_return(representative_server).at_least(:once)
+      expect(representative_server).to receive(:is_dns_correct?).and_return(true)
+
+      expect(Sequel).to receive(:connect).and_return(DB)
+      res = instance_double(Sequel::Dataset)
+      expect(res).to receive(:first)
+      expect(DB).to receive(:[]).with("SELECT 1").and_return(res)
+
+      expect(lantern_resource).to receive(:set_to_readonly).with(status: "off")
+      expect { nx.wait_rollback_switchover }.to hop("wait_servers")
+    end
+  end
 end
diff --git a/spec/routes/api/project/location/lantern_spec.rb b/spec/routes/api/project/location/lantern_spec.rb
index 55e768202..933c1759b 100644
--- a/spec/routes/api/project/location/lantern_spec.rb
+++ b/spec/routes/api/project/location/lantern_spec.rb
@@ -323,7 +323,7 @@
       end
     end
 
-    describe "upgrade-with-replica" do
+    describe "logical-replica" do
       it "creates a new replica" do
         expect(Project).to receive(:from_ubid).and_return(project).at_least(:once)
         query_res = class_double(LanternResource, first: pg)
@@ -336,7 +336,7 @@
           pg_upgrade: nil
         ).and_return(instance_double(Strand, id: pg.id))
 
-        post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/upgrade-with-replica", {lantern_version: "", extras_version: "", minor_version: "", pg_upgrade: ""}
+        post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/logical-replica", {lantern_version: "", extras_version: "", minor_version: "", pg_upgrade: ""}
         expect(last_response.status).to eq(200)
       end
 
@@ -352,7 +352,7 @@
           pg_upgrade: {"lantern_version" => "0.6.0", "extras_version" => "0.6.0", "minor_version" => "1", "pg_version" => "17"}
         ).and_return(instance_double(Strand, id: pg.id))
 
-        post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/upgrade-with-replica", {
+        post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/logical-replica", {
           lantern_version: "0.5.0",
           extras_version: "0.5.0",
           minor_version: "1",
@@ -363,11 +363,35 @@
     end
 
     describe "switchover" do
-      it "performs a switchover" do
+      it "fails because no parent" do
         expect(Project).to receive(:from_ubid).and_return(project).at_least(:once)
         query_res = class_double(LanternResource, first: pg)
         allow(query_res).to receive(:where).and_return(query_res)
         expect(project).to receive(:lantern_resources_dataset).and_return(query_res)
+
+        post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/switchover"
+        expect(last_response.status).to eq(400)
+      end
+
+      it "fails because not in logical replication mode" do
+        expect(Project).to receive(:from_ubid).and_return(project).at_least(:once)
+        query_res = class_double(LanternResource, first: pg)
+        expect(pg).to receive(:parent).and_return(instance_double(LanternResource))
+        expect(pg).to receive(:logical_replication).and_return(false)
+        allow(query_res).to receive(:where).and_return(query_res)
+        expect(project).to receive(:lantern_resources_dataset).and_return(query_res)
+
+        post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/switchover"
+        expect(last_response.status).to eq(400)
+      end
+
+      it "performs switchover" do
+        expect(Project).to receive(:from_ubid).and_return(project).at_least(:once)
+        query_res = class_double(LanternResource, first: pg)
+        expect(pg).to receive(:parent).and_return(instance_double(LanternResource))
+        expect(pg).to receive(:logical_replication).and_return(true)
+        allow(query_res).to receive(:where).and_return(query_res)
+        expect(project).to receive(:lantern_resources_dataset).and_return(query_res)
         expect(pg).to receive(:incr_switchover_with_parent)
 
         post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/switchover"
@@ -376,20 +400,39 @@
     end
 
     describe "rollback-switchover" do
-      it "rolls back a switchover" do
+      it "fails with 404" do
         expect(Project).to receive(:from_ubid).and_return(project).at_least(:once)
+        expect(Authorization).to receive(:authorize).with(user.id, "Postgres:edit", pg.id)
+
         query_res = class_double(LanternResource, first: pg)
+        target_id = LanternResource.generate_uuid
         allow(query_res).to receive(:where).and_return(query_res)
         expect(project).to receive(:lantern_resources_dataset).and_return(query_res)
-        parent_id = LanternResource.generate_uuid
-        parent_resource = instance_double(LanternResource, id: parent_id)
+        expect(pg).to receive(:rollback_target).and_return(target_id)
+
+        expect(Validation).to receive(:validate_rollback_request).with(pg)
+        allow(LanternResource).to receive(:[]).with(target_id).and_return(nil)
+
+        post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/rollback-switchover"
+        expect(last_response.status).to eq(404)
+      end
+
+      it "rolls back a switchover" do
+        expect(Project).to receive(:from_ubid).and_return(project).at_least(:once)
         expect(Authorization).to receive(:authorize).with(user.id, "Postgres:edit", pg.id)
-        expect(Authorization).to receive(:authorize).with(user.id, "Postgres:edit", parent_resource.id)
-        allow(LanternResource).to receive(:[]).with(parent_id).and_return(parent_resource)
-        expect(Validation).to receive(:validate_rollback_request).with(parent_id)
-        expect(MiscOperations).to receive(:rollback_switchover).with(pg, parent_resource)
 
-        post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/rollback-switchover", {parent_id: parent_id}
+        query_res = class_double(LanternResource, first: pg)
+        target_id = LanternResource.generate_uuid
+        allow(query_res).to receive(:where).and_return(query_res)
+        expect(project).to receive(:lantern_resources_dataset).and_return(query_res)
+        expect(pg).to receive(:rollback_target).and_return(target_id)
+
+        expect(Validation).to receive(:validate_rollback_request).with(pg)
+        allow(LanternResource).to receive(:[]).with(target_id).and_return(instance_double(LanternResource))
+
+        expect(pg).to receive(:incr_rollback_switchover)
+
+        post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/rollback-switchover"
         expect(last_response.status).to eq(200)
       end
     end

From cdabd8bb5e80f04ed24471f406fbf190d2cbe256 Mon Sep 17 00:00:00 2001
From: Varik Matevosyan <varikmatevosyan@gmail.com>
Date: Wed, 13 Nov 2024 20:21:29 +0400
Subject: [PATCH 4/5] accept pg_version in api when creating database, validate
 pg version, add before_run to gcp vm

---
 lib/validation.rb                             | 10 +++++++++
 prog/gcp_vm/nexus.rb                          | 10 +++++++++
 prog/lantern/lantern_resource_nexus.rb        |  1 +
 routes/api/project/lantern.rb                 |  1 +
 routes/api/project/location/lantern.rb        |  8 +++----
 spec/lib/validation_spec.rb                   | 14 ++++++++++++
 spec/prog/gcp_vm/nexus_spec.rb                | 22 +++++++++++++++++++
 .../api/project/location/lantern_spec.rb      |  2 ++
 8 files changed, 64 insertions(+), 4 deletions(-)

diff --git a/lib/validation.rb b/lib/validation.rb
index 4aab9f02f..2e92fd2fd 100644
--- a/lib/validation.rb
+++ b/lib/validation.rb
@@ -91,6 +91,16 @@ def self.validate_version(version, field_name)
     fail ValidationFailed.new({version: msg}) unless version && !version.to_s.strip.empty?
   end
 
+  def self.validate_pg_version(version)
+    if version.nil? || version.to_s.empty?
+      return 17
+    end
+
+    msg = "unsupported pg_version"
+    fail ValidationFailed.new({pg_version: msg}) unless [15, 17].include?(version.to_i)
+    version.to_i
+  end
+
   def self.validate_rollback_request(pg)
     msg = "database does not have rollback_target"
     fail ValidationFailed.new({rollback_target: msg}) unless !pg.rollback_target.nil?
diff --git a/prog/gcp_vm/nexus.rb b/prog/gcp_vm/nexus.rb
index 42cc1d25e..4acea5649 100644
--- a/prog/gcp_vm/nexus.rb
+++ b/prog/gcp_vm/nexus.rb
@@ -263,4 +263,14 @@ def host
     end
     pop "gcp vm deleted"
   end
+
+  def before_run
+    when_destroy_set? do
+      if strand.label != "destroy"
+        hop_destroy
+      elsif strand.stack.count > 1
+        pop "operation is cancelled due to the destruction of vm"
+      end
+    end
+  end
 end
diff --git a/prog/lantern/lantern_resource_nexus.rb b/prog/lantern/lantern_resource_nexus.rb
index b84394a53..47b187028 100644
--- a/prog/lantern/lantern_resource_nexus.rb
+++ b/prog/lantern/lantern_resource_nexus.rb
@@ -70,6 +70,7 @@ def self.assemble(project_id:, location:, name:, target_vm_size:, target_storage
           lantern_version = parent.representative_server.lantern_version
           extras_version = parent.representative_server.extras_version
           minor_version = parent.representative_server.minor_version
+          pg_version = parent.pg_version
         end
 
         target_storage_size_gib = parent.representative_server.target_storage_size_gib
diff --git a/routes/api/project/lantern.rb b/routes/api/project/lantern.rb
index 63111c029..df4326e32 100644
--- a/routes/api/project/lantern.rb
+++ b/routes/api/project/lantern.rb
@@ -51,6 +51,7 @@ class CloverApi
         lantern_version: r.params["lantern_version"],
         extras_version: r.params["extras_version"],
         minor_version: r.params["minor_version"],
+        pg_version: Validation.validate_pg_version(r.params["pg_version"]),
         domain: domain,
         db_name: r.params["db_name"],
         db_user: r.params["db_user"],
diff --git a/routes/api/project/location/lantern.rb b/routes/api/project/location/lantern.rb
index c53a47abe..74c3af34a 100644
--- a/routes/api/project/location/lantern.rb
+++ b/routes/api/project/location/lantern.rb
@@ -161,10 +161,10 @@ class CloverApi
       r.post "logical-replica" do
         Authorization.authorize(@current_user.id, "Postgres:edit", pg.id)
         Authorization.authorize(@current_user.id, "Postgres:create", @project.id)
-        lantern_version = r.params["lantern_version"].empty? ? nil : r.params["lantern_version"]
-        extras_version = r.params["extras_version"].empty? ? nil : r.params["extras_version"]
-        minor_version = r.params["minor_version"].empty? ? nil : r.params["minor_version"]
-        pg_upgrade = r.params["pg_upgrade"].empty? ? nil : r.params["pg_upgrade"]
+        lantern_version = (r.params["lantern_version"] && r.params["lantern_version"].empty?) ? nil : r.params["lantern_version"]
+        extras_version = (r.params["extras_version"] && r.params["extras_version"].empty?) ? nil : r.params["extras_version"]
+        minor_version = (r.params["minor_version"] && r.params["minor_version"].empty?) ? nil : r.params["minor_version"]
+        pg_upgrade = (r.params["pg_upgrade"] && r.params["pg_upgrade"].empty?) ? nil : r.params["pg_upgrade"]
         st = pg.create_logical_replica(
           lantern_version: lantern_version,
           extras_version: extras_version,
diff --git a/spec/lib/validation_spec.rb b/spec/lib/validation_spec.rb
index d23c97cff..70ee5c076 100644
--- a/spec/lib/validation_spec.rb
+++ b/spec/lib/validation_spec.rb
@@ -170,6 +170,20 @@
       end
     end
 
+    describe "#validate_pg_version" do
+      it "valid version" do
+        expect(described_class.validate_pg_version("17")).to be(17)
+        expect(described_class.validate_pg_version("15")).to be(15)
+        expect(described_class.validate_pg_version("")).to be(17)
+        expect(described_class.validate_pg_version(nil)).to be(17)
+      end
+
+      it "invalid version" do
+        expect { described_class.validate_pg_version("as") }.to raise_error described_class::ValidationFailed
+        expect { described_class.validate_pg_version("16") }.to raise_error described_class::ValidationFailed
+      end
+    end
+
     describe "#validate_rollback_request" do
       it "valid request" do
         expect(described_class.validate_rollback_request(instance_double(LanternResource, rollback_target: LanternResource.generate_uuid))).to be_nil
diff --git a/spec/prog/gcp_vm/nexus_spec.rb b/spec/prog/gcp_vm/nexus_spec.rb
index 27739cdf6..a6ca33b46 100644
--- a/spec/prog/gcp_vm/nexus_spec.rb
+++ b/spec/prog/gcp_vm/nexus_spec.rb
@@ -325,4 +325,26 @@
       expect(nx.host).to eq("1.1.1.1")
     end
   end
+
+  describe "#before_run" do
+    it "hops to destroy" do
+      expect(nx).to receive(:when_destroy_set?).and_yield
+      expect { nx.before_run }.to hop("destroy")
+    end
+
+    it "pops if already in the destroy state and has stack items" do
+      expect(nx).to receive(:when_destroy_set?).and_yield
+      expect(nx.strand).to receive(:label).and_return("destroy").at_least(:once)
+      frame = {"link" => ["GcpVm::Nexus", "wait"]}
+      expect(nx).to receive(:frame).and_return(frame)
+      expect(nx.strand).to receive(:stack).and_return([JSON.generate(frame), JSON.generate(frame)]).at_least(:once)
+      expect { nx.before_run }.to hop("wait", "GcpVm::Nexus")
+    end
+
+    it "does not hop to destroy if already in the destroy state" do
+      expect(nx).to receive(:when_destroy_set?).and_yield
+      expect(nx.strand).to receive(:label).and_return("destroy").at_least(:once)
+      expect { nx.before_run }.not_to hop("destroy")
+    end
+  end
 end
diff --git a/spec/routes/api/project/location/lantern_spec.rb b/spec/routes/api/project/location/lantern_spec.rb
index 933c1759b..a7e2e4817 100644
--- a/spec/routes/api/project/location/lantern_spec.rb
+++ b/spec/routes/api/project/location/lantern_spec.rb
@@ -338,6 +338,8 @@
 
         post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/logical-replica", {lantern_version: "", extras_version: "", minor_version: "", pg_upgrade: ""}
         expect(last_response.status).to eq(200)
+        expect(JSON.parse(last_response.body)["id"]).to eq(pg.id)
+        expect(JSON.parse(last_response.body)["name"]).to eq(pg.name)
       end
 
       it "creates a new replica with upgrade request" do

From 987074b4bfef92fc1c7fa8b49614394c8890b225 Mon Sep 17 00:00:00 2001
From: Varik Matevosyan <varikmatevosyan@gmail.com>
Date: Wed, 13 Nov 2024 21:22:25 +0400
Subject: [PATCH 5/5] hop to rollback switchover when set

---
 prog/lantern/lantern_resource_nexus.rb           | 7 ++++++-
 spec/prog/lantern/lantern_resource_nexus_spec.rb | 8 ++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/prog/lantern/lantern_resource_nexus.rb b/prog/lantern/lantern_resource_nexus.rb
index 47b187028..2bef42226 100644
--- a/prog/lantern/lantern_resource_nexus.rb
+++ b/prog/lantern/lantern_resource_nexus.rb
@@ -9,7 +9,7 @@ class Prog::Lantern::LanternResourceNexus < Prog::Base
   extend Forwardable
   def_delegators :lantern_resource, :servers, :representative_server
 
-  semaphore :destroy, :swap_leaders_with_parent, :switchover_with_parent
+  semaphore :destroy, :swap_leaders_with_parent, :switchover_with_parent, :rollback_switchover
 
   def self.assemble(project_id:, location:, name:, target_vm_size:, target_storage_size_gib:, ubid: LanternResource.generate_ubid, ha_type: LanternResource::HaType::NONE, parent_id: nil, restore_target: nil, recovery_target_lsn: nil,
     org_id: nil, db_name: "postgres", db_user: "postgres", db_user_password: nil, superuser_password: nil, repl_password: nil, app_env: Config.rack_env,
@@ -206,6 +206,10 @@ def before_run
       lantern_resource.update(display_state: nil)
     end
 
+    when_rollback_switchover_set? do
+      hop_rollback_switchover
+    end
+
     when_swap_leaders_with_parent_set? do
       if lantern_resource.parent.nil?
         decr_swap_leaders_with_parent
@@ -270,6 +274,7 @@ def before_run
   end
 
   label def rollback_switchover
+    decr_rollback_switchover
     lantern_resource.rollback_switchover
     hop_wait_rollback_switchover
   end
diff --git a/spec/prog/lantern/lantern_resource_nexus_spec.rb b/spec/prog/lantern/lantern_resource_nexus_spec.rb
index bf642c02c..21a6d25a8 100644
--- a/spec/prog/lantern/lantern_resource_nexus_spec.rb
+++ b/spec/prog/lantern/lantern_resource_nexus_spec.rb
@@ -282,6 +282,14 @@
       expect(lantern_resource).to receive(:update).with(display_state: "failover")
       expect { nx.wait }.to hop("switchover_with_parent")
     end
+
+    it "hops to rollback_switchover" do
+      expect(lantern_resource).to receive(:required_standby_count).and_return(0)
+      expect(lantern_resource).to receive(:display_state).and_return(nil)
+      expect(lantern_resource).to receive(:servers).and_return([instance_double(LanternServer, strand: instance_double(Strand, label: "wait"))]).at_least(:once)
+      expect(nx).to receive(:when_rollback_switchover_set?).and_yield
+      expect { nx.wait }.to hop("rollback_switchover")
+    end
   end
 
   describe "#destroy" do