Fix index_prewarm to run with deamonizer instead of blocking thread

model/lantern/lantern_server.rb

@@ -56,17 +56,17 @@ def run_query_all(query)
   end
 
   def display_state
+    return "deleting" if destroy_set? || strand.label == "destroy"
+    return "stopped" if vm.display_state == "stopped"
+    return "stopping" if vm.display_state == "stopping"
+    return "starting" if vm.display_state == "starting"
+    return "failed" if vm.display_state == "failed"
     return "domain setup" if strand.label.include?("domain")
-    return "ssl setup" if strand.label.include?("setup_ssl")
+    return "ssl setup" if strand.label == "setup_ssl"
     return "updating" if strand.label.include?("update")
-    return "updating" if strand.label.include?("init_sql")
-    return "stopped" if vm.display_state.include?("stopped")
-    return "stopping" if vm.display_state.include?("stopping")
-    return "starting" if vm.display_state.include?("starting")
-    return "failed" if vm.display_state.include?("failed")
-    return "unavailable" if strand.label.include?("wait_db_available")
-    return "running" if ["wait"].include?(strand.label)
-    return "deleting" if destroy_set? || strand.label == "destroy"
+    return "updating" if strand.label == "init_sql"
+    return "unavailable" if strand.label == "wait_db_available"
+    return "running" if strand.label == "wait"
     "creating"
   end
 
@@ -184,8 +184,8 @@ def check_pulse(session:, previous_pulse:)
     pulse
   end
 
-  def prewarm_indexes
-    query = <<SQL
+  def prewarm_indexes_query
+    <<SQL
     SELECT i.relname, pg_prewarm(i.relname::text)
 FROM pg_class t
 JOIN pg_index ix ON t.oid = ix.indrelid
@@ -194,7 +194,6 @@ def prewarm_indexes
 JOIN pg_namespace n ON n.oid = i.relnamespace
 WHERE a.amname = 'lantern_hnsw';
 SQL
-    run_query_all(query)
   end
 
   # def failover_target

prog/lantern/lantern_resource_nexus.rb

@@ -112,7 +112,8 @@ def before_run
 
   label def start
     nap 5 unless representative_server.vm.strand.label == "wait"
-    register_deadline(:failed_provisioning, 10 * 60)
+    # TODO fix
+    # register_deadline(:failed_provisioning, 10 * 60)
     # bud self.class, frame, :trigger_pg_current_xact_id_on_parent if lantern_resource.parent
 
     # hop_wait_trigger_pg_current_xact_id_on_parent

prog/lantern/lantern_server_nexus.rb

@@ -546,16 +546,21 @@ def available?
   end
 
   label def prewarm_indexes
-    begin
-      lantern_server.prewarm_indexes
-    rescue => e
-      Clog.emit("Error while prewarming indexes") { {lantern_server: lantern_server.id, error: e} }
-      logs = {"stdout" => "", "stderr" => e}
+    case vm.sshable.cmd("common/bin/daemonizer --check prewarm_indexes")
+    when "NotStarted"
+      vm.sshable.cmd("common/bin/daemonizer 'sudo lantern/bin/exec_all' prewarm_indexes", stdin: lantern_server.prewarm_indexes_query)
+    when "Succeeded"
+      vm.sshable.cmd("common/bin/daemonizer --clean prewarm_indexes")
+      Page.from_tag_parts("LanternPrewarmFailed", lantern_server.id)&.incr_resolve
+      pop "lantern index prewarm success"
+    when "Failed"
+      logs = JSON.parse(vm.sshable.cmd("common/bin/daemonizer --logs prewarm_indexes"))
+      Clog.emit("Lantern index prewarm failed") { {logs: logs, name: lantern_server.resource.name, lantern_server: lantern_server.id} }
       Prog::PageNexus.assemble_with_logs("Lantern prewarm indexes failed", [lantern_server.resource.ubid, lantern_server.ubid], logs, "warning", "LanternPrewarmFailed", lantern_server.ubid)
+      vm.sshable.cmd("common/bin/daemonizer --clean prewarm_indexes")
       pop "lantern index prewarm failed"
     end
 
-    Page.from_tag_parts("LanternPrewarmFailed", lantern_server.id)&.incr_resolve
-    pop "lantern index prewarm success"
+    nap 30
   end
 end

spec/model/lantern/lantern_server_spec.rb

@@ -33,21 +33,25 @@
 
   describe "#display_state" do
     it "shows domain setup" do
+      expect(lantern_server.vm).to receive(:display_state).and_return("running").at_least(:once)
       expect(lantern_server).to receive(:strand).and_return(instance_double(Strand, label: "setup domain")).at_least(:once)
       expect(lantern_server.display_state).to eq("domain setup")
     end
 
     it "shows ssl setup" do
+      expect(lantern_server.vm).to receive(:display_state).and_return("running").at_least(:once)
       expect(lantern_server).to receive(:strand).and_return(instance_double(Strand, label: "setup_ssl")).at_least(:once)
       expect(lantern_server.display_state).to eq("ssl setup")
     end
 
     it "shows updating" do
+      expect(lantern_server.vm).to receive(:display_state).and_return("running").at_least(:once)
       expect(lantern_server).to receive(:strand).and_return(instance_double(Strand, label: "update_extension")).at_least(:once)
       expect(lantern_server.display_state).to eq("updating")
     end
 
     it "shows updating if init_sql" do
+      expect(lantern_server.vm).to receive(:display_state).and_return("running").at_least(:once)
       expect(lantern_server).to receive(:strand).and_return(instance_double(Strand, label: "init_sql")).at_least(:once)
       expect(lantern_server.display_state).to eq("updating")
     end
@@ -59,16 +63,13 @@
     end
 
     it "shows deleting" do
-      expect(lantern_server.vm).to receive(:display_state).and_return("running").at_least(:once)
       expect(lantern_server).to receive(:destroy_set?).and_return(false).at_least(:once)
       expect(lantern_server).to receive(:strand).and_return(instance_double(Strand, label: "destroy")).at_least(:once)
       expect(lantern_server.display_state).to eq("deleting")
     end
 
     it "shows deleting if destroy set" do
-      expect(lantern_server.vm).to receive(:display_state).and_return("running").at_least(:once)
       expect(lantern_server).to receive(:destroy_set?).and_return(true).at_least(:once)
-      expect(lantern_server).to receive(:strand).and_return(instance_double(Strand, label: "unknown")).at_least(:once)
       expect(lantern_server.display_state).to eq("deleting")
     end
 
@@ -552,8 +553,7 @@
 JOIN pg_namespace n ON n.oid = i.relnamespace
 WHERE a.amname = 'lantern_hnsw';
 SQL
-      expect(lantern_server).to receive(:run_query_all).with(query)
-      expect { lantern_server.prewarm_indexes }.not_to raise_error
+      expect(lantern_server.prewarm_indexes_query).to eq(query)
     end
   end
 end

spec/prog/lantern/lantern_resource_nexus_spec.rb

@@ -114,7 +114,7 @@
 
     it "registers deadline and hops" do
       expect(lantern_resource.representative_server.vm).to receive(:strand).and_return(instance_double(Strand, label: "wait"))
-      expect(nx).to receive(:register_deadline)
+      # expect(nx).to receive(:register_deadline)
       expect { nx.start }.to hop("wait_servers")
     end
 

spec/prog/lantern/lantern_server_nexus_spec.rb

@@ -828,22 +828,41 @@
   end
 
   describe "#prewarm_indexes" do
-    it "calls prewarm and exits" do
-      expect(lantern_server).to receive(:prewarm_indexes)
-      expect(Page).to receive(:from_tag_parts).at_least(:once)
-      expect { nx.prewarm_indexes }.to exit({"msg" => "lantern index prewarm success"})
+    it "naps if in progress" do
+      expect(lantern_server.vm.sshable).to receive(:cmd).with("common/bin/daemonizer --check prewarm_indexes").and_return("InProgress")
+      expect { nx.prewarm_indexes }.to nap(30)
     end
 
-    it "calls prewarm, resolves page and exits" do
-      expect(lantern_server).to receive(:prewarm_indexes)
+    it "calls prewarm if not started" do
+      expect(lantern_server).to receive(:prewarm_indexes_query).and_return("test").at_least(:once)
+      expect(lantern_server.vm.sshable).to receive(:cmd).with("common/bin/daemonizer --check prewarm_indexes").and_return("NotStarted")
+      expect(lantern_server.vm.sshable).to receive(:cmd).with("common/bin/daemonizer 'sudo lantern/bin/exec_all' prewarm_indexes", stdin: lantern_server.prewarm_indexes_query)
+      expect { nx.prewarm_indexes }.to nap(30)
+    end
+
+    it "cleans and resolves page" do
+      expect(lantern_server.vm.sshable).to receive(:cmd).with("common/bin/daemonizer --check prewarm_indexes").and_return("Succeeded")
+      expect(lantern_server.vm.sshable).to receive(:cmd).with("common/bin/daemonizer --clean prewarm_indexes")
       page = instance_double(Page)
       expect(page).to receive(:incr_resolve).at_least(:once)
       expect(Page).to receive(:from_tag_parts).at_least(:once).and_return(page)
       expect { nx.prewarm_indexes }.to exit({"msg" => "lantern index prewarm success"})
     end
 
-    it "calls prewarm and creates page" do
-      expect(lantern_server).to receive(:prewarm_indexes).and_raise
+    it "cleans and exits" do
+      expect(lantern_server.vm.sshable).to receive(:cmd).with("common/bin/daemonizer --check prewarm_indexes").and_return("Succeeded")
+      expect(lantern_server.vm.sshable).to receive(:cmd).with("common/bin/daemonizer --clean prewarm_indexes")
+      expect(Page).to receive(:from_tag_parts).at_least(:once).and_return(nil)
+      expect { nx.prewarm_indexes }.to exit({"msg" => "lantern index prewarm success"})
+    end
+
+    it "fails and creates page" do
+      expect(lantern_server.vm.sshable).to receive(:cmd).with("common/bin/daemonizer --check prewarm_indexes").and_return("Failed")
+      expect(lantern_server.vm.sshable).to receive(:cmd).with("common/bin/daemonizer --logs prewarm_indexes").and_return(JSON.generate({"stdout" => "test"}))
+      expect(lantern_server.vm.sshable).to receive(:cmd).with("common/bin/daemonizer --clean prewarm_indexes")
+      page = instance_double(Page)
+      expect(page).to receive(:incr_resolve).at_least(:once)
+      expect(Page).to receive(:from_tag_parts).at_least(:once).and_return(page)
       expect(lantern_server).to receive(:ubid)
       expect(lantern_server.resource).to receive(:ubid)
       expect(Prog::PageNexus).to receive(:assemble_with_logs).at_least(:once)