handle rhizome updates for lantern doctor automatically based on heal…

…thcheck output

prog/lantern/lantern_doctor_nexus.rb

@@ -80,6 +80,17 @@ def before_run
           logs = JSON.parse(vm.sshable.cmd("common/bin/daemonizer --logs #{query.task_name}"))
           all_output = []
 
+          if status == "Failed" && logs["stderr"].chomp == "update_needed"
+            is_updating = server.strand.label == "wait_update_rhizome"
+            will_update = !Semaphore.where(strand_id: server.strand.id, name: "update_rhizome").first.nil?
+            if !is_updating && !will_update
+              server.incr_update_rhizome
+            end
+
+            vm.sshable.cmd("common/bin/daemonizer --clean #{query.task_name}")
+            next
+          end
+
           if !logs["stdout"].empty?
             # stdout will be [{ "db": string, "result": string, "success": bool }]
             begin

rhizome/lantern/bin/doctor/run_query

@@ -146,6 +146,15 @@ end
 exit_code = 0
 
 response = []
+fn_label = $configure_hash["query"]["fn_label"]
+is_system = $configure_hash["query"]["is_system"]
+
+if is_system && fn_label && !SystemQueries.respond_to?(fn_label)
+  # rhyzome is not synced yet
+  warn "update_needed"
+  exit(1)
+end
+
 dbs.each do |db|
   success, res = run_for_db(db)
   if !success

spec/prog/lantern/lantern_doctor_nexus_spec.rb

@@ -194,6 +194,51 @@
         expect { nx.wait_queries }.to hop("wait")
       end
 
+      it "handles update_needed" do
+        query = instance_double(LanternDoctorQuery, servers: [server], db_name: "postgres", task_name: "test_query")
+        expect(lantern_doctor).to receive(:queries).and_return([query])
+        allow(sshable).to receive(:cmd).with("common/bin/daemonizer --check test_query").and_return("Failed")
+        allow(sshable).to receive(:cmd).with("common/bin/daemonizer --logs test_query").and_return(JSON.generate({"stdout" => "error parse", "stderr" => "update_needed"}))
+        expect(sshable).to receive(:cmd).with("common/bin/daemonizer --clean test_query")
+
+        expect(server).to receive(:strand).and_return(instance_double(Strand, id: "test", label: "wait")).at_least(:once)
+        expect(Semaphore).to receive(:where).with(name: "update_rhizome", strand_id: "test").and_return(instance_double(Sequel::Dataset, first: nil))
+        expect(query).not_to receive(:update).with(condition: "failed", last_checked: instance_of(Time))
+        expect(server).to receive(:incr_update_rhizome)
+
+        expect { nx.wait_queries }.to hop("wait")
+      end
+
+      it "skips update_needed if already updating" do
+        query = instance_double(LanternDoctorQuery, servers: [server], db_name: "postgres", task_name: "test_query")
+        expect(lantern_doctor).to receive(:queries).and_return([query])
+        allow(sshable).to receive(:cmd).with("common/bin/daemonizer --check test_query").and_return("Failed")
+        allow(sshable).to receive(:cmd).with("common/bin/daemonizer --logs test_query").and_return(JSON.generate({"stdout" => "error parse", "stderr" => "update_needed"}))
+        expect(sshable).to receive(:cmd).with("common/bin/daemonizer --clean test_query")
+
+        expect(server).to receive(:strand).and_return(instance_double(Strand, id: "test", label: "wait_update_rhizome")).at_least(:once)
+        expect(Semaphore).to receive(:where).with(name: "update_rhizome", strand_id: "test").and_return(instance_double(Sequel::Dataset, first: nil))
+        expect(query).not_to receive(:update).with(condition: "failed", last_checked: instance_of(Time))
+        expect(server).not_to receive(:incr_update_rhizome)
+
+        expect { nx.wait_queries }.to hop("wait")
+      end
+
+      it "skips update_needed if will update" do
+        query = instance_double(LanternDoctorQuery, servers: [server], db_name: "postgres", task_name: "test_query")
+        expect(lantern_doctor).to receive(:queries).and_return([query])
+        allow(sshable).to receive(:cmd).with("common/bin/daemonizer --check test_query").and_return("Failed")
+        allow(sshable).to receive(:cmd).with("common/bin/daemonizer --logs test_query").and_return(JSON.generate({"stdout" => "error parse", "stderr" => "update_needed"}))
+        expect(sshable).to receive(:cmd).with("common/bin/daemonizer --clean test_query")
+
+        expect(server).to receive(:strand).and_return(instance_double(Strand, id: "test", label: "wait")).at_least(:once)
+        expect(Semaphore).to receive(:where).with(name: "update_rhizome", strand_id: "test").and_return(instance_double(Sequel::Dataset, first: instance_double(Semaphore)))
+        expect(query).not_to receive(:update).with(condition: "failed", last_checked: instance_of(Time))
+        expect(server).not_to receive(:incr_update_rhizome)
+
+        expect { nx.wait_queries }.to hop("wait")
+      end
+
       it "handles error" do
         query = instance_double(LanternDoctorQuery, servers: [server], db_name: "postgres", task_name: "test_query")
         expect(lantern_doctor).to receive(:queries).and_return([query])