Add doctor query to check disk usage

migrate/20240519_lantern_doctor_queries.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    # check disk space usage
+    run "INSERT INTO lantern_doctor_query (id, name, db_name, schedule, condition, fn_label, type, severity, response_type)
+         VALUES ('09b1b1d1-7095-89b7-8ae4-158e15e11871', 'Lantern Server Disk Usage', 'postgres', '*/5 * * * *', 'unknown', 'check_disk_space_usage', 'system', 'error', 'rows')"
+
+    # Create semaphores for all lantern doctors to sync system queries
+    run "INSERT INTO semaphore (id, strand_id, name) SELECT gen_random_uuid(), id, 'sync_system_queries' FROM strand s WHERE s.prog = 'Lantern::LanternDoctorNexus'"
+  end
+end

model/lantern/lantern_doctor_query.rb

@@ -151,4 +151,16 @@ def check_daemon_embedding_jobs(db, query_user)
 
     failed ? "t" : "f"
   end
+
+  def check_disk_space_usage(_db, _query_user)
+    output = ""
+    doctor.resource.servers.each do |serv|
+      usage_percent = serv.vm.sshable.cmd("df | awk '$1 == \"/dev/root\" {print $5}' | sed 's/%//'").strip.to_i
+      if usage_percent > 90
+        server_type = serv.primary? ? "primary" : "standby"
+        output += "#{server_type} server - usage #{usage_percent}%\n"
+      end
+    end
+    output.chomp
+  end
 end

spec/model/lantern/lantern_doctor_query_spec.rb

@@ -341,6 +341,38 @@
     end
   end
 
+  describe "#check_disk_space_usage" do
+    it "fails if primary disk server usage is above 90%" do
+      serv1 = instance_double(LanternServer, primary?: true, vm: instance_double(GcpVm, sshable: instance_double(Sshable)))
+      serv2 = instance_double(LanternServer, primary?: false, vm: instance_double(GcpVm, sshable: instance_double(Sshable)))
+      expect(serv1.vm.sshable).to receive(:cmd).and_return("91")
+      expect(serv2.vm.sshable).to receive(:cmd).and_return("80")
+      doctor = instance_double(LanternDoctor, resource: instance_double(LanternResource, servers: [serv1, serv2]), ubid: "test-ubid")
+      expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once)
+      expect(lantern_doctor_query.check_disk_space_usage("postgres", "postgres")).to eq("primary server - usage 91%")
+    end
+
+    it "fails if standby disk usage is above 90%" do
+      serv1 = instance_double(LanternServer, primary?: true, vm: instance_double(GcpVm, sshable: instance_double(Sshable)))
+      serv2 = instance_double(LanternServer, primary?: false, vm: instance_double(GcpVm, sshable: instance_double(Sshable)))
+      expect(serv1.vm.sshable).to receive(:cmd).and_return("11")
+      expect(serv2.vm.sshable).to receive(:cmd).and_return("92")
+      doctor = instance_double(LanternDoctor, resource: instance_double(LanternResource, servers: [serv1, serv2]), ubid: "test-ubid")
+      expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once)
+      expect(lantern_doctor_query.check_disk_space_usage("postgres", "postgres")).to eq("standby server - usage 92%")
+    end
+
+    it "succceds if all servers disk usage is under 90%" do
+      serv1 = instance_double(LanternServer, primary?: true, vm: instance_double(GcpVm, sshable: instance_double(Sshable)))
+      serv2 = instance_double(LanternServer, primary?: false, vm: instance_double(GcpVm, sshable: instance_double(Sshable)))
+      expect(serv1.vm.sshable).to receive(:cmd).and_return("11")
+      expect(serv2.vm.sshable).to receive(:cmd).and_return("22")
+      doctor = instance_double(LanternDoctor, resource: instance_double(LanternResource, servers: [serv1, serv2]), ubid: "test-ubid")
+      expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once)
+      expect(lantern_doctor_query.check_disk_space_usage("postgres", "postgres")).to eq("")
+    end
+  end
+
   describe "#check_daemon_embedding_jobs" do
     it "fails if not backend db connection" do
       expect(LanternBackend).to receive(:db).and_return(nil)