remove checks for old embeddings table

lanterndata · Jun 4, 2024 · ec960c6 · ec960c6
1 parent 9f3b7e3
commit ec960c6
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 73 deletions.
diff --git a/config.rb b/config.rb
@@ -153,7 +153,6 @@ def self.e2e_test?
   override :lantern_backup_bucket, "walg-dev-backups"
   override :e2e_test, "0"
   override :backup_retention_days, 7, int
-  optional :lantern_backend_database_url, string
   override :lantern_log_dataset, "lantern_logs", string
 
   # Cloudflare

diff --git a/db.rb b/db.rb
@@ -26,11 +26,3 @@
 # DB.extension :date_arithmetic
 DB.extension :pg_json, :pg_auto_parameterize, :pg_timestamptz, :pg_range, :pg_array
 Sequel.extension :pg_range_ops
-
-module LanternBackend
-  @@db = Sequel.connect(Config.lantern_backend_database_url, max_connections: Config.db_pool, pool_timeout: Config.database_timeout) if Config.lantern_backend_database_url
-
-  def self.db
-    @@db
-  end
-end
diff --git a/model/lantern/lantern_doctor_query.rb b/model/lantern/lantern_doctor_query.rb
@@ -127,22 +127,8 @@ def run
   end
 
   def check_daemon_embedding_jobs(db, query_user)
-    if !LanternBackend.db
-      fail "No connection to lantern backend database specified"
-    end
-
-    # TODO:: the backend db connection will be removed after daemon version update is done
-    jobs = LanternBackend.db
-      .select(:schema, :table, :src_column, :dst_column)
-      .from(:embedding_generation_jobs)
-      .where(database_id: doctor.resource.name)
-      .where(Sequel.like(:db_connection, "%/#{db}"))
-      .where(Sequel.lit("init_finished_at IS NOT NULL"))
-      .where(Sequel.lit("canceled_at IS NULL"))
-      .all
-
     lantern_server = doctor.resource.representative_server
-    new_jobs_exists = lantern_server.run_query(<<SQL).chomp.strip
+    jobs_table_exists = lantern_server.run_query(<<SQL).chomp.strip
       SELECT EXISTS (
        SELECT FROM pg_catalog.pg_class c
        JOIN   pg_catalog.pg_namespace n ON n.oid = c.relnamespace
@@ -152,15 +138,15 @@ def check_daemon_embedding_jobs(db, query_user)
      );
 SQL
 
-    if new_jobs_exists == "t"
-      new_jobs = lantern_server.run_query("SELECT \"schema\", \"table\", src_column, dst_column FROM _lantern_internal.embedding_generation_jobs WHERE init_finished_at IS NOT NULL AND canceled_at IS NULL;")
+    if jobs_table_exists == "f"
+      return "f"
+    end
 
-      new_jobs = new_jobs.split("\n").map do |row|
-        values = row.split(",")
-        {schema: values[0], table: values[1], src_column: values[2], dst_column: values[3]}
-      end
+    jobs = lantern_server.run_query("SELECT \"schema\", \"table\", src_column, dst_column FROM _lantern_internal.embedding_generation_jobs WHERE init_finished_at IS NOT NULL AND canceled_at IS NULL;")
 
-      jobs.concat new_jobs
+    jobs = jobs.chomp.strip.split("\n").map do |row|
+      values = row.split(",")
+      {schema: values[0], table: values[1], src_column: values[2], dst_column: values[3]}
     end
 
     if jobs.empty?

diff --git a/spec/model/lantern/lantern_doctor_query_spec.rb b/spec/model/lantern/lantern_doctor_query_spec.rb
@@ -374,29 +374,14 @@
   end
 
   describe "#check_daemon_embedding_jobs" do
-    it "fails if not backend db connection" do
-      expect(LanternBackend).to receive(:db).and_return(nil)
-      expect { lantern_doctor_query.check_daemon_embedding_jobs "postgres", "postgres" }.to raise_error "No connection to lantern backend database specified"
-    end
-
     it "get jobs and fails" do
       serv = instance_double(LanternServer, ubid: "test-ubid")
       resource = instance_double(LanternResource, representative_server: serv, db_user: "test", name: "test-res", id: "6181ddb3-0002-8ad0-9aeb-084832c9273b")
       doctor = instance_double(LanternDoctor, resource: resource, ubid: "test-ubid")
       expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once)
-      expect(LanternBackend).to receive(:db).and_return(DB).at_least(:once)
-      expect(LanternBackend.db).to receive(:select)
-        .and_return(instance_double(Sequel::Dataset,
-          from: instance_double(Sequel::Dataset,
-            where: instance_double(Sequel::Dataset,
-              where: instance_double(Sequel::Dataset,
-                where: instance_double(Sequel::Dataset,
-                  where: instance_double(Sequel::Dataset,
-                    all: [{schema: "public", table: "test", src_column: "test-src", dst_column: "test-dst"}])))))))
       expect(serv).to receive(:run_query).and_return("t")
       expect(serv).to receive(:run_query).and_return("public,test2,test-src,test-dst\npublic,test3,test-src,test-dst")
       expect(serv).to receive(:run_query).and_return("f")
-      expect(serv).to receive(:run_query).and_return("f")
       expect(serv).to receive(:run_query).and_return("t")
       expect(lantern_doctor_query.check_daemon_embedding_jobs("postgres", "postgres")).to eq("t")
     end
@@ -406,37 +391,31 @@
       resource = instance_double(LanternResource, representative_server: serv, db_user: "test", name: "test-res", id: "6181ddb3-0002-8ad0-9aeb-084832c9273b")
       doctor = instance_double(LanternDoctor, resource: resource, ubid: "test-ubid")
       expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once)
-      expect(LanternBackend).to receive(:db).and_return(DB).at_least(:once)
-      expect(LanternBackend.db).to receive(:select)
-        .and_return(instance_double(Sequel::Dataset,
-          from: instance_double(Sequel::Dataset,
-            where: instance_double(Sequel::Dataset,
-              where: instance_double(Sequel::Dataset,
-                where: instance_double(Sequel::Dataset,
-                  where: instance_double(Sequel::Dataset,
-                    all: [])))))))
+      expect(serv).to receive(:run_query).and_return("t")
+      expect(serv).to receive(:run_query).and_return(" \n")
+      expect(lantern_doctor_query.check_daemon_embedding_jobs("postgres", "postgres")).to eq("f")
+    end
+
+    it "get jobs and succceds" do
+      serv = instance_double(LanternServer, ubid: "test-ubid")
+      resource = instance_double(LanternResource, representative_server: serv, db_user: "test", name: "test-res", id: "6181ddb3-0002-8ad0-9aeb-084832c9273b")
+      doctor = instance_double(LanternDoctor, resource: resource, ubid: "test-ubid")
+      expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once)
+      expect(serv).to receive(:run_query).and_return("t")
+      expect(serv).to receive(:run_query).and_return("public,test2,test-src,test-dst\npublic,test3,test-src,test-dst")
+      expect(serv).to receive(:run_query).and_return("f")
       expect(serv).to receive(:run_query).and_return("f")
       expect(lantern_doctor_query.check_daemon_embedding_jobs("postgres", "postgres")).to eq("f")
     end
-  end
 
-  it "get jobs and succceds" do
-    serv = instance_double(LanternServer, ubid: "test-ubid")
-    resource = instance_double(LanternResource, representative_server: serv, db_user: "test", name: "test-res", id: "6181ddb3-0002-8ad0-9aeb-084832c9273b")
-    doctor = instance_double(LanternDoctor, resource: resource, ubid: "test-ubid")
-    expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once)
-    expect(LanternBackend).to receive(:db).and_return(DB).at_least(:once)
-    expect(LanternBackend.db).to receive(:select)
-      .and_return(instance_double(Sequel::Dataset,
-        from: instance_double(Sequel::Dataset,
-          where: instance_double(Sequel::Dataset,
-            where: instance_double(Sequel::Dataset,
-              where: instance_double(Sequel::Dataset,
-                where: instance_double(Sequel::Dataset,
-                  all: [{schema: "public", table: "test", src_column: "test-src", dst_column: "test-dst"}])))))))
-    expect(serv).to receive(:run_query).and_return("f")
-    expect(serv).to receive(:run_query).and_return("f")
-    expect(lantern_doctor_query.check_daemon_embedding_jobs("postgres", "postgres")).to eq("f")
+    it "job table does not exist" do
+      serv = instance_double(LanternServer, ubid: "test-ubid")
+      resource = instance_double(LanternResource, representative_server: serv, db_user: "test", name: "test-res", id: "6181ddb3-0002-8ad0-9aeb-084832c9273b")
+      doctor = instance_double(LanternDoctor, resource: resource, ubid: "test-ubid")
+      expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once)
+      expect(serv).to receive(:run_query).and_return("f")
+      expect(lantern_doctor_query.check_daemon_embedding_jobs("postgres", "postgres")).to eq("f")
+    end
   end
 
   describe "#page" do