add new mix operations

migrate/20240711_lantern_doctor_queries.rb

@@ -4,7 +4,7 @@
   up do
     # check duplicate sources for embedding jobs
     run "INSERT INTO lantern_doctor_query (id, name, db_name, schedule, condition, fn_label, type, severity, response_type)
-         VALUES ('09f5de22-13fb-89b7-bf5e-75b26faef139', 'Whitespace tokens for embedding job', '*', '*/8 * * * *', 'unknown', 'check_embedding_source_whitespaces', 'system', 'error', 'rows')"
+         VALUES ('09f5de22-13fb-89b7-bf5e-75b26faef139', 'Whitespace tokens for embedding job', '*', '0 5 * * *', 'unknown', 'check_embedding_source_whitespaces', 'system', 'error', 'rows')"
 
     # Create semaphores for all lantern doctors to sync system queries
     run "INSERT INTO semaphore (id, strand_id, name) SELECT gen_random_uuid(), id, 'sync_system_queries' FROM strand s WHERE s.prog = 'Lantern::LanternDoctorNexus'"

misc/misc_operations.rb

@@ -91,6 +91,37 @@ def self.query_on_db(vm, db_name, query)
     vm.sshable.cmd("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec -T postgresql psql -U postgres -t --csv #{db_name}", stdin: query)
   end
 
+  def self.run_script_daemonized(resource_name, filename, task_name)
+    serv = LanternResource[name: resource_name].representative_server
+    serv.vm.sshable.cmd("common/bin/daemonizer 'sudo #{filename}' #{task_name}")
+
+    task_logs resource_name, "#{task_name}"
+  end
+
+  def self.create_all_indexes_concurrently_script(resource_name, filename)
+    serv = LanternResource[name: resource_name].representative_server
+    all_dbs = serv.vm.sshable.cmd("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec postgresql psql -U postgres -P \"footer=off\" -c 'SELECT datname from pg_database' | tail -n +3 | grep -v 'template0' | grep -v 'template1'").strip.split("\n")
+    command_list = []
+    all_dbs.each do |db|
+      db_name = db.strip
+      indexes = MiscOperations.query_on_db(serv.vm, db_name, "SELECT indexdef FROM pg_indexes WHERE indexdef ILIKE '%lantern_hnsw%';").strip.split("\n")
+
+      if !indexes.empty?
+        queries = []
+        indexes.each {
+          queries.push(_1.gsub(/create\s+index/i, 'CREATE INDEX CONCURRENTLY')[1..-2])
+        }
+
+        queries.map do |query|
+          command_list.push("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec -T postgresql psql -U postgres #{db_name} -c \"#{query}\"")
+        end
+      end
+    end
+    command = command_list.join("\n")
+
+    serv.vm.sshable.cmd("cp /dev/stdin #{filename} && chmod +x #{filename}", stdin: command)
+  end
+
   def self.reindex_all_concurrently(resource_name, disable_indexes: false)
     serv = LanternResource[name: resource_name].representative_server
     all_dbs = serv.vm.sshable.cmd("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec postgresql psql -U postgres -P \"footer=off\" -c 'SELECT datname from pg_database' | tail -n +3 | grep -v 'template0' | grep -v 'template1'").strip.split("\n")