From 02abb4a3b9e145c27c3afcf34d601b761d6463aa Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Thu, 18 Jul 2024 22:47:20 +0400 Subject: [PATCH] add new mix operations --- migrate/20240711_lantern_doctor_queries.rb | 2 +- misc/misc_operations.rb | 31 ++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/migrate/20240711_lantern_doctor_queries.rb b/migrate/20240711_lantern_doctor_queries.rb index e13512876..1b12b470f 100644 --- a/migrate/20240711_lantern_doctor_queries.rb +++ b/migrate/20240711_lantern_doctor_queries.rb @@ -4,7 +4,7 @@ up do # check duplicate sources for embedding jobs run "INSERT INTO lantern_doctor_query (id, name, db_name, schedule, condition, fn_label, type, severity, response_type) - VALUES ('09f5de22-13fb-89b7-bf5e-75b26faef139', 'Whitespace tokens for embedding job', '*', '*/8 * * * *', 'unknown', 'check_embedding_source_whitespaces', 'system', 'error', 'rows')" + VALUES ('09f5de22-13fb-89b7-bf5e-75b26faef139', 'Whitespace tokens for embedding job', '*', '0 5 * * *', 'unknown', 'check_embedding_source_whitespaces', 'system', 'error', 'rows')" # Create semaphores for all lantern doctors to sync system queries run "INSERT INTO semaphore (id, strand_id, name) SELECT gen_random_uuid(), id, 'sync_system_queries' FROM strand s WHERE s.prog = 'Lantern::LanternDoctorNexus'" diff --git a/misc/misc_operations.rb b/misc/misc_operations.rb index e6a0fdbb7..c60634994 100644 --- a/misc/misc_operations.rb +++ b/misc/misc_operations.rb @@ -91,6 +91,37 @@ def self.query_on_db(vm, db_name, query) vm.sshable.cmd("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec -T postgresql psql -U postgres -t --csv #{db_name}", stdin: query) end + def self.run_script_daemonized(resource_name, filename, task_name) + serv = LanternResource[name: resource_name].representative_server + serv.vm.sshable.cmd("common/bin/daemonizer 'sudo #{filename}' #{task_name}") + + task_logs resource_name, "#{task_name}" + end + + def self.create_all_indexes_concurrently_script(resource_name, filename) + serv = LanternResource[name: resource_name].representative_server + all_dbs = serv.vm.sshable.cmd("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec postgresql psql -U postgres -P \"footer=off\" -c 'SELECT datname from pg_database' | tail -n +3 | grep -v 'template0' | grep -v 'template1'").strip.split("\n") + command_list = [] + all_dbs.each do |db| + db_name = db.strip + indexes = MiscOperations.query_on_db(serv.vm, db_name, "SELECT indexdef FROM pg_indexes WHERE indexdef ILIKE '%lantern_hnsw%';").strip.split("\n") + + if !indexes.empty? + queries = [] + indexes.each { + queries.push(_1.gsub(/create\s+index/i, 'CREATE INDEX CONCURRENTLY')[1..-2]) + } + + queries.map do |query| + command_list.push("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec -T postgresql psql -U postgres #{db_name} -c \"#{query}\"") + end + end + end + command = command_list.join("\n") + + serv.vm.sshable.cmd("cp /dev/stdin #{filename} && chmod +x #{filename}", stdin: command) + end + def self.reindex_all_concurrently(resource_name, disable_indexes: false) serv = LanternResource[name: resource_name].representative_server all_dbs = serv.vm.sshable.cmd("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec postgresql psql -U postgres -P \"footer=off\" -c 'SELECT datname from pg_database' | tail -n +3 | grep -v 'template0' | grep -v 'template1'").strip.split("\n")