From f95ff616684a6ec6cc03380b6008e59fb087dcca Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Sat, 4 May 2024 20:59:57 -0700 Subject: [PATCH] add more misc ops, force fetch timeline before generating walg creds --- loader.rb | 2 +- misc/misc_operations.rb | 78 +++++++++++++++++++++++ misc/misc_queries.rb | 23 ------- model/lantern/lantern_server.rb | 4 +- rhizome/lantern/lib/common.rb | 11 +++- spec/model/lantern/lantern_server_spec.rb | 2 + spec/routes/api/project/lantern_spec.rb | 2 +- 7 files changed, 95 insertions(+), 27 deletions(-) delete mode 100644 misc/misc_queries.rb diff --git a/loader.rb b/loader.rb index f35796009..9faa60ea2 100644 --- a/loader.rb +++ b/loader.rb @@ -77,7 +77,7 @@ end autoload_normal.call("model", flat: true) -%w[lib clover.rb clover_web.rb clover_api.rb routes/clover_base.rb routes/clover_error.rb misc/misc_queries.rb misc/misc_operations.rb].each { autoload_normal.call(_1) } +%w[lib clover.rb clover_web.rb clover_api.rb routes/clover_base.rb routes/clover_error.rb misc/misc_operations.rb].each { autoload_normal.call(_1) } %w[scheduling prog serializers serializers/web serializers/api].each { autoload_normal.call(_1, include_first: true) } AUTOLOAD_CONSTANTS.freeze diff --git a/misc/misc_operations.rb b/misc/misc_operations.rb index 33f82cf09..b3f656927 100644 --- a/misc/misc_operations.rb +++ b/misc/misc_operations.rb @@ -31,6 +31,26 @@ def update_rhizome(sshable, target_folder, user) end class MiscOperations + def self.update_collation_on_all_databases + # https://postgresql.verite.pro/blog/2018/08/27/glibc-upgrade.html + # We didn't need to update rebuild indexes this time as we didn't have any indexes with collation coming from libc + resources = LanternResource.all + resources.each do |resource| + update_collation resource + end + end + + def self.update_collation(resource) + all_dbs = resource.representative_server.run_query("SELECT datname from pg_database WHERE datname != 'template0'").split("\n") + all_dbs.each do |db| + resource.representative_server.run_query("ALTER DATABASE #{db} REFRESH COLLATION VERSION") + end + end + + def self.active_queries(resource_name) + LanternResource[name: resource_name].representative_server.run_query("SELECT pid, query FROM pg_stat_activity") + end + def self.update_rhizome_from_local(vm, target_folder: "lantern", user: "lantern") update_rhizome(vm.sshable, target_folder, user) end @@ -54,4 +74,62 @@ def self.kill_query(resource_name, pid) def self.task_logs(resource_name, task_name) LanternResource[name: resource_name].representative_server.vm.sshable.cmd("common/bin/daemonizer --logs #{task_name}") end + + def self.query_on_db(vm, db_name, query) + vm.sshable.cmd("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec -T postgresql psql -U postgres -t --csv #{db_name}", stdin: query) + end + + def self.reindex_all_concurrently(resource_name, disable_indexes: false) + serv = LanternResource[name: resource_name].representative_server + all_dbs = serv.vm.sshable.cmd("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec postgresql psql -U postgres -P \"footer=off\" -c 'SELECT datname from pg_database' | tail -n +3 | grep -v 'template0' | grep -v 'template1'").strip.split("\n") + command_list = [] + all_dbs.each do |db| + db_name = db.strip + indexes = MiscOperations.query_on_db(serv.vm, db_name, "SELECT i.relname::text + FROM pg_class t + JOIN pg_index ix ON t.oid = ix.indrelid + JOIN pg_class i ON i.oid = ix.indexrelid + JOIN pg_am a ON i.relam = a.oid + JOIN pg_namespace n ON n.oid = i.relnamespace + WHERE a.amname = 'lantern_hnsw';").strip.split("\n") + + if !indexes.empty? + queries = [] + indexes.each { + queries.push("REINDEX INDEX CONCURRENTLY #{_1};") + if disable_indexes + queries.push("UPDATE pg_index SET indisvalid = false, indisready = false WHERE indexrelid = quote_ident('#{_1}')::regclass::oid;") + end + } + + queries.map do |query| + command_list.push("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec -T postgresql psql -U postgres #{db_name} -c \"#{query}\"") + end + end + end + command = command_list.join("\n") + + serv.vm.sshable.cmd("cp /dev/stdin /tmp/reindex_all_concurrently.sh && chmod +x /tmp/reindex_all_concurrently.sh", stdin: command) + serv.vm.sshable.cmd("common/bin/daemonizer 'sudo /tmp/reindex_all_concurrently.sh' reindex_all_concurrently") + + task_logs resource_name, "reindex_all_concurrently" + end + + def self.get_all_lantern_indexes(resource_name) + serv = LanternResource[name: resource_name].representative_server + all_dbs = serv.vm.sshable.cmd("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec postgresql psql -U postgres -P \"footer=off\" -c 'SELECT datname from pg_database' | tail -n +3 | grep -v 'template0' | grep -v 'template1'").strip.split("\n") + all_indexes = [] + all_dbs.each do |db| + db_name = db.strip + indexes = MiscOperations.query_on_db(serv.vm, db_name, "SELECT i.relname::text + FROM pg_class t + JOIN pg_index ix ON t.oid = ix.indrelid + JOIN pg_class i ON i.oid = ix.indexrelid + JOIN pg_am a ON i.relam = a.oid + JOIN pg_namespace n ON n.oid = i.relnamespace + WHERE a.amname = 'lantern_hnsw';").strip.split("\n") + all_indexes += indexes + end + all_indexes + end end diff --git a/misc/misc_queries.rb b/misc/misc_queries.rb deleted file mode 100644 index b47041fb2..000000000 --- a/misc/misc_queries.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -class MiscQueries - def self.update_collation_on_all_databases - # https://postgresql.verite.pro/blog/2018/08/27/glibc-upgrade.html - # We didn't need to update rebuild indexes this time as we didn't have any indexes with collation coming from libc - resources = LanternResource.all - resources.each do |resource| - update_collation resource - end - end - - def self.update_collation(resource) - all_dbs = resource.representative_server.run_query("SELECT datname from pg_database WHERE datname != 'template0'").split("\n") - all_dbs.each do |db| - resource.representative_server.run_query("ALTER DATABASE #{db} REFRESH COLLATION VERSION") - end - end - - def self.active_queries(resource_name) - LanternResource[name: resource_name].representative_server.run_query("SELECT pid, query FROM pg_stat_activity") - end -end diff --git a/model/lantern/lantern_server.rb b/model/lantern/lantern_server.rb index b86d7e3f7..0bf3f50ee 100644 --- a/model/lantern/lantern_server.rb +++ b/model/lantern/lantern_server.rb @@ -129,7 +129,9 @@ def configure_hash end def update_walg_creds - walg_config = timeline.generate_walg_config + # fetching again to be sure it is the updated data + lantern_timeline = LanternTimeline[timeline.id] + walg_config = lantern_timeline.generate_walg_config vm.sshable.cmd("sudo lantern/bin/update_env", stdin: JSON.generate([ ["WALG_GS_PREFIX", walg_config[:walg_gs_prefix]], ["GOOGLE_APPLICATION_CREDENTIALS_WALG_B64", walg_config[:gcp_creds_b64]], diff --git a/rhizome/lantern/lib/common.rb b/rhizome/lantern/lib/common.rb index 964fa3403..bdaf869fb 100755 --- a/rhizome/lantern/lib/common.rb +++ b/rhizome/lantern/lib/common.rb @@ -18,9 +18,18 @@ def configure_gcr(gcp_creds_gcr_b64, container_image) def update_extensions_in_sql all_dbs = (r "docker compose -f #{$compose_file} exec postgresql psql -U postgres -P \"footer=off\" -c 'SELECT datname from pg_database' | tail -n +3 | grep -v 'template0' | grep -v 'template1'").strip!.split("\n") + tmp_cmd = "" all_dbs.each do |db| - $stdout.puts r "docker compose -f #{$compose_file} exec postgresql psql -U postgres -f /lantern-init.sql #{db}" + tmp_cmd += "docker compose -f #{$compose_file} exec postgresql psql -U postgres -f /lantern-init.sql #{db} \n" end + + # There was a strange issue with open3 + # If for example the database count would be >300 it might fail without error + # But if we put all the commands in one bash file and execute that file it would succeed + + File.write("/tmp/update_extensions_in_sql.sh", tmp_cmd) + $stdout.puts r "bash /tmp/update_extensions_in_sql.sh" + $stdout.puts r "rm -rf /tmp/update_extensions_in_sql.sh" end def wait_for_pg diff --git a/spec/model/lantern/lantern_server_spec.rb b/spec/model/lantern/lantern_server_spec.rb index e91dd9833..8943e7ef8 100644 --- a/spec/model/lantern/lantern_server_spec.rb +++ b/spec/model/lantern/lantern_server_spec.rb @@ -448,7 +448,9 @@ it "calls update_env on vm" do timeline = instance_double(LanternTimeline) expect(timeline).to receive(:generate_walg_config).and_return({gcp_creds_b64: "test-creds-push", walg_gs_prefix: "test-bucket-push"}).at_least(:once) + expect(timeline).to receive(:id).and_return("test").at_least(:once) expect(lantern_server).to receive(:timeline).and_return(timeline).at_least(:once) + expect(LanternTimeline).to receive(:[]).and_return(timeline).at_least(:once) walg_config = timeline.generate_walg_config expect(vm.sshable).to receive(:cmd).with("sudo lantern/bin/update_env", stdin: JSON.generate([ ["WALG_GS_PREFIX", walg_config[:walg_gs_prefix]], diff --git a/spec/routes/api/project/lantern_spec.rb b/spec/routes/api/project/lantern_spec.rb index c31693f3e..b06b1b604 100644 --- a/spec/routes/api/project/lantern_spec.rb +++ b/spec/routes/api/project/lantern_spec.rb @@ -114,7 +114,7 @@ end it "creates new lantern database from backup with wrong restore time" do - post "/api/project/#{project.ubid}/lantern", {size: "n1-standard-2", name: "instance-from-backup", org_id: 0, location: "us-central1", domain: "test.db.lantern.dev", parent_id: pg.id, restore_target: "Test" } + post "/api/project/#{project.ubid}/lantern", {size: "n1-standard-2", name: "instance-from-backup", org_id: 0, location: "us-central1", domain: "test.db.lantern.dev", parent_id: pg.id, restore_target: "Test"} expect(last_response.status).to eq(400) end