add more misc ops, force fetch timeline before generating walg creds

loader.rb

@@ -77,7 +77,7 @@
 end
 
 autoload_normal.call("model", flat: true)
-%w[lib clover.rb clover_web.rb clover_api.rb routes/clover_base.rb routes/clover_error.rb misc/misc_queries.rb misc/misc_operations.rb].each { autoload_normal.call(_1) }
+%w[lib clover.rb clover_web.rb clover_api.rb routes/clover_base.rb routes/clover_error.rb misc/misc_operations.rb].each { autoload_normal.call(_1) }
 %w[scheduling prog serializers serializers/web serializers/api].each { autoload_normal.call(_1, include_first: true) }
 
 AUTOLOAD_CONSTANTS.freeze

misc/misc_operations.rb

@@ -31,6 +31,26 @@ def update_rhizome(sshable, target_folder, user)
 end
 
 class MiscOperations
+  def self.update_collation_on_all_databases
+    # https://postgresql.verite.pro/blog/2018/08/27/glibc-upgrade.html
+    # We didn't need to update rebuild indexes this time as we didn't have any indexes with collation coming from libc
+    resources = LanternResource.all
+    resources.each do |resource|
+      update_collation resource
+    end
+  end
+
+  def self.update_collation(resource)
+    all_dbs = resource.representative_server.run_query("SELECT datname from pg_database WHERE datname != 'template0'").split("\n")
+    all_dbs.each do |db|
+      resource.representative_server.run_query("ALTER DATABASE #{db} REFRESH COLLATION VERSION")
+    end
+  end
+
+  def self.active_queries(resource_name)
+    LanternResource[name: resource_name].representative_server.run_query("SELECT pid, query FROM pg_stat_activity")
+  end
+
   def self.update_rhizome_from_local(vm, target_folder: "lantern", user: "lantern")
     update_rhizome(vm.sshable, target_folder, user)
   end
@@ -54,4 +74,62 @@ def self.kill_query(resource_name, pid)
   def self.task_logs(resource_name, task_name)
     LanternResource[name: resource_name].representative_server.vm.sshable.cmd("common/bin/daemonizer --logs #{task_name}")
   end
+
+  def self.query_on_db(vm, db_name, query)
+    vm.sshable.cmd("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec -T postgresql psql -U postgres -t --csv #{db_name}", stdin: query)
+  end
+
+  def self.reindex_all_concurrently(resource_name, disable_indexes: false)
+    serv = LanternResource[name: resource_name].representative_server
+    all_dbs = serv.vm.sshable.cmd("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec postgresql psql -U postgres -P \"footer=off\" -c 'SELECT datname from pg_database' | tail -n +3 | grep -v 'template0' | grep -v 'template1'").strip.split("\n")
+    command_list = []
+    all_dbs.each do |db|
+      db_name = db.strip
+      indexes = MiscOperations.query_on_db(serv.vm, db_name, "SELECT i.relname::text
+                              FROM pg_class t
+                              JOIN pg_index ix ON t.oid = ix.indrelid
+                              JOIN pg_class i ON i.oid = ix.indexrelid
+                              JOIN pg_am a ON i.relam = a.oid
+                              JOIN pg_namespace n ON n.oid = i.relnamespace
+                              WHERE a.amname = 'lantern_hnsw';").strip.split("\n")
+
+      if !indexes.empty?
+        queries = []
+        indexes.each {
+          queries.push("REINDEX INDEX CONCURRENTLY #{_1};")
+          if disable_indexes
+            queries.push("UPDATE pg_index SET indisvalid = false, indisready = false WHERE indexrelid = quote_ident('#{_1}')::regclass::oid;")
+          end
+        }
+
+        queries.map do |query|
+          command_list.push("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec -T postgresql psql -U postgres #{db_name} -c \"#{query}\"")
+        end
+      end
+    end
+    command = command_list.join("\n")
+
+    serv.vm.sshable.cmd("cp /dev/stdin /tmp/reindex_all_concurrently.sh && chmod +x /tmp/reindex_all_concurrently.sh", stdin: command)
+    serv.vm.sshable.cmd("common/bin/daemonizer 'sudo /tmp/reindex_all_concurrently.sh' reindex_all_concurrently")
+
+    task_logs resource_name, "reindex_all_concurrently"
+  end
+
+  def self.get_all_lantern_indexes(resource_name)
+    serv = LanternResource[name: resource_name].representative_server
+    all_dbs = serv.vm.sshable.cmd("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec postgresql psql -U postgres -P \"footer=off\" -c 'SELECT datname from pg_database' | tail -n +3 | grep -v 'template0' | grep -v 'template1'").strip.split("\n")
+    all_indexes = []
+    all_dbs.each do |db|
+      db_name = db.strip
+      indexes = MiscOperations.query_on_db(serv.vm, db_name, "SELECT i.relname::text
+                              FROM pg_class t
+                              JOIN pg_index ix ON t.oid = ix.indrelid
+                              JOIN pg_class i ON i.oid = ix.indexrelid
+                              JOIN pg_am a ON i.relam = a.oid
+                              JOIN pg_namespace n ON n.oid = i.relnamespace
+                              WHERE a.amname = 'lantern_hnsw';").strip.split("\n")
+      all_indexes += indexes
+    end
+    all_indexes
+  end
 end

model/lantern/lantern_server.rb

@@ -129,7 +129,9 @@ def configure_hash
   end
 
   def update_walg_creds
-    walg_config = timeline.generate_walg_config
+    # fetching again to be sure it is the updated data
+    lantern_timeline = LanternTimeline[timeline.id]
+    walg_config = lantern_timeline.generate_walg_config
     vm.sshable.cmd("sudo lantern/bin/update_env", stdin: JSON.generate([
       ["WALG_GS_PREFIX", walg_config[:walg_gs_prefix]],
       ["GOOGLE_APPLICATION_CREDENTIALS_WALG_B64", walg_config[:gcp_creds_b64]],

rhizome/lantern/lib/common.rb

@@ -18,9 +18,18 @@ def configure_gcr(gcp_creds_gcr_b64, container_image)
 
 def update_extensions_in_sql
   all_dbs = (r "docker compose -f #{$compose_file} exec postgresql psql -U postgres -P \"footer=off\" -c 'SELECT datname from pg_database' | tail -n +3 | grep -v 'template0' | grep -v 'template1'").strip!.split("\n")
+  tmp_cmd = ""
   all_dbs.each do |db|
-    $stdout.puts r "docker compose -f #{$compose_file} exec postgresql psql -U postgres -f /lantern-init.sql #{db}"
+    tmp_cmd += "docker compose -f #{$compose_file} exec postgresql psql -U postgres -f /lantern-init.sql #{db} \n"
   end
+
+  # There was a strange issue with open3
+  # If for example the database count would be >300 it might fail without error
+  # But if we put all the commands in one bash file and execute that file it would succeed
+
+  File.write("/tmp/update_extensions_in_sql.sh", tmp_cmd)
+  $stdout.puts r "bash /tmp/update_extensions_in_sql.sh"
+  $stdout.puts r "rm -rf /tmp/update_extensions_in_sql.sh"
 end
 
 def wait_for_pg

spec/model/lantern/lantern_server_spec.rb

@@ -448,7 +448,9 @@
     it "calls update_env on vm" do
       timeline = instance_double(LanternTimeline)
       expect(timeline).to receive(:generate_walg_config).and_return({gcp_creds_b64: "test-creds-push", walg_gs_prefix: "test-bucket-push"}).at_least(:once)
+      expect(timeline).to receive(:id).and_return("test").at_least(:once)
       expect(lantern_server).to receive(:timeline).and_return(timeline).at_least(:once)
+      expect(LanternTimeline).to receive(:[]).and_return(timeline).at_least(:once)
       walg_config = timeline.generate_walg_config
       expect(vm.sshable).to receive(:cmd).with("sudo lantern/bin/update_env", stdin: JSON.generate([
         ["WALG_GS_PREFIX", walg_config[:walg_gs_prefix]],

spec/routes/api/project/lantern_spec.rb

@@ -114,7 +114,7 @@
       end
 
       it "creates new lantern database from backup with wrong restore time" do
-        post "/api/project/#{project.ubid}/lantern", {size: "n1-standard-2", name: "instance-from-backup", org_id: 0, location: "us-central1", domain: "test.db.lantern.dev", parent_id: pg.id, restore_target: "Test" }
+        post "/api/project/#{project.ubid}/lantern", {size: "n1-standard-2", name: "instance-from-backup", org_id: 0, location: "us-central1", domain: "test.db.lantern.dev", parent_id: pg.id, restore_target: "Test"}
 
         expect(last_response.status).to eq(400)
       end