From fa00159ddeea0c4af66e1709b61342e857dd8312 Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Sun, 5 May 2024 11:34:00 -0700 Subject: [PATCH 01/13] change lantern timeline waiting condition in lantern_server_nexus --- prog/lantern/lantern_server_nexus.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prog/lantern/lantern_server_nexus.rb b/prog/lantern/lantern_server_nexus.rb index fc88c9f65..e220f6718 100644 --- a/prog/lantern/lantern_server_nexus.rb +++ b/prog/lantern/lantern_server_nexus.rb @@ -124,7 +124,7 @@ def before_run end # wait for service account to be created - nap 10 if lantern_server.timeline.strand.label != "wait_leader" + nap 10 if lantern_server.timeline.strand.label == "start" case vm.sshable.cmd("common/bin/daemonizer --check configure_lantern") when "Succeeded" @@ -212,7 +212,7 @@ def before_run end label def wait_timeline_available - nap 10 if lantern_server.timeline.strand.label != "wait_leader" + nap 10 if lantern_server.timeline.strand.label == "start" lantern_server.update_walg_creds decr_initial_provisioning hop_wait_db_available From cf27213285b00a900246d6d859eab9fe31bc3088 Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Sun, 5 May 2024 12:16:31 -0700 Subject: [PATCH 02/13] Add labels to lantern_resources (#21) * add labels to lantern resources * add labels to lantern resource in api as well --- migrate/20240505_lantern_resource_label.rb | 9 +++++++++ prog/lantern/lantern_resource_nexus.rb | 6 ++++-- routes/api/project/lantern.rb | 1 + routes/web/project/lantern.rb | 1 + serializers/api/lantern.rb | 1 + serializers/web/lantern.rb | 1 + spec/routes/api/project/lantern_spec.rb | 3 ++- spec/routes/web/project/lantern_spec.rb | 23 +++++++++++++++++++++- views/lantern/create.erb | 13 ++++++++++++ views/lantern/index.erb | 2 +- views/lantern/show.erb | 1 + 11 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 migrate/20240505_lantern_resource_label.rb diff --git a/migrate/20240505_lantern_resource_label.rb b/migrate/20240505_lantern_resource_label.rb new file mode 100644 index 000000000..bc63362bc --- /dev/null +++ b/migrate/20240505_lantern_resource_label.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +Sequel.migration do + change do + alter_table(:lantern_resource) do + add_column :label, :text, null: true + end + end +end diff --git a/prog/lantern/lantern_resource_nexus.rb b/prog/lantern/lantern_resource_nexus.rb index 733384138..328356d4e 100644 --- a/prog/lantern/lantern_resource_nexus.rb +++ b/prog/lantern/lantern_resource_nexus.rb @@ -14,7 +14,8 @@ class Prog::Lantern::LanternResourceNexus < Prog::Base def self.assemble(project_id:, location:, name:, target_vm_size:, target_storage_size_gib:, ha_type: LanternResource::HaType::NONE, parent_id: nil, restore_target: nil, org_id: nil, db_name: "postgres", db_user: "postgres", db_user_password: nil, superuser_password: nil, repl_password: nil, app_env: Config.rack_env, - lantern_version: Config.lantern_default_version, extras_version: Config.lantern_extras_default_version, minor_version: Config.lantern_minor_default_version, domain: nil, enable_debug: false) + lantern_version: Config.lantern_default_version, extras_version: Config.lantern_extras_default_version, minor_version: Config.lantern_minor_default_version, domain: nil, enable_debug: false, + label: "") unless (project = Project[project_id]) fail "No existing project" end @@ -77,7 +78,8 @@ def self.assemble(project_id:, location:, name:, target_vm_size:, target_storage project_id: project_id, location: location, name: name, org_id: org_id, app_env: app_env, superuser_password: superuser_password, ha_type: ha_type, parent_id: parent_id, restore_target: restore_target, db_name: db_name, db_user: db_user, - db_user_password: db_user_password, repl_user: repl_user, repl_password: repl_password + db_user_password: db_user_password, repl_user: repl_user, repl_password: repl_password, + label: label ) { _1.id = ubid.to_uuid } lantern_resource.associate_with_project(project) diff --git a/routes/api/project/lantern.rb b/routes/api/project/lantern.rb index 451e81f60..cdeeed08c 100644 --- a/routes/api/project/lantern.rb +++ b/routes/api/project/lantern.rb @@ -43,6 +43,7 @@ class CloverApi project_id: @project.id, location: r.params["location"], name: r.params["name"], + label: r.params["label"], org_id: r.params["org_id"].to_i, target_vm_size: parsed_size.vm_size, target_storage_size_gib: r.params["storage_size_gib"] || parsed_size.storage_size_gib, diff --git a/routes/web/project/lantern.rb b/routes/web/project/lantern.rb index 83a0a6b08..f5622ec8f 100644 --- a/routes/web/project/lantern.rb +++ b/routes/web/project/lantern.rb @@ -23,6 +23,7 @@ class CloverWeb project_id: @project.id, location: r.params["location"], name: r.params["name"], + label: r.params["label"], org_id: r.params["org_id"], target_vm_size: parsed_size.vm_size, target_storage_size_gib: parsed_size.storage_size_gib, diff --git a/serializers/api/lantern.rb b/serializers/api/lantern.rb index c3afa9501..60cd8d24b 100644 --- a/serializers/api/lantern.rb +++ b/serializers/api/lantern.rb @@ -8,6 +8,7 @@ def self.base(pg) parent_id: pg.parent_id, path: pg.path, name: pg.name, + label: pg.label, state: pg.display_state, instance_type: pg.representative_server&.instance_type, location: pg.location, diff --git a/serializers/web/lantern.rb b/serializers/web/lantern.rb index 533552e01..23655aad0 100644 --- a/serializers/web/lantern.rb +++ b/serializers/web/lantern.rb @@ -7,6 +7,7 @@ def self.base(pg) ubid: pg.ubid, path: pg.path, name: pg.name, + label: pg.label, state: pg.display_state, primary?: pg.representative_server&.primary?, instance_type: pg.representative_server&.instance_type, diff --git a/spec/routes/api/project/lantern_spec.rb b/spec/routes/api/project/lantern_spec.rb index b06b1b604..a40b9c71f 100644 --- a/spec/routes/api/project/lantern_spec.rb +++ b/spec/routes/api/project/lantern_spec.rb @@ -59,12 +59,13 @@ describe "create" do it "creates new lantern database" do - post "/api/project/#{project.ubid}/lantern", {size: "n1-standard-2", name: "instance-2", org_id: 0, location: "us-central1", storage_size_gib: 100, lantern_version: "0.2.2", extras_version: "0.1.4", minor_version: "1", domain: "test.db.lantern.dev", app_env: "test", repl_password: "test-repl-pass", enable_telemetry: true, postgres_password: "test-pg-pass"} + post "/api/project/#{project.ubid}/lantern", {size: "n1-standard-2", name: "instance-2", label: "test-label", org_id: 0, location: "us-central1", storage_size_gib: 100, lantern_version: "0.2.2", extras_version: "0.1.4", minor_version: "1", domain: "test.db.lantern.dev", app_env: "test", repl_password: "test-repl-pass", enable_telemetry: true, postgres_password: "test-pg-pass"} body = JSON.parse(last_response.body) expect(last_response.status).to eq(200) expect(body["name"]).to eq("instance-2") + expect(body["label"]).to eq("test-label") expect(body["state"]).to eq("creating") expect(body["instance_type"]).to eq("writer") expect(body["location"]).to eq("us-central1") diff --git a/spec/routes/web/project/lantern_spec.rb b/spec/routes/web/project/lantern_spec.rb index 3130e058e..fd6411e65 100644 --- a/spec/routes/web/project/lantern_spec.rb +++ b/spec/routes/web/project/lantern_spec.rb @@ -69,6 +69,7 @@ location: "us-central1", name: "instance-2", target_vm_size: "n1-standard-2", + label: "test", target_storage_size_gib: 100, org_id: 0 ) @@ -76,7 +77,25 @@ visit "#{project.path}/lantern" expect(page.title).to eq("Ubicloud - Lantern Databases") - expect(page).to have_content "instance-2" + expect(page).to have_content "instance-2 (test)" + end + + it "show no-label if no label was specified" do + Prog::Lantern::LanternResourceNexus.assemble( + project_id: project.id, + location: "us-central1", + name: "instance-4", + target_vm_size: "n1-standard-2", + target_storage_size_gib: 100, + org_id: 0 + ) + + LanternResource[name: "instance-4"].update(label: nil) + + visit "#{project.path}/lantern" + + expect(page.title).to eq("Ubicloud - Lantern Databases") + expect(page).to have_content "instance-4 (no-label)" end end @@ -87,6 +106,7 @@ expect(page.title).to eq("Ubicloud - Create Lantern Database") name = "new-pg-db" fill_in "Name", with: name + fill_in "Label", with: "test-label" choose option: "us-central1" choose option: "n1-standard-2" find_by_id("parent_id").find(:xpath, "option[1]").select_option @@ -97,6 +117,7 @@ expect(page).to have_content "'#{name}' will be ready in a few minutes" expect(LanternResource.count).to eq(1) expect(LanternResource.first.projects.first.id).to eq(project.id) + expect(LanternResource.first.label).to eq("test-label") end it "can create new Lantern database with domain" do diff --git a/views/lantern/create.erb b/views/lantern/create.erb index 42517c80d..36c44aae2 100644 --- a/views/lantern/create.erb +++ b/views/lantern/create.erb @@ -42,6 +42,19 @@ } ) %> +
+ <%== render( + "components/form/text", + locals: { + name: "label", + label: "Label", + attributes: { + required: false, + placeholder: "Enter label" + } + } + ) %> +
<%== render( "components/form/text", diff --git a/views/lantern/index.erb b/views/lantern/index.erb index 1b5bcaa0e..bc4297037 100644 --- a/views/lantern/index.erb +++ b/views/lantern/index.erb @@ -37,7 +37,7 @@ <% @lantern_databases.each do |pg| %> - <%= pg[:name] %> + <%= pg[:name] %> (<%= (!pg[:label] || pg[:label].empty?) ? "no-label" : pg[:label] %>) <%= pg[:location] %> <%== render("components/pg_state_label", locals: { state: pg[:state] }) %> diff --git a/views/lantern/show.erb b/views/lantern/show.erb index 533df3a57..ebe1a1168 100644 --- a/views/lantern/show.erb +++ b/views/lantern/show.erb @@ -26,6 +26,7 @@ <% data = [ ["ID", @pg[:ubid]], ["Name", @pg[:name]], + ["Label", @pg[:label]], ["Org Id", @pg[:org_id]], ["Instance Type", @pg[:instance_type]], ["Lantern Version", @pg[:lantern_version]], From 2067579f683295e246d45cd035d7f19d67b0819d Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Sun, 5 May 2024 22:11:48 -0700 Subject: [PATCH 03/13] add lantern doctor and test query --- Gemfile | 1 + Gemfile.lock | 2 + config.rb | 1 + db.rb | 8 + migrate/20240505_lantern_doctor.rb | 34 ++ migrate/20240506_lantern_doctor_queries.rb | 8 + model/lantern/lantern_doctor.rb | 49 +++ model/lantern/lantern_doctor_query.rb | 135 +++++++ model/lantern/lantern_server.rb | 10 +- prog/lantern/lantern_doctor_nexus.rb | 47 +++ prog/lantern/lantern_resource_nexus.rb | 4 +- .../lantern/lantern_doctor_query_spec.rb | 343 ++++++++++++++++++ spec/model/lantern/lantern_doctor_spec.rb | 56 +++ spec/model/lantern/lantern_server_spec.rb | 25 +- .../prog/lantern/lantern_doctor_nexus_spec.rb | 79 ++++ ubid.rb | 2 + views/lantern/create.erb | 1 - 17 files changed, 797 insertions(+), 8 deletions(-) create mode 100644 migrate/20240505_lantern_doctor.rb create mode 100644 migrate/20240506_lantern_doctor_queries.rb create mode 100644 model/lantern/lantern_doctor.rb create mode 100644 model/lantern/lantern_doctor_query.rb create mode 100644 prog/lantern/lantern_doctor_nexus.rb create mode 100644 spec/model/lantern/lantern_doctor_query_spec.rb create mode 100644 spec/model/lantern/lantern_doctor_spec.rb create mode 100644 spec/prog/lantern/lantern_doctor_nexus_spec.rb diff --git a/Gemfile b/Gemfile index 671940838..9e8f5ee7d 100644 --- a/Gemfile +++ b/Gemfile @@ -33,6 +33,7 @@ gem "octokit" gem "argon2-kdf" gem "googleauth" gem "simplecov" +gem "parse-cron" group :development do gem "brakeman" diff --git a/Gemfile.lock b/Gemfile.lock index 189f12b94..7ce44d80c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -157,6 +157,7 @@ GEM parallel (1.24.0) parallel_tests (4.5.2) parallel + parse-cron (0.1.4) parser (3.3.0.5) ast (~> 2.4.1) racc @@ -346,6 +347,7 @@ DEPENDENCIES nokogiri octokit pagerduty (>= 4.0) + parse-cron pry pry-byebug puma (>= 6.2.2) diff --git a/config.rb b/config.rb index fbad952ce..bd9080e57 100644 --- a/config.rb +++ b/config.rb @@ -142,6 +142,7 @@ def self.e2e_test? override :lantern_backup_bucket, "walg-dev-backups" override :e2e_test, "0" override :backup_retention_days, 7, int + optional :lantern_backend_database_url, string # GCP override :gcp_project_id, "lantern-development", string diff --git a/db.rb b/db.rb index cf731c5ba..88230811b 100644 --- a/db.rb +++ b/db.rb @@ -26,3 +26,11 @@ # DB.extension :date_arithmetic DB.extension :pg_json, :pg_auto_parameterize, :pg_timestamptz, :pg_range, :pg_array Sequel.extension :pg_range_ops + +module LanternBackend + @@db = Sequel.connect(Config.lantern_backend_database_url, max_connections: Config.db_pool, pool_timeout: Config.database_timeout) if Config.lantern_backend_database_url + + def self.db + @@db + end +end diff --git a/migrate/20240505_lantern_doctor.rb b/migrate/20240505_lantern_doctor.rb new file mode 100644 index 000000000..8cc7539d5 --- /dev/null +++ b/migrate/20240505_lantern_doctor.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +Sequel.migration do + change do + # doctor + create_table(:lantern_doctor) do + column :id, :uuid, primary_key: true, default: nil + column :created_at, :timestamptz, null: false, default: Sequel.lit("now()") + column :updated_at, :timestamptz, null: false, default: Sequel.lit("now()") + end + alter_table(:lantern_resource) do + add_foreign_key :doctor_id, :lantern_doctor, type: :uuid, null: true + end + # queries + create_enum(:query_condition, %w[unknown healthy failed]) + create_enum(:query_type, %w[system udf]) + create_table(:lantern_doctor_query) do + column :id, :uuid, primary_key: true, default: nil + foreign_key :parent_id, :lantern_doctor_query, type: :uuid + foreign_key :doctor_id, :lantern_doctor, type: :uuid + column :name, :text, null: true + column :db_name, :text, null: true + column :schedule, :text, null: true + column :condition, :query_condition, null: false, default: "unknown" + column :fn_label, :text, null: true + column :sql, :text, null: true + column :type, :query_type, null: false + column :severity, :severity, default: "error", null: true + column :last_checked, :timestamptz, null: true + column :created_at, :timestamptz, null: false, default: Sequel.lit("now()") + column :updated_at, :timestamptz, null: false, default: Sequel.lit("now()") + end + end +end diff --git a/migrate/20240506_lantern_doctor_queries.rb b/migrate/20240506_lantern_doctor_queries.rb new file mode 100644 index 000000000..df48ed2a7 --- /dev/null +++ b/migrate/20240506_lantern_doctor_queries.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +Sequel.migration do + change do + run "INSERT INTO lantern_doctor_query (id, name, db_name, schedule, condition, fn_label, type, severity) + VALUES ('4f916f44-3c7a-89b7-9795-1ccd417b45ba', 'Check Daemon Embedding Job', '*', '*/5 * * * *', 'unknown', 'check_daemon_embedding_jobs', 'system', 'error')" + end +end diff --git a/model/lantern/lantern_doctor.rb b/model/lantern/lantern_doctor.rb new file mode 100644 index 000000000..73f7c461e --- /dev/null +++ b/model/lantern/lantern_doctor.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require_relative "../../model" + +class LanternDoctor < Sequel::Model + one_to_one :strand, key: :id + one_to_one :resource, class: LanternResource, key: :doctor_id + one_to_many :queries, key: :doctor_id, class: LanternDoctorQuery + + plugin :association_dependencies, queries: :destroy + + include ResourceMethods + include SemaphoreMethods + + semaphore :destroy + + def system_queries + @system_queries ||= LanternDoctorQuery.where(type: "system") + end + + def has_system_query?(queries, query) + queries.any? { _1.parent_id == query.id } + end + + def list_queries + doctor_query_list = queries + system_query_list = system_queries + merged_queries = doctor_query_list + + system_query_list.each { + if !has_system_query?(doctor_query_list, _1) + doctor_query = LanternDoctorQuery.create_with_id(parent_id: _1.id, condition: "unknown", type: "system") + doctor_query.parent = _1 + merged_queries.push(doctor_query) + end + } + + puts "Merged queries are ->>>>. #{merged_queries}" + merged_queries + end + + def list_incidents + # TODO:: + # Find all unhealthy queries + # Get all pages for that queries + # Map results + [] + end +end diff --git a/model/lantern/lantern_doctor_query.rb b/model/lantern/lantern_doctor_query.rb new file mode 100644 index 000000000..06bbe5b27 --- /dev/null +++ b/model/lantern/lantern_doctor_query.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +require "parse-cron" +require_relative "../../model" +require_relative "../../db" + +class LanternDoctorQuery < Sequel::Model + one_to_one :strand, key: :id + one_to_one :doctor, key: :id, primary_key: :doctor_id + one_to_one :parent, class: self, key: :parent_id + one_to_many :children, key: :parent_id, class: self + + plugin :association_dependencies, children: :destroy + + include ResourceMethods + include SemaphoreMethods + + semaphore :destroy + + def sql + parent&.sql || super + end + + def name + parent&.name || super + end + + def db_name + parent&.db_name || super + end + + def severity + parent&.severity || super + end + + def schedule + parent&.schedule || super + end + + def fn_label + parent&.fn_label || super + end + + def should_run? + puts "Parent ->>>>> #{parent}" + puts "Schedule is ->>>>>>> #{schedule}" + CronParser.new(schedule).next(last_checked || Time.new - 61) <= Time.new + end + + def is_system? + !parent.nil? + end + + def user + return "postgres" if is_system? + doctor.resource.db_user + end + + def run + if !should_run? + return nil + end + + puts "RUNNING QUERY::::::::::::::::: #{name}" + + lantern_server = doctor.resource.representative_server + dbs = (db_name == "*") ? lantern_server.list_all_databases : [db_name] + query_user = user + + any_failed = false + dbs.each do |db| + err_msg = "" + + failed = false + begin + if is_system? && fn_label && LanternDoctorQuery.method_defined?(fn_label) + res = send(fn_label, db, query_user) + elsif sql + res = lantern_server.run_query(sql, db: db, user: query_user).strip + else + fail "BUG: non-system query without sql" + end + + if res != "f" + failed = true + any_failed = true + end + rescue => e + failed = true + any_failed = true + Clog.emit("LanternDoctorQuery failed") { {error: e, query_name: name, db: db, resource_name: doctor.resource.name} } + err_msg = e.message + end + + pg = Page.from_tag_parts("LanternDoctorQueryFailed", id, db) + + if failed && !pg + Prog::PageNexus.assemble_with_logs("Query #{name} failed on #{doctor.resource.name} (#{db})", [ubid, doctor.ubid, lantern_server.ubid], {"stderr" => err_msg}, severity, "LanternDoctorQueryFailed", id, db) + elsif !failed && pg + pg.incr_resolve + end + end + + update(condition: any_failed ? "failed" : "healthy", last_checked: Time.new) + end + + def check_daemon_embedding_jobs(db, query_user) + if !LanternBackend.db + fail "No connection to lantern backend database specified" + end + + puts "HERE FINDING JOBS" + jobs = LanternBackend.db + .select(:schema, :table, :src_column, :dst_column) + .from(:embedding_generation_jobs) + .where(database_id: doctor.resource.name) + .where(Sequel.like(:db_connection, "%/#{db}")) + # :nocov: + .where { !init_finished_at.nil? } + # :nocov: + .all + + if jobs.empty? + return "f" + end + + lantern_server = doctor.resource.representative_server + failed = jobs.any? do |job| + res = lantern_server.run_query("SELECT EXISTS(SELECT (SELECT COUNT(*) FROM \"#{job[:schema]}\".\"#{job[:table]}\" WHERE \"#{job[:src_column]}\" IS NOT NULL AND \"#{job[:src_column]}\" != '' AND \"#{job[:dst_column]}\" IS NULL) > 1000)", db: db, user: query_user).strip + res == "t" + end + + failed ? "t" : "f" + end +end diff --git a/model/lantern/lantern_server.rb b/model/lantern/lantern_server.rb index b86d7e3f7..56b1f5da1 100644 --- a/model/lantern/lantern_server.rb +++ b/model/lantern/lantern_server.rb @@ -47,12 +47,12 @@ def connection_string ).to_s end - def run_query(query) - vm.sshable.cmd("sudo lantern/bin/exec", stdin: query).chomp + def run_query(query, db: "postgres", user: "postgres") + vm.sshable.cmd("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec -T postgresql psql -U #{user} -t --csv #{db}", stdin: query).chomp end def run_query_all(query) - vm.sshable.cmd("sudo lantern/bin/exec_all", stdin: query).chomp + list_all_databases.map { [_1, run_query(query, db: _1)] } end def display_state @@ -196,6 +196,10 @@ def prewarm_indexes_query SQL end + def list_all_databases + vm.sshable.cmd("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec postgresql psql -U postgres -P \"footer=off\" -c 'SELECT datname from pg_database' | tail -n +3 | grep -v 'template0' | grep -v 'template1'").chomp.strip.split("\n") + end + # def failover_target # nil # end diff --git a/prog/lantern/lantern_doctor_nexus.rb b/prog/lantern/lantern_doctor_nexus.rb new file mode 100644 index 000000000..cbf00b0d9 --- /dev/null +++ b/prog/lantern/lantern_doctor_nexus.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require "forwardable" + +class Prog::Lantern::LanternDoctorNexus < Prog::Base + subject_is :lantern_doctor + + extend Forwardable + def_delegators :lantern_doctor + + semaphore :destroy + + def self.assemble + DB.transaction do + lantern_doctor = LanternDoctor.create_with_id + Strand.create(prog: "Lantern::LanternDoctorNexus", label: "start") { _1.id = lantern_doctor.id } + end + end + + def before_run + when_destroy_set? do + if strand.label != "destroy" + hop_destroy + end + end + end + + label def start + hop_wait_resource + end + + label def wait_resource + nap 5 if lantern_doctor.resource.strand.label != "wait" + hop_wait + end + + label def wait + lantern_doctor.list_queries.each { _1.run } + nap 10 + end + + label def destroy + decr_destroy + lantern_doctor.destroy + pop "lantern doctor is deleted" + end +end diff --git a/prog/lantern/lantern_resource_nexus.rb b/prog/lantern/lantern_resource_nexus.rb index 328356d4e..47635f034 100644 --- a/prog/lantern/lantern_resource_nexus.rb +++ b/prog/lantern/lantern_resource_nexus.rb @@ -74,12 +74,14 @@ def self.assemble(project_id:, location:, name:, target_vm_size:, target_storage end + lantern_doctor = Prog::Lantern::LanternDoctorNexus.assemble + lantern_resource = LanternResource.create( project_id: project_id, location: location, name: name, org_id: org_id, app_env: app_env, superuser_password: superuser_password, ha_type: ha_type, parent_id: parent_id, restore_target: restore_target, db_name: db_name, db_user: db_user, db_user_password: db_user_password, repl_user: repl_user, repl_password: repl_password, - label: label + label: label, doctor_id: lantern_doctor.id ) { _1.id = ubid.to_uuid } lantern_resource.associate_with_project(project) diff --git a/spec/model/lantern/lantern_doctor_query_spec.rb b/spec/model/lantern/lantern_doctor_query_spec.rb new file mode 100644 index 000000000..75047acf4 --- /dev/null +++ b/spec/model/lantern/lantern_doctor_query_spec.rb @@ -0,0 +1,343 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +RSpec.describe LanternDoctorQuery do + subject(:lantern_doctor_query) { + described_class.new do |r| + r.sql = "SELECT 1" + r.name = "test" + r.db_name = "postgres" + r.severity = "error" + r.schedule = "*/1 * * * *" + r.id = "6181ddb3-0002-8ad0-9aeb-084832c9273b" + end + } + + let(:parent) { + instance_double( + described_class, + sql: "SELECT 2", + name: "test-parent", + db_name: "*", + severity: "critical", + schedule: "*/2 * * * *" + ) + } + + describe "#parent_properties" do + it "returns parent sql if parent_id is defined else self sql" do + expect(lantern_doctor_query).to receive(:parent).and_return(nil) + expect(lantern_doctor_query.sql).to be("SELECT 1") + + expect(lantern_doctor_query).to receive(:parent).and_return(parent) + expect(lantern_doctor_query.sql).to be("SELECT 2") + end + + it "returns parent name if parent_id is defined else self name" do + expect(lantern_doctor_query).to receive(:parent).and_return(nil) + expect(lantern_doctor_query.name).to be("test") + + expect(lantern_doctor_query).to receive(:parent).and_return(parent) + expect(lantern_doctor_query.name).to be("test-parent") + end + + it "returns parent db_name if parent_id is defined else self db_name" do + expect(lantern_doctor_query).to receive(:parent).and_return(nil) + expect(lantern_doctor_query.db_name).to be("postgres") + + expect(lantern_doctor_query).to receive(:parent).and_return(parent) + expect(lantern_doctor_query.db_name).to be("*") + end + + it "returns parent severity if parent_id is defined else self severity" do + expect(lantern_doctor_query).to receive(:parent).and_return(nil) + expect(lantern_doctor_query.severity).to be("error") + + expect(lantern_doctor_query).to receive(:parent).and_return(parent) + expect(lantern_doctor_query.severity).to be("critical") + end + + it "returns parent schedule if parent_id is defined else self schedule" do + expect(lantern_doctor_query).to receive(:parent).and_return(nil) + expect(lantern_doctor_query.schedule).to be("*/1 * * * *") + + expect(lantern_doctor_query).to receive(:parent).and_return(parent) + expect(lantern_doctor_query.schedule).to be("*/2 * * * *") + end + + it "returns parent fn_label if parent_id is defined else self fn_label" do + expect(lantern_doctor_query).to receive(:parent).and_return(nil) + expect(lantern_doctor_query.fn_label).to be_nil + + expect(lantern_doctor_query).to receive(:parent).and_return(parent) + expect(parent).to receive(:fn_label).and_return("test") + expect(lantern_doctor_query.fn_label).to be("test") + end + end + + describe "#should_run?" do + it "return false if not yet time for run" do + min = Time.new.min + modified_min = (min == 59) ? 0 : min + 1 + + expect(lantern_doctor_query).to receive(:schedule).and_return("#{modified_min} * * * *").at_least(:once) + expect(lantern_doctor_query.should_run?).to be(false) + end + + it "return true if it is the same time for run" do + ts = Time.new + min = ts.min + expect(Time).to receive(:new).and_return(ts).at_least(:once) + expect(lantern_doctor_query).to receive(:last_checked).and_return(nil) + expect(lantern_doctor_query).to receive(:schedule).and_return("#{min} * * * *").at_least(:once) + expect(lantern_doctor_query.should_run?).to be(true) + end + + it "return true if the running time was passed but didn't run yet" do + min = Time.new.min + modified_min = (min == 0) ? 59 : min - 1 + + expect(lantern_doctor_query).to receive(:last_checked).and_return(Time.new - 60 * 5).at_least(:once) + expect(lantern_doctor_query).to receive(:schedule).and_return("#{modified_min} * * * *").at_least(:once) + expect(lantern_doctor_query.should_run?).to be(true) + end + end + + describe "#is_system?" do + it "returns false if no parent" do + expect(lantern_doctor_query).to receive(:parent).and_return(nil) + expect(lantern_doctor_query.is_system?).to be(false) + end + + it "returns true if has parent" do + expect(lantern_doctor_query).to receive(:parent).and_return(parent) + expect(lantern_doctor_query.is_system?).to be(true) + end + end + + describe "#user" do + it "returns postgres if system query" do + expect(lantern_doctor_query).to receive(:is_system?).and_return(true) + expect(lantern_doctor_query.user).to be("postgres") + end + + it "returns db user if not system query" do + expect(lantern_doctor_query).to receive(:is_system?).and_return(false) + expect(lantern_doctor_query).to receive(:doctor).and_return(instance_double(LanternDoctor, resource: instance_double(LanternResource, db_user: "test"))) + expect(lantern_doctor_query.user).to be("test") + end + end + + describe "#run" do + it "returns if should not run yet" do + expect(lantern_doctor_query).to receive(:should_run?).and_return(false) + expect(lantern_doctor_query.run).to be_nil + end + + it "runs query on specified database" do + serv = instance_double(LanternServer) + resource = instance_double(LanternResource, representative_server: serv, db_user: "test") + doctor = instance_double(LanternDoctor, resource: resource) + expect(serv).to receive(:run_query).with(lantern_doctor_query.sql, db: lantern_doctor_query.db_name, user: resource.db_user).and_return("f") + expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once) + expect(lantern_doctor_query).to receive(:should_run?).and_return(true) + expect(lantern_doctor_query).to receive(:update).with(hash_including(condition: "healthy")) + expect { lantern_doctor_query.run }.not_to raise_error + end + + it "throws error if no sql defined" do + serv = instance_double(LanternServer, ubid: "test-ubid") + resource = instance_double(LanternResource, representative_server: serv, db_user: "test", name: "test-res", id: "6181ddb3-0002-8ad0-9aeb-084832c9273b") + doctor = instance_double(LanternDoctor, resource: resource, ubid: "test-ubid") + + expect(lantern_doctor_query).to receive(:sql).and_return(nil).at_least(:once) + expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once) + expect(lantern_doctor_query).to receive(:should_run?).and_return(true) + expect(lantern_doctor_query).to receive(:update).with(hash_including(condition: "failed")) + expect(Prog::PageNexus).to receive(:assemble_with_logs).with("Query #{lantern_doctor_query.name} failed on #{doctor.resource.name} (postgres)", [lantern_doctor_query.ubid, doctor.ubid, serv.ubid], {"stderr" => "BUG: non-system query without sql"}, lantern_doctor_query.severity, "LanternDoctorQueryFailed", lantern_doctor_query.id, "postgres") + + expect { lantern_doctor_query.run }.not_to raise_error + end + + it "runs function for specified database" do + serv = instance_double(LanternServer) + resource = instance_double(LanternResource, representative_server: serv, db_user: "test") + doctor = instance_double(LanternDoctor, resource: resource) + + expect(parent).to receive(:db_name).and_return("postgres") + expect(lantern_doctor_query).to receive(:fn_label).and_return("check_daemon_embedding_jobs").at_least(:once) + expect(lantern_doctor_query).to receive(:parent).and_return(parent).at_least(:once) + + expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once) + expect(lantern_doctor_query).to receive(:is_system?).and_return(true).at_least(:once) + expect(lantern_doctor_query).to receive(:should_run?).and_return(true).at_least(:once) + expect(lantern_doctor_query).to receive(:check_daemon_embedding_jobs).and_return("f") + + expect(lantern_doctor_query).to receive(:update).with(hash_including(condition: "healthy")) + + expect { lantern_doctor_query.run }.not_to raise_error + end + + it "runs query on all databases" do + serv = instance_double(LanternServer) + resource = instance_double(LanternResource, representative_server: serv, db_user: "test") + doctor = instance_double(LanternDoctor, resource: resource) + dbs = ["db1", "db2"] + + expect(serv).to receive(:list_all_databases).and_return(dbs) + + expect(serv).to receive(:run_query).with(lantern_doctor_query.sql, db: "db1", user: resource.db_user).and_return("f") + expect(serv).to receive(:run_query).with(lantern_doctor_query.sql, db: "db2", user: resource.db_user).and_return("f") + + expect(lantern_doctor_query).to receive(:db_name).and_return("*") + expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once) + expect(lantern_doctor_query).to receive(:should_run?).and_return(true) + expect(lantern_doctor_query).to receive(:update).with(hash_including(condition: "healthy")) + + expect { lantern_doctor_query.run }.not_to raise_error + end + + it "runs query on all databases and errors" do + serv = instance_double(LanternServer, ubid: "test-ubid") + resource = instance_double(LanternResource, representative_server: serv, db_user: "test", name: "test-res", id: "6181ddb3-0002-8ad0-9aeb-084832c9273b") + doctor = instance_double(LanternDoctor, resource: resource, ubid: "test-ubid") + dbs = ["db1", "db2"] + + expect(serv).to receive(:list_all_databases).and_return(dbs) + + expect(serv).to receive(:run_query).with(lantern_doctor_query.sql, db: "db1", user: resource.db_user).and_raise("test-err") + expect(serv).to receive(:run_query).with(lantern_doctor_query.sql, db: "db2", user: resource.db_user).and_return("f") + + expect(lantern_doctor_query).to receive(:db_name).and_return("*") + expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once) + expect(lantern_doctor_query).to receive(:should_run?).and_return(true) + expect(lantern_doctor_query).to receive(:update).with(hash_including(condition: "failed")) + expect(Prog::PageNexus).to receive(:assemble_with_logs).with("Query #{lantern_doctor_query.name} failed on #{doctor.resource.name} (db1)", [lantern_doctor_query.ubid, doctor.ubid, serv.ubid], {"stderr" => "test-err"}, lantern_doctor_query.severity, "LanternDoctorQueryFailed", lantern_doctor_query.id, "db1") + + expect { lantern_doctor_query.run }.not_to raise_error + end + + it "runs query on all databases and fails" do + serv = instance_double(LanternServer, ubid: "test-ubid") + resource = instance_double(LanternResource, representative_server: serv, db_user: "test", name: "test-res", id: "6181ddb3-0002-8ad0-9aeb-084832c9273b") + doctor = instance_double(LanternDoctor, resource: resource, ubid: "test-ubid") + dbs = ["db1", "db2"] + + expect(serv).to receive(:list_all_databases).and_return(dbs) + + expect(serv).to receive(:run_query).with(lantern_doctor_query.sql, db: "db1", user: resource.db_user).and_return("t") + expect(serv).to receive(:run_query).with(lantern_doctor_query.sql, db: "db2", user: resource.db_user).and_return("f") + + expect(lantern_doctor_query).to receive(:db_name).and_return("*") + expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once) + expect(lantern_doctor_query).to receive(:should_run?).and_return(true) + expect(lantern_doctor_query).to receive(:update).with(hash_including(condition: "failed")) + expect(Prog::PageNexus).to receive(:assemble_with_logs).with("Query #{lantern_doctor_query.name} failed on #{doctor.resource.name} (db1)", [lantern_doctor_query.ubid, doctor.ubid, serv.ubid], {"stderr" => ""}, lantern_doctor_query.severity, "LanternDoctorQueryFailed", lantern_doctor_query.id, "db1") + + expect { lantern_doctor_query.run }.not_to raise_error + end + + it "does not create duplicate page" do + serv = instance_double(LanternServer, ubid: "test-ubid") + resource = instance_double(LanternResource, representative_server: serv, db_user: "test", name: "test-res", id: "6181ddb3-0002-8ad0-9aeb-084832c9273b") + doctor = instance_double(LanternDoctor, resource: resource, ubid: "test-ubid") + dbs = ["db1", "db2"] + + expect(serv).to receive(:list_all_databases).and_return(dbs) + + expect(serv).to receive(:run_query).with(lantern_doctor_query.sql, db: "db1", user: resource.db_user).and_return("t") + expect(serv).to receive(:run_query).with(lantern_doctor_query.sql, db: "db2", user: resource.db_user).and_return("f") + + expect(lantern_doctor_query).to receive(:db_name).and_return("*") + expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once) + expect(lantern_doctor_query).to receive(:should_run?).and_return(true) + expect(lantern_doctor_query).to receive(:update).with(hash_including(condition: "failed")) + expect(Page).to receive(:from_tag_parts).with("LanternDoctorQueryFailed", lantern_doctor_query.id, "db1").and_return(instance_double(Page)) + expect(Page).to receive(:from_tag_parts).with("LanternDoctorQueryFailed", lantern_doctor_query.id, "db2").and_return(nil) + expect(Prog::PageNexus).not_to receive(:assemble_with_logs) + + expect { lantern_doctor_query.run }.not_to raise_error + end + + it "runs query on all databases and resolves previous error" do + serv = instance_double(LanternServer, ubid: "test-ubid") + resource = instance_double(LanternResource, representative_server: serv, db_user: "test", name: "test-res", id: "6181ddb3-0002-8ad0-9aeb-084832c9273b") + doctor = instance_double(LanternDoctor, resource: resource, ubid: "test-ubid") + dbs = ["db1", "db2"] + + expect(serv).to receive(:list_all_databases).and_return(dbs) + + expect(serv).to receive(:run_query).with(lantern_doctor_query.sql, db: "db1", user: resource.db_user).and_return("f") + expect(serv).to receive(:run_query).with(lantern_doctor_query.sql, db: "db2", user: resource.db_user).and_return("f") + + expect(lantern_doctor_query).to receive(:db_name).and_return("*") + expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once) + expect(lantern_doctor_query).to receive(:should_run?).and_return(true) + expect(lantern_doctor_query).to receive(:update).with(hash_including(condition: "healthy")) + page = instance_double(Page) + expect(Page).to receive(:from_tag_parts).with("LanternDoctorQueryFailed", lantern_doctor_query.id, "db1").and_return(page) + expect(Page).to receive(:from_tag_parts).with("LanternDoctorQueryFailed", lantern_doctor_query.id, "db2").and_return(nil) + expect(page).to receive(:incr_resolve) + + expect { lantern_doctor_query.run }.not_to raise_error + end + end + + describe "#check_daemon_embedding_jobs" do + it "fails if not backend db connection" do + expect(LanternBackend).to receive(:db).and_return(nil) + expect { lantern_doctor_query.check_daemon_embedding_jobs "postgres", "postgres" }.to raise_error "No connection to lantern backend database specified" + end + + it "get jobs and fails" do + serv = instance_double(LanternServer, ubid: "test-ubid") + resource = instance_double(LanternResource, representative_server: serv, db_user: "test", name: "test-res", id: "6181ddb3-0002-8ad0-9aeb-084832c9273b") + doctor = instance_double(LanternDoctor, resource: resource, ubid: "test-ubid") + expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once) + expect(LanternBackend).to receive(:db).and_return(DB).at_least(:once) + expect(LanternBackend.db).to receive(:select) + .and_return(instance_double(Sequel::Dataset, + from: instance_double(Sequel::Dataset, + where: instance_double(Sequel::Dataset, + where: instance_double(Sequel::Dataset, + where: instance_double(Sequel::Dataset, + all: [{schema: "public", table: "test", src_column: "test-src", dst_column: "test-dst"}])))))) + expect(serv).to receive(:run_query).with("SELECT EXISTS(SELECT (SELECT COUNT(*) FROM \"public\".\"test\" WHERE \"test-src\" IS NOT NULL AND \"test-src\" != '' AND \"test-dst\" IS NULL) > 1000)", db: "postgres", user: "postgres").and_return("t") + expect(lantern_doctor_query.check_daemon_embedding_jobs("postgres", "postgres")).to eq("t") + end + + it "get jobs as empty" do + serv = instance_double(LanternServer, ubid: "test-ubid") + resource = instance_double(LanternResource, representative_server: serv, db_user: "test", name: "test-res", id: "6181ddb3-0002-8ad0-9aeb-084832c9273b") + doctor = instance_double(LanternDoctor, resource: resource, ubid: "test-ubid") + expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once) + expect(LanternBackend).to receive(:db).and_return(DB).at_least(:once) + expect(LanternBackend.db).to receive(:select) + .and_return(instance_double(Sequel::Dataset, + from: instance_double(Sequel::Dataset, + where: instance_double(Sequel::Dataset, + where: instance_double(Sequel::Dataset, + where: instance_double(Sequel::Dataset, + all: [])))))) + expect(lantern_doctor_query.check_daemon_embedding_jobs("postgres", "postgres")).to eq("f") + end + end + + it "get jobs and succceds" do + serv = instance_double(LanternServer, ubid: "test-ubid") + resource = instance_double(LanternResource, representative_server: serv, db_user: "test", name: "test-res", id: "6181ddb3-0002-8ad0-9aeb-084832c9273b") + doctor = instance_double(LanternDoctor, resource: resource, ubid: "test-ubid") + expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once) + expect(LanternBackend).to receive(:db).and_return(DB).at_least(:once) + expect(LanternBackend.db).to receive(:select) + .and_return(instance_double(Sequel::Dataset, + from: instance_double(Sequel::Dataset, + where: instance_double(Sequel::Dataset, + where: instance_double(Sequel::Dataset, + where: instance_double(Sequel::Dataset, + all: [{schema: "public", table: "test", src_column: "test-src", dst_column: "test-dst"}])))))) + expect(serv).to receive(:run_query).with("SELECT EXISTS(SELECT (SELECT COUNT(*) FROM \"public\".\"test\" WHERE \"test-src\" IS NOT NULL AND \"test-src\" != '' AND \"test-dst\" IS NULL) > 1000)", db: "postgres", user: "postgres").and_return("f") + expect(lantern_doctor_query.check_daemon_embedding_jobs("postgres", "postgres")).to eq("f") + end +end diff --git a/spec/model/lantern/lantern_doctor_spec.rb b/spec/model/lantern/lantern_doctor_spec.rb new file mode 100644 index 000000000..ff3429ca5 --- /dev/null +++ b/spec/model/lantern/lantern_doctor_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +RSpec.describe LanternDoctor do + subject(:lantern_doctor) { + described_class.new do |r| + r.id = "6181ddb3-0002-8ad0-9aeb-084832c9273b" + end + } + + describe "#system_queries" do + it "returns cached queries" do + expect(lantern_doctor).to receive(:system_queries).and_return([instance_double(LanternDoctorQuery)]) + expect(lantern_doctor.system_queries.size).to be(1) + end + + it "fetches system queries" do + expect(LanternDoctorQuery).to receive(:where).with(type: "system").and_return([instance_double(LanternDoctorQuery), instance_double(LanternDoctorQuery)]) + expect(lantern_doctor.system_queries.size).to be(2) + end + end + + describe "#has_system_query?" do + it "returns true if system query exists in query list" do + system_query = instance_double(LanternDoctorQuery, id: "test-parent-id") + query_list = [instance_double(LanternDoctorQuery, parent_id: "test-parent-id")] + expect(lantern_doctor.has_system_query?(query_list, system_query)).to be(true) + end + + it "returns false if system query does not exist in query list" do + system_query = instance_double(LanternDoctorQuery, id: "test-parent-id") + query_list = [instance_double(LanternDoctorQuery, parent_id: "test-parent-id2"), instance_double(LanternDoctorQuery, parent_id: nil)] + expect(lantern_doctor.has_system_query?(query_list, system_query)).to be(false) + end + end + + describe "#list_queries" do + it "creates new system query if not exists" do + system_queries = [instance_double(LanternDoctorQuery, id: "test-parent-id"), instance_double(LanternDoctorQuery, id: "test-parent-id2")] + query_list = [instance_double(LanternDoctorQuery, parent_id: "test-parent-id2"), instance_double(LanternDoctorQuery, parent_id: nil)] + expect(lantern_doctor).to receive(:queries).and_return(query_list) + expect(lantern_doctor).to receive(:system_queries).and_return(system_queries) + new_query = instance_double(LanternDoctorQuery, parent_id: "test-parent-id") + expect(LanternDoctorQuery).to receive(:create_with_id).with(parent_id: "test-parent-id", condition: "unknown").and_return(new_query) + # query list is changed by reference in list_queries method + expect(lantern_doctor.list_queries).to eq(query_list) + end + end + + describe "#list_incidents" do + it "returns all incidents" do + expect(lantern_doctor.list_incidents).to eq([]) + end + end +end diff --git a/spec/model/lantern/lantern_server_spec.rb b/spec/model/lantern/lantern_server_spec.rb index e91dd9833..844f4f070 100644 --- a/spec/model/lantern/lantern_server_spec.rb +++ b/spec/model/lantern/lantern_server_spec.rb @@ -115,13 +115,25 @@ end it "runs query on vm" do - expect(lantern_server.vm.sshable).to receive(:cmd).with("sudo lantern/bin/exec", stdin: "SELECT 1").and_return("1\n") + expect(lantern_server.vm.sshable).to receive(:cmd).with("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec -T postgresql psql -U postgres -t --csv postgres", stdin: "SELECT 1").and_return("1\n") expect(lantern_server.run_query("SELECT 1")).to eq("1") end + it "runs query on vm with different user and db" do + expect(lantern_server.vm.sshable).to receive(:cmd).with("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec -T postgresql psql -U lantern -t --csv db2", stdin: "SELECT 1").and_return("1\n") + expect(lantern_server.run_query("SELECT 1", db: "db2", user: "lantern")).to eq("1") + end + it "runs query on vm for all databases" do - expect(lantern_server.vm.sshable).to receive(:cmd).with("sudo lantern/bin/exec_all", stdin: "SELECT 1").and_return("1\n") - expect(lantern_server.run_query_all("SELECT 1")).to eq("1") + expect(lantern_server).to receive(:list_all_databases).and_return(["postgres", "db2"]) + expect(lantern_server.vm.sshable).to receive(:cmd).with("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec -T postgresql psql -U postgres -t --csv postgres", stdin: "SELECT 1").and_return("1\n") + expect(lantern_server.vm.sshable).to receive(:cmd).with("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec -T postgresql psql -U postgres -t --csv db2", stdin: "SELECT 1").and_return("2\n") + expect(lantern_server.run_query_all("SELECT 1")).to eq( + [ + ["postgres", "1"], + ["db2", "2"] + ] + ) end describe "#standby?" do @@ -587,4 +599,11 @@ expect(lantern_server.prewarm_indexes_query).to eq(query) end end + + describe "#list_all_databases" do + it "returns list of all databases" do + expect(lantern_server.vm.sshable).to receive(:cmd).with("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec postgresql psql -U postgres -P \"footer=off\" -c 'SELECT datname from pg_database' | tail -n +3 | grep -v 'template0' | grep -v 'template1'").and_return("postgres\ndb2\n") + expect(lantern_server.list_all_databases).to eq(["postgres", "db2"]) + end + end end diff --git a/spec/prog/lantern/lantern_doctor_nexus_spec.rb b/spec/prog/lantern/lantern_doctor_nexus_spec.rb new file mode 100644 index 000000000..34c9597ed --- /dev/null +++ b/spec/prog/lantern/lantern_doctor_nexus_spec.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require_relative "../../model/spec_helper" + +RSpec.describe Prog::Lantern::LanternDoctorNexus do + subject(:nx) { described_class.new(Strand.create(id: "6ae7e513-c34a-8039-a72a-7be45b53f2a0", prog: "Lantern::LanternDoctorNexus", label: "start")) } + + let(:lantern_doctor) { + instance_double( + LanternDoctor, + ubid: "6ae7e513-c34a-8039-a72a-7be45b53f2a0", + id: "6ae7e513-c34a-8039-a72a-7be45b53f2a0" + ) + } + + before do + allow(nx).to receive(:lantern_doctor).and_return(lantern_doctor) + end + + describe ".assemble" do + it "creates lantern doctor" do + st = described_class.assemble + doctor = LanternDoctor[st.id] + expect(doctor).not_to be_nil + end + end + + describe "#start" do + it "hops to wait resource" do + expect { nx.start }.to hop("wait_resource") + end + end + + describe "#wait_resource" do + it "naps if resource is not available" do + expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, strand: instance_double(Strand, label: "start"))) + expect { nx.wait_resource }.to nap(5) + end + + it "hops to wait" do + expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, strand: instance_double(Strand, label: "wait"))) + expect { nx.wait_resource }.to hop("wait") + end + end + + describe "#wait" do + it "runs queries and naps" do + queries = [instance_double(LanternDoctorQuery)] + expect(queries[0]).to receive(:run) + expect(lantern_doctor).to receive(:queries).and_return(queries) + expect { nx.wait }.to nap(60) + end + end + + describe "#before_run" do + it "hops to destroy" do + expect(nx).to receive(:when_destroy_set?).and_yield + expect { nx.before_run }.to hop("destroy") + end + + it "does not hop to destroy as strand label is not destroy" do + expect(nx).to receive(:when_destroy_set?).and_yield + expect(nx).to receive(:strand).and_return(instance_double(Strand, label: "destroy")) + expect(nx.before_run).to be_nil + end + + it "does not hop to destroy" do + expect(nx.before_run).to be_nil + end + end + + describe "#destroy" do + it "exits with message" do + expect(nx).to receive(:decr_destroy) + expect(lantern_doctor).to receive(:destroy) + expect { nx.destroy }.to exit({"msg" => "lantern doctor is deleted"}) + end + end +end diff --git a/ubid.rb b/ubid.rb index 53e762e85..b35b5ad0d 100644 --- a/ubid.rb +++ b/ubid.rb @@ -35,6 +35,8 @@ class UBID TYPE_LANTERN_RESOURCE = "1r" TYPE_LANTERN_SERVER = "1s" TYPE_LANTERN_TIMELINE = "1t" + TYPE_LANTERN_DOCTOR = "1d" + TYPE_LANTERN_DOCTOR_QUERY = "dq" TYPE_PROJECT = "pj" TYPE_ACCESS_TAG = "tg" TYPE_ACCESS_POLICY = "pc" diff --git a/views/lantern/create.erb b/views/lantern/create.erb index 36c44aae2..66c7546f5 100644 --- a/views/lantern/create.erb +++ b/views/lantern/create.erb @@ -49,7 +49,6 @@ name: "label", label: "Label", attributes: { - required: false, placeholder: "Enter label" } } From dc5d6b007b66bbdf4e90be9ccaaee726f85ef0fa Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Mon, 6 May 2024 11:06:59 -0700 Subject: [PATCH 04/13] Fix lantern doctor queries, add routes [WIP] --- migrate/20240505_lantern_doctor.rb | 2 +- migrate/20240506_lantern_doctor_queries.rb | 2 +- model/lantern/lantern_doctor.rb | 21 +- model/lantern/lantern_doctor_query.rb | 17 +- model/lantern/lantern_server.rb | 6 +- prog/lantern/lantern_doctor_nexus.rb | 19 +- routes/api/project/location/lantern_doctor.rb | 21 ++ serializers/api/lantern_doctor_query.rb | 21 ++ .../lantern/lantern_doctor_query_spec.rb | 10 +- spec/model/lantern/lantern_doctor_spec.rb | 13 +- .../prog/lantern/lantern_doctor_nexus_spec.rb | 24 ++ .../project/location/lantern_doctor_spec.rb | 324 ++++++++++++++++++ 12 files changed, 429 insertions(+), 51 deletions(-) create mode 100644 routes/api/project/location/lantern_doctor.rb create mode 100644 serializers/api/lantern_doctor_query.rb create mode 100644 spec/routes/api/project/location/lantern_doctor_spec.rb diff --git a/migrate/20240505_lantern_doctor.rb b/migrate/20240505_lantern_doctor.rb index 8cc7539d5..527bb1d23 100644 --- a/migrate/20240505_lantern_doctor.rb +++ b/migrate/20240505_lantern_doctor.rb @@ -13,7 +13,7 @@ end # queries create_enum(:query_condition, %w[unknown healthy failed]) - create_enum(:query_type, %w[system udf]) + create_enum(:query_type, %w[system user]) create_table(:lantern_doctor_query) do column :id, :uuid, primary_key: true, default: nil foreign_key :parent_id, :lantern_doctor_query, type: :uuid diff --git a/migrate/20240506_lantern_doctor_queries.rb b/migrate/20240506_lantern_doctor_queries.rb index df48ed2a7..b197ed9d3 100644 --- a/migrate/20240506_lantern_doctor_queries.rb +++ b/migrate/20240506_lantern_doctor_queries.rb @@ -3,6 +3,6 @@ Sequel.migration do change do run "INSERT INTO lantern_doctor_query (id, name, db_name, schedule, condition, fn_label, type, severity) - VALUES ('4f916f44-3c7a-89b7-9795-1ccd417b45ba', 'Check Daemon Embedding Job', '*', '*/5 * * * *', 'unknown', 'check_daemon_embedding_jobs', 'system', 'error')" + VALUES ('4f916f44-3c7a-89b7-9795-1ccd417b45ba', 'Lantern Embedding Generation Job', '*', '*/30 * * * *', 'unknown', 'check_daemon_embedding_jobs', 'system', 'error')" end end diff --git a/model/lantern/lantern_doctor.rb b/model/lantern/lantern_doctor.rb index 73f7c461e..806fd489d 100644 --- a/model/lantern/lantern_doctor.rb +++ b/model/lantern/lantern_doctor.rb @@ -6,13 +6,14 @@ class LanternDoctor < Sequel::Model one_to_one :strand, key: :id one_to_one :resource, class: LanternResource, key: :doctor_id one_to_many :queries, key: :doctor_id, class: LanternDoctorQuery + one_to_many :failed_queries, key: :doctor_id, class: LanternDoctorQuery, conditions: {condition: "failed"} plugin :association_dependencies, queries: :destroy include ResourceMethods include SemaphoreMethods - semaphore :destroy + semaphore :destroy, :sync_system_queries def system_queries @system_queries ||= LanternDoctorQuery.where(type: "system") @@ -22,28 +23,14 @@ def has_system_query?(queries, query) queries.any? { _1.parent_id == query.id } end - def list_queries + def sync_system_queries doctor_query_list = queries system_query_list = system_queries - merged_queries = doctor_query_list system_query_list.each { if !has_system_query?(doctor_query_list, _1) - doctor_query = LanternDoctorQuery.create_with_id(parent_id: _1.id, condition: "unknown", type: "system") - doctor_query.parent = _1 - merged_queries.push(doctor_query) + LanternDoctorQuery.create_with_id(parent_id: _1.id, doctor_id: id, condition: "unknown", type: "user") end } - - puts "Merged queries are ->>>>. #{merged_queries}" - merged_queries - end - - def list_incidents - # TODO:: - # Find all unhealthy queries - # Get all pages for that queries - # Map results - [] end end diff --git a/model/lantern/lantern_doctor_query.rb b/model/lantern/lantern_doctor_query.rb index 06bbe5b27..fa7c46437 100644 --- a/model/lantern/lantern_doctor_query.rb +++ b/model/lantern/lantern_doctor_query.rb @@ -6,8 +6,8 @@ class LanternDoctorQuery < Sequel::Model one_to_one :strand, key: :id - one_to_one :doctor, key: :id, primary_key: :doctor_id - one_to_one :parent, class: self, key: :parent_id + many_to_one :doctor, class: LanternDoctor, key: :doctor_id, primary_key: :id + many_to_one :parent, class: self, key: :parent_id one_to_many :children, key: :parent_id, class: self plugin :association_dependencies, children: :destroy @@ -42,8 +42,6 @@ def fn_label end def should_run? - puts "Parent ->>>>> #{parent}" - puts "Schedule is ->>>>>>> #{schedule}" CronParser.new(schedule).next(last_checked || Time.new - 61) <= Time.new end @@ -61,8 +59,6 @@ def run return nil end - puts "RUNNING QUERY::::::::::::::::: #{name}" - lantern_server = doctor.resource.representative_server dbs = (db_name == "*") ? lantern_server.list_all_databases : [db_name] query_user = user @@ -95,7 +91,7 @@ def run pg = Page.from_tag_parts("LanternDoctorQueryFailed", id, db) if failed && !pg - Prog::PageNexus.assemble_with_logs("Query #{name} failed on #{doctor.resource.name} (#{db})", [ubid, doctor.ubid, lantern_server.ubid], {"stderr" => err_msg}, severity, "LanternDoctorQueryFailed", id, db) + Prog::PageNexus.assemble_with_logs("Healthcheck: #{name} failed on #{doctor.resource.name} (#{db})", [ubid, doctor.ubid, lantern_server.ubid], {"stderr" => err_msg}, severity, "LanternDoctorQueryFailed", id, db) elsif !failed && pg pg.incr_resolve end @@ -109,15 +105,12 @@ def check_daemon_embedding_jobs(db, query_user) fail "No connection to lantern backend database specified" end - puts "HERE FINDING JOBS" jobs = LanternBackend.db .select(:schema, :table, :src_column, :dst_column) .from(:embedding_generation_jobs) .where(database_id: doctor.resource.name) .where(Sequel.like(:db_connection, "%/#{db}")) - # :nocov: - .where { !init_finished_at.nil? } - # :nocov: + .where(Sequel.lit('init_finished_at IS NOT NULL')) .all if jobs.empty? @@ -126,7 +119,7 @@ def check_daemon_embedding_jobs(db, query_user) lantern_server = doctor.resource.representative_server failed = jobs.any? do |job| - res = lantern_server.run_query("SELECT EXISTS(SELECT (SELECT COUNT(*) FROM \"#{job[:schema]}\".\"#{job[:table]}\" WHERE \"#{job[:src_column]}\" IS NOT NULL AND \"#{job[:src_column]}\" != '' AND \"#{job[:dst_column]}\" IS NULL) > 1000)", db: db, user: query_user).strip + res = lantern_server.run_query("SELECT (SELECT COUNT(*) FROM \"#{job[:schema]}\".\"#{job[:table]}\" WHERE \"#{job[:src_column]}\" IS NOT NULL AND \"#{job[:src_column]}\" != '' AND \"#{job[:dst_column]}\" IS NULL) > 1000", db: db, user: query_user).strip res == "t" end diff --git a/model/lantern/lantern_server.rb b/model/lantern/lantern_server.rb index 56b1f5da1..98300bc34 100644 --- a/model/lantern/lantern_server.rb +++ b/model/lantern/lantern_server.rb @@ -197,7 +197,11 @@ def prewarm_indexes_query end def list_all_databases - vm.sshable.cmd("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec postgresql psql -U postgres -P \"footer=off\" -c 'SELECT datname from pg_database' | tail -n +3 | grep -v 'template0' | grep -v 'template1'").chomp.strip.split("\n") + vm.sshable.cmd("sudo docker compose -f /var/lib/lantern/docker-compose.yaml exec postgresql psql -U postgres -P \"footer=off\" -c 'SELECT datname from pg_database' | tail -n +3 | grep -v 'template0' | grep -v 'template1'") + .chomp + .strip + .split("\n") + .map { _1.strip } end # def failover_target diff --git a/prog/lantern/lantern_doctor_nexus.rb b/prog/lantern/lantern_doctor_nexus.rb index cbf00b0d9..4f608d552 100644 --- a/prog/lantern/lantern_doctor_nexus.rb +++ b/prog/lantern/lantern_doctor_nexus.rb @@ -8,7 +8,7 @@ class Prog::Lantern::LanternDoctorNexus < Prog::Base extend Forwardable def_delegators :lantern_doctor - semaphore :destroy + semaphore :destroy, :sync_system_queries def self.assemble DB.transaction do @@ -26,17 +26,28 @@ def before_run end label def start + lantern_doctor.sync_system_queries hop_wait_resource end label def wait_resource - nap 5 if lantern_doctor.resource.strand.label != "wait" + nap 5 if lantern_doctor.resource&.strand&.label != "wait" hop_wait end label def wait - lantern_doctor.list_queries.each { _1.run } - nap 10 + when_sync_system_queries_set? do + hop_sync_system_queries + end + + lantern_doctor.queries.each { _1.run } + nap 60 + end + + label def sync_system_queries + decr_sync_system_queries + lantern_doctor.sync_system_queries + hop_wait end label def destroy diff --git a/routes/api/project/location/lantern_doctor.rb b/routes/api/project/location/lantern_doctor.rb new file mode 100644 index 000000000..705b71233 --- /dev/null +++ b/routes/api/project/location/lantern_doctor.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class CloverApi + hash_branch(:project_location_lantern_prefix, "doctor") do |r| + @serializer = Serializers::Api::LanternDoctorQuery + + r.get true do + result = @pg.doctor.queries.authorized(@current_user.id, "Postgres:view").eager(:semaphores, :strand).paginated_result( + cursor: r.params["cursor"], + page_size: r.params["page_size"], + order_column: r.params["order_column"] + ) + + { + items: serialize(result[:records]), + next_cursor: result[:next_cursor], + count: result[:count] + } + end + end +end diff --git a/serializers/api/lantern_doctor_query.rb b/serializers/api/lantern_doctor_query.rb new file mode 100644 index 000000000..f6fc0c52a --- /dev/null +++ b/serializers/api/lantern_doctor_query.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class Serializers::Api::LanternDoctorQuery < Serializers::Base + def self.base() + { + name: query.name, + type: query.type, + condition: query.condition + } + end + + structure(:default) do |pg| + base(pg) + end + + structure(:detailed) do |pg| + base(pg).merge({ + incidents: [] + }) + end +end diff --git a/spec/model/lantern/lantern_doctor_query_spec.rb b/spec/model/lantern/lantern_doctor_query_spec.rb index 75047acf4..beacbbfa1 100644 --- a/spec/model/lantern/lantern_doctor_query_spec.rb +++ b/spec/model/lantern/lantern_doctor_query_spec.rb @@ -155,7 +155,7 @@ expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once) expect(lantern_doctor_query).to receive(:should_run?).and_return(true) expect(lantern_doctor_query).to receive(:update).with(hash_including(condition: "failed")) - expect(Prog::PageNexus).to receive(:assemble_with_logs).with("Query #{lantern_doctor_query.name} failed on #{doctor.resource.name} (postgres)", [lantern_doctor_query.ubid, doctor.ubid, serv.ubid], {"stderr" => "BUG: non-system query without sql"}, lantern_doctor_query.severity, "LanternDoctorQueryFailed", lantern_doctor_query.id, "postgres") + expect(Prog::PageNexus).to receive(:assemble_with_logs).with("Healthcheck: #{lantern_doctor_query.name} failed on #{doctor.resource.name} (postgres)", [lantern_doctor_query.ubid, doctor.ubid, serv.ubid], {"stderr" => "BUG: non-system query without sql"}, lantern_doctor_query.severity, "LanternDoctorQueryFailed", lantern_doctor_query.id, "postgres") expect { lantern_doctor_query.run }.not_to raise_error end @@ -213,7 +213,7 @@ expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once) expect(lantern_doctor_query).to receive(:should_run?).and_return(true) expect(lantern_doctor_query).to receive(:update).with(hash_including(condition: "failed")) - expect(Prog::PageNexus).to receive(:assemble_with_logs).with("Query #{lantern_doctor_query.name} failed on #{doctor.resource.name} (db1)", [lantern_doctor_query.ubid, doctor.ubid, serv.ubid], {"stderr" => "test-err"}, lantern_doctor_query.severity, "LanternDoctorQueryFailed", lantern_doctor_query.id, "db1") + expect(Prog::PageNexus).to receive(:assemble_with_logs).with("Healthcheck: #{lantern_doctor_query.name} failed on #{doctor.resource.name} (db1)", [lantern_doctor_query.ubid, doctor.ubid, serv.ubid], {"stderr" => "test-err"}, lantern_doctor_query.severity, "LanternDoctorQueryFailed", lantern_doctor_query.id, "db1") expect { lantern_doctor_query.run }.not_to raise_error end @@ -233,7 +233,7 @@ expect(lantern_doctor_query).to receive(:doctor).and_return(doctor).at_least(:once) expect(lantern_doctor_query).to receive(:should_run?).and_return(true) expect(lantern_doctor_query).to receive(:update).with(hash_including(condition: "failed")) - expect(Prog::PageNexus).to receive(:assemble_with_logs).with("Query #{lantern_doctor_query.name} failed on #{doctor.resource.name} (db1)", [lantern_doctor_query.ubid, doctor.ubid, serv.ubid], {"stderr" => ""}, lantern_doctor_query.severity, "LanternDoctorQueryFailed", lantern_doctor_query.id, "db1") + expect(Prog::PageNexus).to receive(:assemble_with_logs).with("Healthcheck: #{lantern_doctor_query.name} failed on #{doctor.resource.name} (db1)", [lantern_doctor_query.ubid, doctor.ubid, serv.ubid], {"stderr" => ""}, lantern_doctor_query.severity, "LanternDoctorQueryFailed", lantern_doctor_query.id, "db1") expect { lantern_doctor_query.run }.not_to raise_error end @@ -303,7 +303,7 @@ where: instance_double(Sequel::Dataset, where: instance_double(Sequel::Dataset, all: [{schema: "public", table: "test", src_column: "test-src", dst_column: "test-dst"}])))))) - expect(serv).to receive(:run_query).with("SELECT EXISTS(SELECT (SELECT COUNT(*) FROM \"public\".\"test\" WHERE \"test-src\" IS NOT NULL AND \"test-src\" != '' AND \"test-dst\" IS NULL) > 1000)", db: "postgres", user: "postgres").and_return("t") + expect(serv).to receive(:run_query).with("SELECT (SELECT COUNT(*) FROM \"public\".\"test\" WHERE \"test-src\" IS NOT NULL AND \"test-src\" != '' AND \"test-dst\" IS NULL) > 1000", db: "postgres", user: "postgres").and_return("t") expect(lantern_doctor_query.check_daemon_embedding_jobs("postgres", "postgres")).to eq("t") end @@ -337,7 +337,7 @@ where: instance_double(Sequel::Dataset, where: instance_double(Sequel::Dataset, all: [{schema: "public", table: "test", src_column: "test-src", dst_column: "test-dst"}])))))) - expect(serv).to receive(:run_query).with("SELECT EXISTS(SELECT (SELECT COUNT(*) FROM \"public\".\"test\" WHERE \"test-src\" IS NOT NULL AND \"test-src\" != '' AND \"test-dst\" IS NULL) > 1000)", db: "postgres", user: "postgres").and_return("f") + expect(serv).to receive(:run_query).with("SELECT (SELECT COUNT(*) FROM \"public\".\"test\" WHERE \"test-src\" IS NOT NULL AND \"test-src\" != '' AND \"test-dst\" IS NULL) > 1000", db: "postgres", user: "postgres").and_return("f") expect(lantern_doctor_query.check_daemon_embedding_jobs("postgres", "postgres")).to eq("f") end end diff --git a/spec/model/lantern/lantern_doctor_spec.rb b/spec/model/lantern/lantern_doctor_spec.rb index ff3429ca5..7a1f0dee8 100644 --- a/spec/model/lantern/lantern_doctor_spec.rb +++ b/spec/model/lantern/lantern_doctor_spec.rb @@ -35,22 +35,15 @@ end end - describe "#list_queries" do + describe "#sync_system_queries" do it "creates new system query if not exists" do system_queries = [instance_double(LanternDoctorQuery, id: "test-parent-id"), instance_double(LanternDoctorQuery, id: "test-parent-id2")] query_list = [instance_double(LanternDoctorQuery, parent_id: "test-parent-id2"), instance_double(LanternDoctorQuery, parent_id: nil)] expect(lantern_doctor).to receive(:queries).and_return(query_list) expect(lantern_doctor).to receive(:system_queries).and_return(system_queries) new_query = instance_double(LanternDoctorQuery, parent_id: "test-parent-id") - expect(LanternDoctorQuery).to receive(:create_with_id).with(parent_id: "test-parent-id", condition: "unknown").and_return(new_query) - # query list is changed by reference in list_queries method - expect(lantern_doctor.list_queries).to eq(query_list) - end - end - - describe "#list_incidents" do - it "returns all incidents" do - expect(lantern_doctor.list_incidents).to eq([]) + expect(LanternDoctorQuery).to receive(:create_with_id).with(parent_id: "test-parent-id", doctor_id: lantern_doctor.id, type: "user", condition: "unknown").and_return(new_query) + expect{lantern_doctor.sync_system_queries}.not_to raise_error end end end diff --git a/spec/prog/lantern/lantern_doctor_nexus_spec.rb b/spec/prog/lantern/lantern_doctor_nexus_spec.rb index 34c9597ed..22a4f7c6c 100644 --- a/spec/prog/lantern/lantern_doctor_nexus_spec.rb +++ b/spec/prog/lantern/lantern_doctor_nexus_spec.rb @@ -27,11 +27,22 @@ describe "#start" do it "hops to wait resource" do + expect(lantern_doctor).to receive(:sync_system_queries) expect { nx.start }.to hop("wait_resource") end end describe "#wait_resource" do + it "naps if no resource yet" do + expect(lantern_doctor).to receive(:resource).and_return(nil) + expect { nx.wait_resource }.to nap(5) + end + + it "naps if no resource strand yet" do + expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, strand: nil)) + expect { nx.wait_resource }.to nap(5) + end + it "naps if resource is not available" do expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, strand: instance_double(Strand, label: "start"))) expect { nx.wait_resource }.to nap(5) @@ -44,6 +55,11 @@ end describe "#wait" do + it "syncs system queries" do + expect(nx).to receive(:when_sync_system_queries_set?).and_yield + expect { nx.wait }.to hop("sync_system_queries") + end + it "runs queries and naps" do queries = [instance_double(LanternDoctorQuery)] expect(queries[0]).to receive(:run) @@ -69,6 +85,14 @@ end end + describe "#sync_system_queries" do + it "calls sync_system_queries" do + expect(lantern_doctor).to receive(:sync_system_queries) + expect(nx).to receive(:decr_sync_system_queries) + expect{nx.sync_system_queries}.to hop("wait") + end + end + describe "#destroy" do it "exits with message" do expect(nx).to receive(:decr_destroy) diff --git a/spec/routes/api/project/location/lantern_doctor_spec.rb b/spec/routes/api/project/location/lantern_doctor_spec.rb new file mode 100644 index 000000000..9708e9b42 --- /dev/null +++ b/spec/routes/api/project/location/lantern_doctor_spec.rb @@ -0,0 +1,324 @@ +# frozen_string_literal: true + +require_relative "../../spec_helper" + +RSpec.describe Clover, "lantern" do + let(:user) { create_account } + + let(:project) { user.create_project_with_default_policy("project-1", provider: "gcp") } + + let(:project_wo_permissions) { user.create_project_with_default_policy("project-2", provider: "gcp", policy_body: []) } + + let(:pg) do + st = Prog::Lantern::LanternResourceNexus.assemble( + project_id: project.id, + location: "us-central1", + name: "instance-1", + target_vm_size: "n1-standard-2", + target_storage_size_gib: 100, + org_id: 0 + ) + LanternResource[st.id] + end + + let(:pg_wo_pwermission) do + st = Prog::Lantern::LanternResourceNexus.assemble( + project_id: project_wo_permissions.id, + location: "us-central1", + name: "lantern-foo-1", + target_vm_size: "n1-standard-2", + target_storage_size_gib: 100, + org_id: 0 + ) + + LanternResource[st.id] + end + + describe "unauthenticated" do + before do + Project.create_with_id(name: "default", provider: "gcp").tap { _1.associate_with_project(_1) } + end + + it "not location list" do + get "/api/project/#{project.ubid}/location/#{pg.location}/lantern" + + expect(last_response.status).to eq(401) + expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") + end + + it "not create" do + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/postgres_name" + + expect(last_response.status).to eq(401) + expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") + end + + it "not delete" do + delete "/api/project/#{project.ubid}/location/#{pg.location}/lantern/#{pg.name}" + + expect(last_response.status).to eq(401) + expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") + end + + it "not get" do + get "/api/project/#{project.ubid}/location/#{pg.location}/lantern/#{pg.name}" + + expect(last_response.status).to eq(401) + expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") + end + + it "not reset super user password" do + post "/api/project/#{project.ubid}/location/#{pg.location}/postgres/#{pg.name}/reset-superuser-password" + + expect(last_response.status).to eq(401) + expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") + end + + it "not update extension" do + post "/api/project/#{project.ubid}/location/#{pg.location}/postgres/#{pg.name}/update-extension" + + expect(last_response.status).to eq(401) + expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") + end + + it "not update image" do + post "/api/project/#{project.ubid}/location/#{pg.location}/postgres/#{pg.name}/update-image" + + expect(last_response.status).to eq(401) + expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") + end + + it "not add domain" do + post "/api/project/#{project.ubid}/location/#{pg.location}/postgres/#{pg.name}/add-domain" + + expect(last_response.status).to eq(401) + expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") + end + + it "not update rhizome" do + post "/api/project/#{project.ubid}/location/#{pg.location}/postgres/#{pg.name}/update-rhizome" + + expect(last_response.status).to eq(401) + expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") + end + end + + describe "authenticated" do + before do + login_api(user.email) + Project.create_with_id(name: "default", provider: "gcp").tap { _1.associate_with_project(_1) } + end + + describe "list" do + it "empty" do + get "/api/project/#{project.ubid}/location/#{TEST_LOCATION}/lantern" + + expect(last_response.status).to eq(200) + expect(JSON.parse(last_response.body)["items"]).to eq([]) + end + + it "success single" do + get "/api/project/#{project.ubid}/location/#{pg.location}/lantern" + + expect(last_response.status).to eq(200) + expect(JSON.parse(last_response.body)["items"].length).to eq(1) + end + + it "success multiple" do + Prog::Lantern::LanternResourceNexus.assemble( + project_id: project.id, + location: "us-central1", + name: "lantern-foo-2", + target_vm_size: "n1-standard-2", + target_storage_size_gib: 100, + org_id: 0 + ) + + get "/api/project/#{project.ubid}/location/#{pg.location}/lantern" + + expect(last_response.status).to eq(200) + expect(JSON.parse(last_response.body)["items"].length).to eq(2) + end + end + + describe "#update-extension" do + it "updates lantern extension" do + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/update-extension", {lantern_version: "0.2.4", extras_version: Config.lantern_extras_default_version} + expect(last_response.status).to eq(200) + end + + it "updates extras extension" do + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/update-extension", {extras_version: "0.2.3", lantern_version: Config.lantern_default_version} + expect(last_response.status).to eq(200) + end + end + + describe "#update-image" do + it "updates image" do + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/update-image", {lantern_version: "0.2.3", extras_version: "0.2.3", minor_version: "1"} + expect(last_response.status).to eq(200) + end + end + + describe "get" do + it "returns 404" do + get "/api/project/#{project.ubid}/location/#{pg.location}/lantern/test" + expect(last_response.status).to eq(404) + end + + it "returns instance" do + get "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1" + data = JSON.parse(last_response.body) + expect(data["name"]).to eq("instance-1") + expect(last_response.status).to eq(200) + end + end + + describe "delete" do + it "does not delete instance if has forks" do + expect(Project).to receive(:from_ubid).and_return(project).at_least(:once) + query_res = class_double(LanternResource, first: pg) + allow(query_res).to receive(:where).and_return(query_res) + expect(project).to receive(:lantern_resources_dataset).and_return(query_res) + expect(pg).to receive(:forks).and_return([instance_double(LanternResource)]) + + delete "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1" + expect(last_response.status).to eq(409) + expect(JSON.parse(last_response.body)).to eq({"error" => "Can not delete resource which has active forks"}) + end + + it "deletes instance" do + delete "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1" + expect(last_response.status).to eq(200) + end + end + + describe "start,stop,restart" do + it "starts instance" do + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/start" + expect(last_response.status).to eq(200) + end + + it "stops instance" do + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/stop" + expect(last_response.status).to eq(200) + end + + it "restarts instance" do + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/restart" + expect(last_response.status).to eq(200) + end + end + + describe "add-domain" do + it "adds domain" do + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/add-domain", {domain: "example.com"} + server = LanternServer.where(id: pg.representative_server.id).first + expect(server.domain).to eq("example.com") + expect(last_response.status).to eq(200) + end + end + + describe "reset-user-password" do + it "fails validation" do + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/reset-user-password", {original_password: "password123!", repeat_password: "test"} + expect(last_response.status).to eq(400) + end + + it "resets password" do + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/reset-user-password", {original_password: "Password123!", repeat_password: "Password123!"} + pg = LanternResource.first + expect(pg.db_user_password).to eq("Password123!") + expect(last_response.status).to eq(200) + end + end + + describe "update-vm" do + it "fails validation" do + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/update-vm", {storage_size_gib: 10} + expect(last_response.status).to eq(400) + end + + it "updates storage" do + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/update-vm", {storage_size_gib: 200} + server = LanternServer.where(id: pg.representative_server.id).first + expect(server.target_storage_size_gib).to eq(200) + expect(server.vm.storage_size_gib).to eq(200) + expect(last_response.status).to eq(200) + end + + it "updates vm size" do + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/update-vm", {size: "n1-standard-4"} + server = LanternServer.where(id: pg.representative_server.id).first + expect(server.target_vm_size).to eq("n1-standard-4") + expect(last_response.status).to eq(200) + end + end + + describe "backups" do + it "maps and prettifies backup keys correctly" do + time1 = Time.now - 20 * 60 + time2 = Time.now - 10 * 60 + ubid = LanternServer.where(id: pg.representative_server.id).first.timeline.ubid + backups = [{last_modified: time1, key: "#{ubid}/basebackups_005/1_backup_stop_sentinel.json"}, {last_modified: time2, key: "#{ubid}/basebackups_005/2_backup_stop_sentinel.json"}] + res_backups = JSON.parse(JSON.generate([{"time" => time1, "label" => "1", "compressed_size" => 10, "uncompressed_size" => 20}, {"time" => time2, "label" => "2", "compressed_size" => 10, "uncompressed_size" => 20}])) + gcp_client = instance_double(Hosting::GcpApis) + expect(gcp_client).to receive(:list_objects).and_return(backups) + expect(gcp_client).to receive(:get_json_object).and_return({"CompressedSize" => 10, "UncompressedSize" => 20}).at_least(:once) + allow(Hosting::GcpApis).to receive(:new).and_return(gcp_client) + + get "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/backups" + expect(last_response.status).to eq(200) + expect(JSON.parse(last_response.body)).to eq(res_backups) + end + end + + describe "push-backup" do + it "creates new backup" do + expect(Project).to receive(:from_ubid).and_return(project).at_least(:once) + query_res = class_double(LanternResource, first: pg) + allow(query_res).to receive(:where).and_return(query_res) + expect(project).to receive(:lantern_resources_dataset).and_return(query_res) + expect(pg.timeline).to receive(:take_manual_backup) + + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/push-backup" + expect(last_response.status).to eq(200) + end + + it "fails to create new backup" do + expect(Project).to receive(:from_ubid).and_return(project).at_least(:once) + query_res = class_double(LanternResource, first: pg) + allow(query_res).to receive(:where).and_return(query_res) + expect(project).to receive(:lantern_resources_dataset).and_return(query_res) + expect(pg.timeline).to receive(:take_manual_backup).and_raise "Another backup is in progress please try again later" + + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/push-backup" + expect(last_response.status).to eq(409) + expect(JSON.parse(last_response.body)).to eq({"error" => "Another backup is in progress please try again later"}) + end + + it "fails to create new backup with unknown" do + expect(Project).to receive(:from_ubid).and_return(project).at_least(:once) + query_res = class_double(LanternResource, first: pg) + allow(query_res).to receive(:where).and_return(query_res) + expect(project).to receive(:lantern_resources_dataset).and_return(query_res) + expect(pg.timeline).to receive(:take_manual_backup).and_raise "Unknown error" + + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/push-backup" + expect(last_response.status).to eq(400) + end + end + + describe "dissociate-forks" do + it "dissociate-forkses" do + expect(Project).to receive(:from_ubid).and_return(project).at_least(:once) + query_res = class_double(LanternResource, first: pg) + allow(query_res).to receive(:where).and_return(query_res) + expect(project).to receive(:lantern_resources_dataset).and_return(query_res) + expect(pg).to receive(:dissociate_forks) + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/dissociate-forks" + expect(last_response.status).to eq(200) + end + end + end +end From 9c2bb5174ce6adaa6a1b5e11e20340f5acbb36a7 Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Mon, 6 May 2024 12:56:46 -0700 Subject: [PATCH 05/13] Add routes to get active incidents --- model/lantern/lantern_doctor.rb | 2 +- model/lantern/lantern_doctor_query.rb | 13 +- model/lantern/lantern_resource.rb | 1 + routes/api/project/location/lantern.rb | 1 + routes/api/project/location/lantern/doctor.rb | 51 +++ routes/api/project/location/lantern_doctor.rb | 21 - serializers/api/lantern_doctor_query.rb | 21 +- .../lantern/lantern_doctor_query_spec.rb | 25 ++ spec/model/lantern/lantern_doctor_spec.rb | 4 +- .../prog/lantern/lantern_doctor_nexus_spec.rb | 6 +- .../project/location/lantern_doctor_spec.rb | 360 ++++-------------- 11 files changed, 191 insertions(+), 314 deletions(-) create mode 100644 routes/api/project/location/lantern/doctor.rb delete mode 100644 routes/api/project/location/lantern_doctor.rb diff --git a/model/lantern/lantern_doctor.rb b/model/lantern/lantern_doctor.rb index 806fd489d..ca25b5666 100644 --- a/model/lantern/lantern_doctor.rb +++ b/model/lantern/lantern_doctor.rb @@ -16,7 +16,7 @@ class LanternDoctor < Sequel::Model semaphore :destroy, :sync_system_queries def system_queries - @system_queries ||= LanternDoctorQuery.where(type: "system") + @system_queries ||= LanternDoctorQuery.where(type: "system").all end def has_system_query?(queries, query) diff --git a/model/lantern/lantern_doctor_query.rb b/model/lantern/lantern_doctor_query.rb index fa7c46437..ee38c3a01 100644 --- a/model/lantern/lantern_doctor_query.rb +++ b/model/lantern/lantern_doctor_query.rb @@ -11,6 +11,7 @@ class LanternDoctorQuery < Sequel::Model one_to_many :children, key: :parent_id, class: self plugin :association_dependencies, children: :destroy + dataset_module Pagination include ResourceMethods include SemaphoreMethods @@ -54,6 +55,16 @@ def user doctor.resource.db_user end + def active_pages + tag = Page.generate_tag("LanternDoctorQueryFailed", id) + Page.active.where(Sequel.like(:tag, "%#{tag}-%")).all + end + + def all_pages + tag = Page.generate_tag("LanternDoctorQueryFailed", id) + Page.where(Sequel.like(:tag, "%#{tag}-%")).all + end + def run if !should_run? return nil @@ -110,7 +121,7 @@ def check_daemon_embedding_jobs(db, query_user) .from(:embedding_generation_jobs) .where(database_id: doctor.resource.name) .where(Sequel.like(:db_connection, "%/#{db}")) - .where(Sequel.lit('init_finished_at IS NOT NULL')) + .where(Sequel.lit("init_finished_at IS NOT NULL")) .all if jobs.empty? diff --git a/model/lantern/lantern_resource.rb b/model/lantern/lantern_resource.rb index 913aeb194..c0b2fa03d 100644 --- a/model/lantern/lantern_resource.rb +++ b/model/lantern/lantern_resource.rb @@ -10,6 +10,7 @@ class LanternResource < Sequel::Model one_to_many :servers, class: LanternServer, key: :resource_id one_to_one :representative_server, class: LanternServer, key: :resource_id, conditions: Sequel.~(representative_at: nil) one_through_one :timeline, class: LanternTimeline, join_table: :lantern_server, left_key: :resource_id, right_key: :timeline_id + one_to_one :doctor, class: LanternDoctor, key: :id, primary_key: :doctor_id dataset_module Authorization::Dataset dataset_module Pagination diff --git a/routes/api/project/location/lantern.rb b/routes/api/project/location/lantern.rb index 8ed4d0b52..70ec3b605 100644 --- a/routes/api/project/location/lantern.rb +++ b/routes/api/project/location/lantern.rb @@ -12,6 +12,7 @@ class CloverApi end @pg = serialize(pg, :detailed) + r.hash_branches(:project_location_lantern_prefix) r.get true do Authorization.authorize(@current_user.id, "Postgres:view", pg.id) diff --git a/routes/api/project/location/lantern/doctor.rb b/routes/api/project/location/lantern/doctor.rb new file mode 100644 index 000000000..1404a02eb --- /dev/null +++ b/routes/api/project/location/lantern/doctor.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +class CloverApi + hash_branch(:project_location_lantern_prefix, "doctor") do |r| + @serializer = Serializers::Api::LanternDoctorQuery + @lantern_doctor = LanternResource[@pg[:id]].doctor + + r.get true do + result = LanternDoctorQuery.where(doctor_id: @lantern_doctor.id).paginated_result( + cursor: r.params["cursor"], + page_size: r.params["page_size"], + order_column: r.params["order_column"] + ) + + { + items: serialize(result[:records]), + next_cursor: result[:next_cursor], + count: result[:count] + } + end + + r.get "incidents" do + result = LanternDoctorQuery.where(doctor_id: @lantern_doctor.id, condition: "failed").paginated_result( + cursor: r.params["cursor"], + page_size: r.params["page_size"], + order_column: r.params["order_column"] + ) + + { + items: serialize(result[:records], :detailed), + next_cursor: result[:next_cursor], + count: result[:count] + } + end + + r.on String do |query_id| + query = LanternDoctorQuery[query_id] + + unless query + response.status = 404 + r.halt + end + + r.post "run" do + query.update(last_checked: nil) + response.status = 204 + r.halt + end + end + end +end diff --git a/routes/api/project/location/lantern_doctor.rb b/routes/api/project/location/lantern_doctor.rb deleted file mode 100644 index 705b71233..000000000 --- a/routes/api/project/location/lantern_doctor.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -class CloverApi - hash_branch(:project_location_lantern_prefix, "doctor") do |r| - @serializer = Serializers::Api::LanternDoctorQuery - - r.get true do - result = @pg.doctor.queries.authorized(@current_user.id, "Postgres:view").eager(:semaphores, :strand).paginated_result( - cursor: r.params["cursor"], - page_size: r.params["page_size"], - order_column: r.params["order_column"] - ) - - { - items: serialize(result[:records]), - next_cursor: result[:next_cursor], - count: result[:count] - } - end - end -end diff --git a/serializers/api/lantern_doctor_query.rb b/serializers/api/lantern_doctor_query.rb index f6fc0c52a..c1f3b9e1b 100644 --- a/serializers/api/lantern_doctor_query.rb +++ b/serializers/api/lantern_doctor_query.rb @@ -1,21 +1,28 @@ # frozen_string_literal: true class Serializers::Api::LanternDoctorQuery < Serializers::Base - def self.base() + def self.base(query) { + id: query.id, name: query.name, type: query.type, - condition: query.condition + condition: query.condition, + last_checked: query.last_checked, + schedule: query.schedule, + db_name: query.db_name, + severity: query.severity } end - structure(:default) do |pg| - base(pg) + structure(:default) do |query| + base(query) end - structure(:detailed) do |pg| - base(pg).merge({ - incidents: [] + structure(:detailed) do |query| + base(query).merge({ + incidents: query.active_pages.map { + {summary: _1.summary, logs: _1.details["logs"]["stderr"]} + } }) end end diff --git a/spec/model/lantern/lantern_doctor_query_spec.rb b/spec/model/lantern/lantern_doctor_query_spec.rb index beacbbfa1..158675046 100644 --- a/spec/model/lantern/lantern_doctor_query_spec.rb +++ b/spec/model/lantern/lantern_doctor_query_spec.rb @@ -340,4 +340,29 @@ expect(serv).to receive(:run_query).with("SELECT (SELECT COUNT(*) FROM \"public\".\"test\" WHERE \"test-src\" IS NOT NULL AND \"test-src\" != '' AND \"test-dst\" IS NULL) > 1000", db: "postgres", user: "postgres").and_return("f") expect(lantern_doctor_query.check_daemon_embedding_jobs("postgres", "postgres")).to eq("f") end + + describe "#page" do + it "lists active pages" do + p1 = Prog::PageNexus.assemble_with_logs("test", [lantern_doctor_query.ubid], {"stderr" => ""}, "error", "LanternDoctorQueryFailed", lantern_doctor_query.id, "postgres") + p2 = Prog::PageNexus.assemble_with_logs("test", [lantern_doctor_query.ubid], {"stderr" => ""}, "error", "LanternDoctorQueryFailed", lantern_doctor_query.id, "postgres2") + + Page[p2.id].update(resolved_at: Time.new) + + pages = lantern_doctor_query.active_pages + expect(pages.size).to be(1) + expect(pages[0].id).to eq(p1.id) + end + + it "lists all pages" do + p1 = Prog::PageNexus.assemble_with_logs("test", [lantern_doctor_query.ubid], {"stderr" => ""}, "error", "LanternDoctorQueryFailed", lantern_doctor_query.id, "postgres") + p2 = Prog::PageNexus.assemble_with_logs("test", [lantern_doctor_query.ubid], {"stderr" => ""}, "error", "LanternDoctorQueryFailed", lantern_doctor_query.id, "postgres2") + + Page[p2.id].update(resolved_at: Time.new) + + pages = lantern_doctor_query.all_pages + expect(pages.size).to be(2) + expect(pages[0].id).to eq(p1.id) + expect(pages[1].id).to eq(p2.id) + end + end end diff --git a/spec/model/lantern/lantern_doctor_spec.rb b/spec/model/lantern/lantern_doctor_spec.rb index 7a1f0dee8..584d628bb 100644 --- a/spec/model/lantern/lantern_doctor_spec.rb +++ b/spec/model/lantern/lantern_doctor_spec.rb @@ -16,7 +16,7 @@ end it "fetches system queries" do - expect(LanternDoctorQuery).to receive(:where).with(type: "system").and_return([instance_double(LanternDoctorQuery), instance_double(LanternDoctorQuery)]) + expect(LanternDoctorQuery).to receive(:where).with(type: "system").and_return(instance_double(Sequel::Dataset, all: [instance_double(LanternDoctorQuery), instance_double(LanternDoctorQuery)])) expect(lantern_doctor.system_queries.size).to be(2) end end @@ -43,7 +43,7 @@ expect(lantern_doctor).to receive(:system_queries).and_return(system_queries) new_query = instance_double(LanternDoctorQuery, parent_id: "test-parent-id") expect(LanternDoctorQuery).to receive(:create_with_id).with(parent_id: "test-parent-id", doctor_id: lantern_doctor.id, type: "user", condition: "unknown").and_return(new_query) - expect{lantern_doctor.sync_system_queries}.not_to raise_error + expect { lantern_doctor.sync_system_queries }.not_to raise_error end end end diff --git a/spec/prog/lantern/lantern_doctor_nexus_spec.rb b/spec/prog/lantern/lantern_doctor_nexus_spec.rb index 22a4f7c6c..2bf4f06ad 100644 --- a/spec/prog/lantern/lantern_doctor_nexus_spec.rb +++ b/spec/prog/lantern/lantern_doctor_nexus_spec.rb @@ -87,9 +87,9 @@ describe "#sync_system_queries" do it "calls sync_system_queries" do - expect(lantern_doctor).to receive(:sync_system_queries) - expect(nx).to receive(:decr_sync_system_queries) - expect{nx.sync_system_queries}.to hop("wait") + expect(lantern_doctor).to receive(:sync_system_queries) + expect(nx).to receive(:decr_sync_system_queries) + expect { nx.sync_system_queries }.to hop("wait") end end diff --git a/spec/routes/api/project/location/lantern_doctor_spec.rb b/spec/routes/api/project/location/lantern_doctor_spec.rb index 9708e9b42..89cb2b861 100644 --- a/spec/routes/api/project/location/lantern_doctor_spec.rb +++ b/spec/routes/api/project/location/lantern_doctor_spec.rb @@ -2,7 +2,7 @@ require_relative "../../spec_helper" -RSpec.describe Clover, "lantern" do +RSpec.describe Clover, "lantern-doctor" do let(:user) { create_account } let(:project) { user.create_project_with_default_policy("project-1", provider: "gcp") } @@ -21,88 +21,6 @@ LanternResource[st.id] end - let(:pg_wo_pwermission) do - st = Prog::Lantern::LanternResourceNexus.assemble( - project_id: project_wo_permissions.id, - location: "us-central1", - name: "lantern-foo-1", - target_vm_size: "n1-standard-2", - target_storage_size_gib: 100, - org_id: 0 - ) - - LanternResource[st.id] - end - - describe "unauthenticated" do - before do - Project.create_with_id(name: "default", provider: "gcp").tap { _1.associate_with_project(_1) } - end - - it "not location list" do - get "/api/project/#{project.ubid}/location/#{pg.location}/lantern" - - expect(last_response.status).to eq(401) - expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") - end - - it "not create" do - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/postgres_name" - - expect(last_response.status).to eq(401) - expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") - end - - it "not delete" do - delete "/api/project/#{project.ubid}/location/#{pg.location}/lantern/#{pg.name}" - - expect(last_response.status).to eq(401) - expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") - end - - it "not get" do - get "/api/project/#{project.ubid}/location/#{pg.location}/lantern/#{pg.name}" - - expect(last_response.status).to eq(401) - expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") - end - - it "not reset super user password" do - post "/api/project/#{project.ubid}/location/#{pg.location}/postgres/#{pg.name}/reset-superuser-password" - - expect(last_response.status).to eq(401) - expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") - end - - it "not update extension" do - post "/api/project/#{project.ubid}/location/#{pg.location}/postgres/#{pg.name}/update-extension" - - expect(last_response.status).to eq(401) - expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") - end - - it "not update image" do - post "/api/project/#{project.ubid}/location/#{pg.location}/postgres/#{pg.name}/update-image" - - expect(last_response.status).to eq(401) - expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") - end - - it "not add domain" do - post "/api/project/#{project.ubid}/location/#{pg.location}/postgres/#{pg.name}/add-domain" - - expect(last_response.status).to eq(401) - expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") - end - - it "not update rhizome" do - post "/api/project/#{project.ubid}/location/#{pg.location}/postgres/#{pg.name}/update-rhizome" - - expect(last_response.status).to eq(401) - expect(JSON.parse(last_response.body)["error"]).to eq("Please login to continue") - end - end - describe "authenticated" do before do login_api(user.email) @@ -110,215 +28,99 @@ end describe "list" do - it "empty" do - get "/api/project/#{project.ubid}/location/#{TEST_LOCATION}/lantern" + it "lists empty" do + get "/api/project/#{project.ubid}/location/#{pg.location}/lantern/#{pg.name}/doctor" expect(last_response.status).to eq(200) expect(JSON.parse(last_response.body)["items"]).to eq([]) end - it "success single" do - get "/api/project/#{project.ubid}/location/#{pg.location}/lantern" - - expect(last_response.status).to eq(200) - expect(JSON.parse(last_response.body)["items"].length).to eq(1) - end - - it "success multiple" do - Prog::Lantern::LanternResourceNexus.assemble( - project_id: project.id, - location: "us-central1", - name: "lantern-foo-2", - target_vm_size: "n1-standard-2", - target_storage_size_gib: 100, - org_id: 0 + it "lists queries" do + system_query = LanternDoctorQuery.create_with_id( + name: "test system query", + db_name: "postgres", + schedule: "*/30 * * * *", + condition: "unknown", + sql: "SELECT 1<2", + type: "system", + severity: "error" ) + pg.doctor.sync_system_queries + + get "/api/project/#{project.ubid}/location/#{pg.location}/lantern/#{pg.name}/doctor" + expect(last_response.status).to eq(200) + items = JSON.parse(last_response.body)["items"] + expect(items.size).to eq(1) + first_item = items[0] + expect(first_item["id"]).not_to be_nil + expect(first_item["name"]).to eq(system_query.name) + expect(first_item["db_name"]).to eq(system_query.db_name) + expect(first_item["schedule"]).to eq(system_query.schedule) + expect(first_item["type"]).to eq("user") + expect(first_item["severity"]).to eq(system_query.severity) + end + end + + describe "incidents" do + it "lists active incidents" do + system_query = LanternDoctorQuery.create_with_id( + name: "test system query", + db_name: "postgres", + schedule: "*/30 * * * *", + condition: "unknown", + sql: "SELECT 1<2", + type: "system", + severity: "error" + ) + pg.doctor.sync_system_queries + pg.doctor.queries + first_query = LanternDoctorQuery[doctor_id: pg.doctor.id] + first_query.update(condition: "failed") + summary = "Healthcheck: #{first_query.name} failed on #{pg.name} (postgres)" + Prog::PageNexus.assemble_with_logs(summary, [first_query.ubid, pg.doctor.ubid, "test"], {"stderr" => ""}, system_query.severity, "LanternDoctorQueryFailed", first_query.id, "postgres") + + get "/api/project/#{project.ubid}/location/#{pg.location}/lantern/#{pg.name}/doctor/incidents" + expect(last_response.status).to eq(200) + items = JSON.parse(last_response.body)["items"] + expect(items.size).to eq(1) + first_item = items[0] + expect(first_item["condition"]).to eq("failed") + incidents = first_item["incidents"] + expect(incidents.size).to eq(1) + expect(incidents[0]["summary"]).to eq(summary) + expect(incidents[0]["logs"]).to eq("") + end + + it "changes check time of query to run on next loop" do + LanternDoctorQuery.create_with_id( + name: "test system query", + db_name: "postgres", + schedule: "*/30 * * * *", + condition: "unknown", + sql: "SELECT 1<2", + type: "system", + severity: "error" + ) + pg.doctor.sync_system_queries + pg.doctor.queries + first_query = LanternDoctorQuery[doctor_id: pg.doctor.id] + t = Time.new + first_query.update(last_checked: t) - get "/api/project/#{project.ubid}/location/#{pg.location}/lantern" - - expect(last_response.status).to eq(200) - expect(JSON.parse(last_response.body)["items"].length).to eq(2) - end - end - - describe "#update-extension" do - it "updates lantern extension" do - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/update-extension", {lantern_version: "0.2.4", extras_version: Config.lantern_extras_default_version} - expect(last_response.status).to eq(200) - end - - it "updates extras extension" do - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/update-extension", {extras_version: "0.2.3", lantern_version: Config.lantern_default_version} - expect(last_response.status).to eq(200) - end - end - - describe "#update-image" do - it "updates image" do - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/update-image", {lantern_version: "0.2.3", extras_version: "0.2.3", minor_version: "1"} + post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/#{pg.name}/doctor/#{first_query.id}/run" + expect(last_response.status).to eq(204) + get "/api/project/#{project.ubid}/location/#{pg.location}/lantern/#{pg.name}/doctor" expect(last_response.status).to eq(200) + items = JSON.parse(last_response.body)["items"] + expect(items.size).to eq(1) + first_item = items[0] + expect(first_item["last_checked"]).to be_nil end - end - describe "get" do it "returns 404" do - get "/api/project/#{project.ubid}/location/#{pg.location}/lantern/test" + get "/api/project/#{project.ubid}/location/#{pg.location}/lantern/#{pg.name}/doctor/fbdad2ba-b61e-89b7-b7e5-d3414b94c541/run" expect(last_response.status).to eq(404) end - - it "returns instance" do - get "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1" - data = JSON.parse(last_response.body) - expect(data["name"]).to eq("instance-1") - expect(last_response.status).to eq(200) - end - end - - describe "delete" do - it "does not delete instance if has forks" do - expect(Project).to receive(:from_ubid).and_return(project).at_least(:once) - query_res = class_double(LanternResource, first: pg) - allow(query_res).to receive(:where).and_return(query_res) - expect(project).to receive(:lantern_resources_dataset).and_return(query_res) - expect(pg).to receive(:forks).and_return([instance_double(LanternResource)]) - - delete "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1" - expect(last_response.status).to eq(409) - expect(JSON.parse(last_response.body)).to eq({"error" => "Can not delete resource which has active forks"}) - end - - it "deletes instance" do - delete "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1" - expect(last_response.status).to eq(200) - end - end - - describe "start,stop,restart" do - it "starts instance" do - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/start" - expect(last_response.status).to eq(200) - end - - it "stops instance" do - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/stop" - expect(last_response.status).to eq(200) - end - - it "restarts instance" do - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/restart" - expect(last_response.status).to eq(200) - end - end - - describe "add-domain" do - it "adds domain" do - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/add-domain", {domain: "example.com"} - server = LanternServer.where(id: pg.representative_server.id).first - expect(server.domain).to eq("example.com") - expect(last_response.status).to eq(200) - end - end - - describe "reset-user-password" do - it "fails validation" do - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/reset-user-password", {original_password: "password123!", repeat_password: "test"} - expect(last_response.status).to eq(400) - end - - it "resets password" do - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/reset-user-password", {original_password: "Password123!", repeat_password: "Password123!"} - pg = LanternResource.first - expect(pg.db_user_password).to eq("Password123!") - expect(last_response.status).to eq(200) - end - end - - describe "update-vm" do - it "fails validation" do - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/update-vm", {storage_size_gib: 10} - expect(last_response.status).to eq(400) - end - - it "updates storage" do - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/update-vm", {storage_size_gib: 200} - server = LanternServer.where(id: pg.representative_server.id).first - expect(server.target_storage_size_gib).to eq(200) - expect(server.vm.storage_size_gib).to eq(200) - expect(last_response.status).to eq(200) - end - - it "updates vm size" do - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/update-vm", {size: "n1-standard-4"} - server = LanternServer.where(id: pg.representative_server.id).first - expect(server.target_vm_size).to eq("n1-standard-4") - expect(last_response.status).to eq(200) - end - end - - describe "backups" do - it "maps and prettifies backup keys correctly" do - time1 = Time.now - 20 * 60 - time2 = Time.now - 10 * 60 - ubid = LanternServer.where(id: pg.representative_server.id).first.timeline.ubid - backups = [{last_modified: time1, key: "#{ubid}/basebackups_005/1_backup_stop_sentinel.json"}, {last_modified: time2, key: "#{ubid}/basebackups_005/2_backup_stop_sentinel.json"}] - res_backups = JSON.parse(JSON.generate([{"time" => time1, "label" => "1", "compressed_size" => 10, "uncompressed_size" => 20}, {"time" => time2, "label" => "2", "compressed_size" => 10, "uncompressed_size" => 20}])) - gcp_client = instance_double(Hosting::GcpApis) - expect(gcp_client).to receive(:list_objects).and_return(backups) - expect(gcp_client).to receive(:get_json_object).and_return({"CompressedSize" => 10, "UncompressedSize" => 20}).at_least(:once) - allow(Hosting::GcpApis).to receive(:new).and_return(gcp_client) - - get "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/backups" - expect(last_response.status).to eq(200) - expect(JSON.parse(last_response.body)).to eq(res_backups) - end - end - - describe "push-backup" do - it "creates new backup" do - expect(Project).to receive(:from_ubid).and_return(project).at_least(:once) - query_res = class_double(LanternResource, first: pg) - allow(query_res).to receive(:where).and_return(query_res) - expect(project).to receive(:lantern_resources_dataset).and_return(query_res) - expect(pg.timeline).to receive(:take_manual_backup) - - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/push-backup" - expect(last_response.status).to eq(200) - end - - it "fails to create new backup" do - expect(Project).to receive(:from_ubid).and_return(project).at_least(:once) - query_res = class_double(LanternResource, first: pg) - allow(query_res).to receive(:where).and_return(query_res) - expect(project).to receive(:lantern_resources_dataset).and_return(query_res) - expect(pg.timeline).to receive(:take_manual_backup).and_raise "Another backup is in progress please try again later" - - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/push-backup" - expect(last_response.status).to eq(409) - expect(JSON.parse(last_response.body)).to eq({"error" => "Another backup is in progress please try again later"}) - end - - it "fails to create new backup with unknown" do - expect(Project).to receive(:from_ubid).and_return(project).at_least(:once) - query_res = class_double(LanternResource, first: pg) - allow(query_res).to receive(:where).and_return(query_res) - expect(project).to receive(:lantern_resources_dataset).and_return(query_res) - expect(pg.timeline).to receive(:take_manual_backup).and_raise "Unknown error" - - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/push-backup" - expect(last_response.status).to eq(400) - end - end - - describe "dissociate-forks" do - it "dissociate-forkses" do - expect(Project).to receive(:from_ubid).and_return(project).at_least(:once) - query_res = class_double(LanternResource, first: pg) - allow(query_res).to receive(:where).and_return(query_res) - expect(project).to receive(:lantern_resources_dataset).and_return(query_res) - expect(pg).to receive(:dissociate_forks) - post "/api/project/#{project.ubid}/location/#{pg.location}/lantern/instance-1/dissociate-forks" - expect(last_response.status).to eq(200) - end end end end From 9b047f640ed001b17b0a82108112e41f07b47369 Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Mon, 6 May 2024 13:10:50 -0700 Subject: [PATCH 06/13] Fix destroy for lantern doctor and queries --- prog/lantern/lantern_doctor_nexus.rb | 4 ++++ prog/lantern/lantern_resource_nexus.rb | 1 + spec/prog/lantern/lantern_doctor_nexus_spec.rb | 7 +++++++ 3 files changed, 12 insertions(+) diff --git a/prog/lantern/lantern_doctor_nexus.rb b/prog/lantern/lantern_doctor_nexus.rb index 4f608d552..7831af522 100644 --- a/prog/lantern/lantern_doctor_nexus.rb +++ b/prog/lantern/lantern_doctor_nexus.rb @@ -36,6 +36,10 @@ def before_run end label def wait + if lantern_doctor.resource.nil? + hop_destroy + end + when_sync_system_queries_set? do hop_sync_system_queries end diff --git a/prog/lantern/lantern_resource_nexus.rb b/prog/lantern/lantern_resource_nexus.rb index 47635f034..1d0c47ee8 100644 --- a/prog/lantern/lantern_resource_nexus.rb +++ b/prog/lantern/lantern_resource_nexus.rb @@ -165,6 +165,7 @@ def before_run nap 5 end + lantern_resource.doctor.incr_destroy lantern_resource.dissociate_with_project(lantern_resource.project) lantern_resource.destroy diff --git a/spec/prog/lantern/lantern_doctor_nexus_spec.rb b/spec/prog/lantern/lantern_doctor_nexus_spec.rb index 2bf4f06ad..d3cd0db68 100644 --- a/spec/prog/lantern/lantern_doctor_nexus_spec.rb +++ b/spec/prog/lantern/lantern_doctor_nexus_spec.rb @@ -55,12 +55,19 @@ end describe "#wait" do + it "hops to destroy" do + expect(lantern_doctor).to receive(:resource).and_return(nil) + expect { nx.wait }.to hop("destroy") + end + it "syncs system queries" do + expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, strand: nil)) expect(nx).to receive(:when_sync_system_queries_set?).and_yield expect { nx.wait }.to hop("sync_system_queries") end it "runs queries and naps" do + expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, strand: nil)) queries = [instance_double(LanternDoctorQuery)] expect(queries[0]).to receive(:run) expect(lantern_doctor).to receive(:queries).and_return(queries) From a2e0fc8bbdf55ca0ca565e9cff9dd14e29792c55 Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Mon, 6 May 2024 13:25:28 -0700 Subject: [PATCH 07/13] Resolve all active pages of lantern doctor on destroy --- prog/lantern/lantern_doctor_nexus.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/prog/lantern/lantern_doctor_nexus.rb b/prog/lantern/lantern_doctor_nexus.rb index 7831af522..485443145 100644 --- a/prog/lantern/lantern_doctor_nexus.rb +++ b/prog/lantern/lantern_doctor_nexus.rb @@ -56,6 +56,11 @@ def before_run label def destroy decr_destroy + + lantern_doctor.failed_queries.each { + _1.active_pages.each {|pg| pg.incr_resolve } + } + lantern_doctor.destroy pop "lantern doctor is deleted" end From c22a2dc37e3abde66e4007b869a114e7df30656a Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Mon, 6 May 2024 13:29:08 -0700 Subject: [PATCH 08/13] run rubocop --- prog/lantern/lantern_doctor_nexus.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prog/lantern/lantern_doctor_nexus.rb b/prog/lantern/lantern_doctor_nexus.rb index 485443145..9251c2ba2 100644 --- a/prog/lantern/lantern_doctor_nexus.rb +++ b/prog/lantern/lantern_doctor_nexus.rb @@ -58,7 +58,7 @@ def before_run decr_destroy lantern_doctor.failed_queries.each { - _1.active_pages.each {|pg| pg.incr_resolve } + _1.active_pages.each { |pg| pg.incr_resolve } } lantern_doctor.destroy From 81a887a021e5ed1a2d2a8cb6ca8932ad6d434dae Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Mon, 6 May 2024 13:49:17 -0700 Subject: [PATCH 09/13] fix destroy test for lantern_doctor_nexus --- spec/prog/lantern/lantern_doctor_nexus_spec.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spec/prog/lantern/lantern_doctor_nexus_spec.rb b/spec/prog/lantern/lantern_doctor_nexus_spec.rb index d3cd0db68..9b3898d87 100644 --- a/spec/prog/lantern/lantern_doctor_nexus_spec.rb +++ b/spec/prog/lantern/lantern_doctor_nexus_spec.rb @@ -103,6 +103,10 @@ describe "#destroy" do it "exits with message" do expect(nx).to receive(:decr_destroy) + page = instance_double(Page) + query = instance_double(LanternDoctorQuery, active_pages: [page]) + expect(page).to receive(:incr_resolve) + expect(lantern_doctor).to receive(:failed_queries).and_return([query]) expect(lantern_doctor).to receive(:destroy) expect { nx.destroy }.to exit({"msg" => "lantern doctor is deleted"}) end From 536525d7eb49319a18002ff2dc5af9efbdacd347 Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Mon, 6 May 2024 14:58:34 -0700 Subject: [PATCH 10/13] Do not run doctor queries if database is not in waiting state --- misc/misc_operations.rb | 7 ++++ prog/lantern/lantern_doctor_nexus.rb | 9 ++++- prog/lantern/lantern_resource_nexus.rb | 2 +- .../prog/lantern/lantern_doctor_nexus_spec.rb | 34 +++++++++++++++++++ .../lantern/lantern_resource_nexus_spec.rb | 15 ++++++++ 5 files changed, 65 insertions(+), 2 deletions(-) diff --git a/misc/misc_operations.rb b/misc/misc_operations.rb index 913e03590..f999ac7e5 100644 --- a/misc/misc_operations.rb +++ b/misc/misc_operations.rb @@ -145,4 +145,11 @@ def self.get_all_lantern_indexes(resource_name) end all_indexes end + + def self.add_lantern_doctor_to_all + LanternResource.all.each { + lantern_doctor = Prog::Lantern::LanternDoctorNexus.assemble + _1.update(doctor_id: lantern_doctor.id) + } + end end diff --git a/prog/lantern/lantern_doctor_nexus.rb b/prog/lantern/lantern_doctor_nexus.rb index 9251c2ba2..9c7560525 100644 --- a/prog/lantern/lantern_doctor_nexus.rb +++ b/prog/lantern/lantern_doctor_nexus.rb @@ -25,6 +25,10 @@ def before_run end end + def should_run? + lantern_doctor.resource&.representative_server&.strand&.label == "wait" + end + label def start lantern_doctor.sync_system_queries hop_wait_resource @@ -44,7 +48,10 @@ def before_run hop_sync_system_queries end - lantern_doctor.queries.each { _1.run } + if should_run? + lantern_doctor.queries.each { _1.run } + end + nap 60 end diff --git a/prog/lantern/lantern_resource_nexus.rb b/prog/lantern/lantern_resource_nexus.rb index 1d0c47ee8..4e36d824e 100644 --- a/prog/lantern/lantern_resource_nexus.rb +++ b/prog/lantern/lantern_resource_nexus.rb @@ -165,7 +165,7 @@ def before_run nap 5 end - lantern_resource.doctor.incr_destroy + lantern_resource.doctor&.incr_destroy lantern_resource.dissociate_with_project(lantern_resource.project) lantern_resource.destroy diff --git a/spec/prog/lantern/lantern_doctor_nexus_spec.rb b/spec/prog/lantern/lantern_doctor_nexus_spec.rb index 9b3898d87..da719edb9 100644 --- a/spec/prog/lantern/lantern_doctor_nexus_spec.rb +++ b/spec/prog/lantern/lantern_doctor_nexus_spec.rb @@ -55,6 +55,12 @@ end describe "#wait" do + it "naps" do + expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, strand: nil)) + expect(nx).to receive(:should_run?).and_return(false) + expect { nx.wait }.to nap(60) + end + it "hops to destroy" do expect(lantern_doctor).to receive(:resource).and_return(nil) expect { nx.wait }.to hop("destroy") @@ -67,6 +73,7 @@ end it "runs queries and naps" do + expect(nx).to receive(:should_run?).and_return(true) expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, strand: nil)) queries = [instance_double(LanternDoctorQuery)] expect(queries[0]).to receive(:run) @@ -111,4 +118,31 @@ expect { nx.destroy }.to exit({"msg" => "lantern doctor is deleted"}) end end + + describe "#should_run" do + it "returns true" do + expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, representative_server: instance_double(LanternServer, strand: instance_double(Strand, label: "wait")))) + expect(nx.should_run?).to be(true) + end + + it "returns false" do + expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, representative_server: instance_double(LanternServer, strand: instance_double(Strand, label: "start")))) + expect(nx.should_run?).to be(false) + end + + it "returns false if no resource" do + expect(lantern_doctor).to receive(:resource).and_return(nil) + expect(nx.should_run?).to be(false) + end + + it "returns false if no server" do + expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, representative_server: nil)) + expect(nx.should_run?).to be(false) + end + + it "returns false if no strand" do + expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, representative_server: instance_double(LanternServer, strand: nil))) + expect(nx.should_run?).to be(false) + end + end end diff --git a/spec/prog/lantern/lantern_resource_nexus_spec.rb b/spec/prog/lantern/lantern_resource_nexus_spec.rb index 038e40105..e756a206b 100644 --- a/spec/prog/lantern/lantern_resource_nexus_spec.rb +++ b/spec/prog/lantern/lantern_resource_nexus_spec.rb @@ -193,6 +193,21 @@ expect(lantern_resource).to receive(:servers).and_return([]) expect(lantern_resource).to receive(:dissociate_with_project) expect(lantern_resource).to receive(:destroy) + expect(lantern_resource).to receive(:doctor).and_return(nil) + + expect { nx.destroy }.to exit({"msg" => "lantern resource is deleted"}) + end + + it "triggers server deletion and deletes doctor" do + expect(lantern_resource.servers).to all(receive(:incr_destroy)) + expect { nx.destroy }.to nap(5) + + expect(lantern_resource).to receive(:servers).and_return([]) + expect(lantern_resource).to receive(:dissociate_with_project) + expect(lantern_resource).to receive(:destroy) + doctor = instance_double(LanternDoctor) + expect(lantern_resource).to receive(:doctor).and_return(doctor) + expect(doctor).to receive(:incr_destroy) expect { nx.destroy }.to exit({"msg" => "lantern resource is deleted"}) end From 0e1650a0cfae89464a751cd73115ff5149245be4 Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Mon, 6 May 2024 15:11:25 -0700 Subject: [PATCH 11/13] Do not run doctor queries if database display_state is not running --- model/lantern/lantern_doctor.rb | 4 +++ prog/lantern/lantern_doctor_nexus.rb | 6 +--- spec/model/lantern/lantern_doctor_spec.rb | 32 +++++++++++++++++++ .../prog/lantern/lantern_doctor_nexus_spec.rb | 31 ++---------------- 4 files changed, 39 insertions(+), 34 deletions(-) diff --git a/model/lantern/lantern_doctor.rb b/model/lantern/lantern_doctor.rb index ca25b5666..e01289f37 100644 --- a/model/lantern/lantern_doctor.rb +++ b/model/lantern/lantern_doctor.rb @@ -23,6 +23,10 @@ def has_system_query?(queries, query) queries.any? { _1.parent_id == query.id } end + def should_run? + resource&.representative_server&.display_state == "running" && resource&.representative_server&.strand&.label == "wait" + end + def sync_system_queries doctor_query_list = queries system_query_list = system_queries diff --git a/prog/lantern/lantern_doctor_nexus.rb b/prog/lantern/lantern_doctor_nexus.rb index 9c7560525..0a93c4988 100644 --- a/prog/lantern/lantern_doctor_nexus.rb +++ b/prog/lantern/lantern_doctor_nexus.rb @@ -25,10 +25,6 @@ def before_run end end - def should_run? - lantern_doctor.resource&.representative_server&.strand&.label == "wait" - end - label def start lantern_doctor.sync_system_queries hop_wait_resource @@ -48,7 +44,7 @@ def should_run? hop_sync_system_queries end - if should_run? + if lantern_doctor.should_run? lantern_doctor.queries.each { _1.run } end diff --git a/spec/model/lantern/lantern_doctor_spec.rb b/spec/model/lantern/lantern_doctor_spec.rb index 584d628bb..a5f53e53f 100644 --- a/spec/model/lantern/lantern_doctor_spec.rb +++ b/spec/model/lantern/lantern_doctor_spec.rb @@ -46,4 +46,36 @@ expect { lantern_doctor.sync_system_queries }.not_to raise_error end end + + describe "#should_run" do + it "returns true" do + expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, representative_server: instance_double(LanternServer, display_state: "running", strand: instance_double(Strand, label: "wait")))).at_least(:once) + expect(lantern_doctor.should_run?).to be(true) + end + + it "returns false if not running" do + expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, representative_server: instance_double(LanternServer, display_state: "stopped", strand: instance_double(Strand, label: "start")))).at_least(:once) + expect(lantern_doctor.should_run?).to be(false) + end + + it "returns false" do + expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, representative_server: instance_double(LanternServer, display_state: "running", strand: instance_double(Strand, label: "start")))).at_least(:once) + expect(lantern_doctor.should_run?).to be(false) + end + + it "returns false if no resource" do + expect(lantern_doctor).to receive(:resource).and_return(nil) + expect(lantern_doctor.should_run?).to be(false) + end + + it "returns false if no server" do + expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, representative_server: nil)).at_least(:once) + expect(lantern_doctor.should_run?).to be(false) + end + + it "returns false if no strand" do + expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, representative_server: instance_double(LanternServer, display_state: "running", strand: nil))).at_least(:once) + expect(lantern_doctor.should_run?).to be(false) + end + end end diff --git a/spec/prog/lantern/lantern_doctor_nexus_spec.rb b/spec/prog/lantern/lantern_doctor_nexus_spec.rb index da719edb9..70bf3cdb6 100644 --- a/spec/prog/lantern/lantern_doctor_nexus_spec.rb +++ b/spec/prog/lantern/lantern_doctor_nexus_spec.rb @@ -57,7 +57,7 @@ describe "#wait" do it "naps" do expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, strand: nil)) - expect(nx).to receive(:should_run?).and_return(false) + expect(lantern_doctor).to receive(:should_run?).and_return(false) expect { nx.wait }.to nap(60) end @@ -73,7 +73,7 @@ end it "runs queries and naps" do - expect(nx).to receive(:should_run?).and_return(true) + expect(lantern_doctor).to receive(:should_run?).and_return(true) expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, strand: nil)) queries = [instance_double(LanternDoctorQuery)] expect(queries[0]).to receive(:run) @@ -118,31 +118,4 @@ expect { nx.destroy }.to exit({"msg" => "lantern doctor is deleted"}) end end - - describe "#should_run" do - it "returns true" do - expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, representative_server: instance_double(LanternServer, strand: instance_double(Strand, label: "wait")))) - expect(nx.should_run?).to be(true) - end - - it "returns false" do - expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, representative_server: instance_double(LanternServer, strand: instance_double(Strand, label: "start")))) - expect(nx.should_run?).to be(false) - end - - it "returns false if no resource" do - expect(lantern_doctor).to receive(:resource).and_return(nil) - expect(nx.should_run?).to be(false) - end - - it "returns false if no server" do - expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, representative_server: nil)) - expect(nx.should_run?).to be(false) - end - - it "returns false if no strand" do - expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, representative_server: instance_double(LanternServer, strand: nil))) - expect(nx.should_run?).to be(false) - end - end end From 0b590071ac1e27c3e013847ce112c18fd6f79a76 Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Mon, 6 May 2024 15:22:52 -0700 Subject: [PATCH 12/13] remove optional chaining from should_run? in lantern_doctor --- model/lantern/lantern_doctor.rb | 3 ++- spec/model/lantern/lantern_doctor_spec.rb | 10 ---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/model/lantern/lantern_doctor.rb b/model/lantern/lantern_doctor.rb index e01289f37..4b05eed9f 100644 --- a/model/lantern/lantern_doctor.rb +++ b/model/lantern/lantern_doctor.rb @@ -24,7 +24,8 @@ def has_system_query?(queries, query) end def should_run? - resource&.representative_server&.display_state == "running" && resource&.representative_server&.strand&.label == "wait" + return false unless resource + resource.representative_server.display_state == "running" && resource.representative_server.strand.label == "wait" end def sync_system_queries diff --git a/spec/model/lantern/lantern_doctor_spec.rb b/spec/model/lantern/lantern_doctor_spec.rb index a5f53e53f..5db44dfcc 100644 --- a/spec/model/lantern/lantern_doctor_spec.rb +++ b/spec/model/lantern/lantern_doctor_spec.rb @@ -67,15 +67,5 @@ expect(lantern_doctor).to receive(:resource).and_return(nil) expect(lantern_doctor.should_run?).to be(false) end - - it "returns false if no server" do - expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, representative_server: nil)).at_least(:once) - expect(lantern_doctor.should_run?).to be(false) - end - - it "returns false if no strand" do - expect(lantern_doctor).to receive(:resource).and_return(instance_double(LanternResource, representative_server: instance_double(LanternServer, display_state: "running", strand: nil))).at_least(:once) - expect(lantern_doctor.should_run?).to be(false) - end end end From 226f4ace60925ccb7cceb569b5f0993a7797d74c Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Mon, 6 May 2024 15:34:58 -0700 Subject: [PATCH 13/13] fix reindex query --- misc/misc_operations.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/misc_operations.rb b/misc/misc_operations.rb index f999ac7e5..a4fc09f0d 100644 --- a/misc/misc_operations.rb +++ b/misc/misc_operations.rb @@ -111,7 +111,7 @@ def self.reindex_all_concurrently(resource_name, disable_indexes: false) schema, idx = _1.split(".") queries.push("REINDEX INDEX CONCURRENTLY \\\"#{schema}\\\".\\\"#{idx}\\\";") if disable_indexes - queries.push("UPDATE pg_index SET indisvalid = false, indisready = false WHERE indexrelid = quote_ident('#{_1}')::regclass::oid;") + queries.push("UPDATE pg_index SET indisvalid = false, indisready = false WHERE indexrelid = quote_ident('#{idx}')::regclass::oid;") end }