Automatically detect custom image for lantern

config.rb

@@ -141,6 +141,7 @@ def self.e2e_test?
   optional :gcp_creds_coredumps_b64, string
   optional :gcp_creds_walg_b64, string
   optional :prom_password, string
+  override :gcp_default_image, "projects/ubuntu-os-cloud/global/images/ubuntu-2204-jammy-v20240319", string
   override :gcr_image, "gcr.io/ringed-griffin-394922/lantern-bitnami"
 
   # Lantern
@@ -154,12 +155,6 @@ def self.e2e_test?
   override :backup_retention_days, 7, int
   optional :lantern_backend_database_url, string
 
-  # To use default ubuntu image pass these env vars to overwrite
-  # LANTERN_GCP_IMAGE=projects/ubuntu-os-cloud/global/images/ubuntu-2204-jammy-v20240319
-  # LANTERN_GCP_IMAGE_CACHED=false
-  override :lantern_gcp_image, "projects/#{gcp_project_id}/global/images/ubuntu-lantern-#{lantern_default_version.tr(".", "-")}-extras-#{lantern_extras_default_version.tr(".", "-")}-minor-#{lantern_minor_default_version}", string
-  override :lantern_gcp_image_cached, true, bool
-
   # Cloudflare
   optional :cf_token, string
   optional :cf_zone_id, string

lib/hosting/gcp_apis.rb

@@ -344,9 +344,21 @@ def create_image(name:, vm_name:, zone:, description: "", family: "lantern-ubunt
       ]
     }
 
-    response = connection.post(path: "/compute/beta/projects/#{@project}/global/images", body: JSON.dump(body), expects: 200)
+    response = connection.post(path: "/compute/v1/projects/#{@project}/global/images", body: JSON.dump(body), expects: 200)
     Hosting::GcpApis.check_errors(response)
     data = JSON.parse(response.body)
     wait_for_operation("global", data["id"])
   end
+
+  def get_image(name)
+    connection = Excon.new(@host[:connection_string], headers: @host[:headers])
+    response = connection.get(path: "/compute/v1/projects/#{@project}/global/images/#{name}")
+
+    if response.status == 404
+      return nil
+    end
+
+    Hosting::GcpApis.check_errors(response)
+    JSON.parse(response.body).merge({"resource_name" => "projects/#{@project}/global/images/#{name}"})
+  end
 end

misc/misc_operations.rb

@@ -162,9 +162,10 @@ def self.add_lantern_doctor_to_all
     }
   end
 
-  def self.create_image(name, description, image_tag, vm: nil)
+  def self.create_image(lantern_version: "0.2.7", extras_version: "0.1.5", minor_version: "1", vm: nil)
     gcp_api = Hosting::GcpApis.new
-    container_image = "#{Config.gcr_image}:#{image_tag}"
+    name = "ubuntu-lantern-#{lantern_version.tr(".", "-")}-extras-#{extras_version.tr(".", "-")}-minor-#{minor_version}"
+    container_image = "#{Config.gcr_image}:lantern-#{lantern_version}-extras-#{extras_version}-minor-#{minor_version}"
 
     if vm.nil?
       vm = Prog::GcpVm::Nexus.assemble_with_sshable("lantern", Project.first.id, name: "imagecreation-machine", storage_size_gib: 10)

model/lantern/lantern_server.rb

@@ -101,7 +101,7 @@ def configure_hash
 
     JSON.generate({
       enable_coredumps: true,
-      skip_deps: Config.lantern_gcp_image_cached,
+      skip_deps: vm.boot_image != Config.gcp_default_image,
       org_id: resource.org_id,
       instance_id: resource.name,
       instance_type: instance_type,
@@ -152,6 +152,14 @@ def init_health_monitor_session
     }
   end
 
+  def self.get_vm_image(lantern_version, extras_version, minor_version)
+    api = Hosting::GcpApis.new
+    image_name = "ubuntu-lantern-#{lantern_version.tr(".", "-")}-extras-#{extras_version.tr(".", "-")}-minor-#{minor_version}"
+    image = api.get_image(image_name)
+
+    image.nil? ? Config.gcp_default_image : image["resource_name"]
+  end
+
   def check_pulse(session:, previous_pulse:)
     if destroy_set? || strand&.label != "wait" || display_state != "running"
       # if there's an operation ongoing, do not check the pulse

prog/gcp_vm/nexus.rb

@@ -18,7 +18,7 @@ class Prog::GcpVm::Nexus < Prog::Base
   semaphore :destroy, :start_vm, :stop_vm, :update_storage, :update_size
 
   def self.assemble(public_key, project_id, name: nil, size: "n1-standard-2",
-    unix_user: "lantern", location: "us-central1", boot_image: Config.lantern_gcp_image,
+    unix_user: "lantern", location: "us-central1", boot_image: Config.gcp_default_image,
     storage_size_gib: nil, arch: "x64", labels: {})
 
     unless (project = Project[project_id])

prog/lantern/lantern_server_nexus.rb

@@ -33,7 +33,7 @@ def self.assemble(
         location: resource.location,
         size: target_vm_size,
         storage_size_gib: target_storage_size_gib,
-        boot_image: Config.lantern_gcp_image,
+        boot_image: LanternServer.get_vm_image(lantern_version, extras_version, minor_version),
         labels: {"parent" => resource.name}
       )
 

spec/lib/hosting/gcp_apis_spec.rb

@@ -328,7 +328,7 @@
     describe "#create_image" do
       it "cerates image" do
         stub_request(:post, "https://oauth2.googleapis.com/token").to_return(status: 200, body: JSON.dump({}), headers: {"Content-Type" => "application/json"})
-        stub_request(:post, "https://compute.googleapis.com/compute/beta/projects/test-project/global/images")
+        stub_request(:post, "https://compute.googleapis.com/compute/v1/projects/test-project/global/images")
           .with(
             body: "{\"kind\":\"compute#image\",\"description\":\"test-desc\",\"name\":\"test-name\",\"family\":\"lantern-ubuntu\",\"sourceDisk\":\"projects/test-project/zones/us-central1-a/disks/inst-name\",\"storageLocations\":[\"us\"]}"
           )
@@ -338,5 +338,23 @@
         expect { api.create_image(name: "test-name", vm_name: "inst-name", zone: "us-central1-a", description: "test-desc") }.not_to raise_error
       end
     end
+
+    describe "#get_image" do
+      it "gets image" do
+        stub_request(:post, "https://oauth2.googleapis.com/token").to_return(status: 200, body: JSON.dump({}), headers: {"Content-Type" => "application/json"})
+        stub_request(:get, "https://compute.googleapis.com/compute/v1/projects/test-project/global/images/test-image")
+          .to_return(status: 200, body: JSON.dump({}), headers: {})
+        api = described_class.new
+        expect(api.get_image("test-image")).to eq({"resource_name" => "projects/test-project/global/images/test-image"})
+      end
+
+      it "gets nil" do
+        stub_request(:post, "https://oauth2.googleapis.com/token").to_return(status: 200, body: JSON.dump({}), headers: {"Content-Type" => "application/json"})
+        stub_request(:get, "https://compute.googleapis.com/compute/v1/projects/test-project/global/images/test-image")
+          .to_return(status: 404, body: JSON.dump({}), headers: {})
+        api = described_class.new
+        expect(api.get_image("test-image")).to be_nil
+      end
+    end
   end
 end

spec/lib/pagination_spec.rb

@@ -3,6 +3,10 @@
 require_relative "../model/spec_helper"
 
 RSpec.describe Pagination do
+  before do
+    allow(LanternServer).to receive(:get_vm_image).and_return(Config.gcp_default_image)
+  end
+
   let(:project) { Project.create_with_id(name: "default").tap { _1.associate_with_project(_1) } }
 
   let!(:first_server) do

spec/model/lantern/lantern_doctor_spec.rb

@@ -9,6 +9,10 @@
     end
   }
 
+  before do
+    allow(LanternServer).to receive(:get_vm_image).and_return(Config.gcp_default_image)
+  end
+
   describe "#system_queries" do
     it "returns cached queries" do
       expect(lantern_doctor).to receive(:system_queries).and_return([instance_double(LanternDoctorQuery)])

spec/model/lantern/lantern_server_spec.rb

@@ -7,6 +7,11 @@
     described_class.new { _1.id = "c068cac7-ed45-82db-bf38-a003582b36ee" }
   }
 
+  before do
+    allow(described_class).to receive(:get_vm_image).and_return(Config.gcp_default_image)
+    allow(lantern_server).to receive(:gcp_vm).and_return(vm)
+  end
+
   let(:vm) {
     instance_double(
       GcpVm,
@@ -15,10 +20,6 @@
     )
   }
 
-  before do
-    allow(lantern_server).to receive(:gcp_vm).and_return(vm)
-  end
-
   describe "#instance_type" do
     it "returns reader" do
       expect(lantern_server).to receive(:standby?).and_return(true)
@@ -240,7 +241,6 @@
         db_user_password: "pwd123",
         superuser_password: "pwd1234",
         restore_target: nil)
-      expect(Config).to receive(:lantern_gcp_image_cached).and_return(false).at_least(:once)
       expect(Config).to receive(:prom_password).and_return("pwd123").at_least(:once)
       expect(Config).to receive(:gcp_creds_gcr_b64).and_return("test-creds").at_least(:once)
       expect(Config).to receive(:gcp_creds_logging_b64).and_return("test-creds").at_least(:once)
@@ -251,6 +251,7 @@
       expect(lantern_server).to receive(:lantern_version).and_return("0.2.2").at_least(:once)
       expect(lantern_server).to receive(:extras_version).and_return("0.1.4").at_least(:once)
       expect(lantern_server).to receive(:minor_version).and_return("1").at_least(:once)
+      expect(vm).to receive(:boot_image).and_return(Config.gcp_default_image).at_least(:once)
 
       walg_conf = timeline.generate_walg_config
       expected_conf = JSON.generate({
@@ -311,6 +312,7 @@
       expect(lantern_server).to receive(:lantern_version).and_return("0.2.2").at_least(:once)
       expect(lantern_server).to receive(:extras_version).and_return("0.1.4").at_least(:once)
       expect(lantern_server).to receive(:minor_version).and_return("1").at_least(:once)
+      expect(vm).to receive(:boot_image).and_return("custom-image").at_least(:once)
 
       walg_conf = timeline.generate_walg_config
       expected_conf = JSON.generate({
@@ -369,6 +371,7 @@
       expect(lantern_server).to receive(:lantern_version).and_return("0.2.2").at_least(:once)
       expect(lantern_server).to receive(:extras_version).and_return("0.1.4").at_least(:once)
       expect(lantern_server).to receive(:minor_version).and_return("1").at_least(:once)
+      expect(vm).to receive(:boot_image).and_return("custom-image").at_least(:once)
 
       walg_conf = timeline.generate_walg_config
       expected_conf = JSON.generate({
@@ -428,6 +431,7 @@
       expect(lantern_server).to receive(:lantern_version).and_return("0.2.2").at_least(:once)
       expect(lantern_server).to receive(:extras_version).and_return("0.1.4").at_least(:once)
       expect(lantern_server).to receive(:minor_version).and_return("1").at_least(:once)
+      expect(vm).to receive(:boot_image).and_return("custom-image").at_least(:once)
 
       walg_conf = timeline.generate_walg_config
       expected_conf = JSON.generate({
@@ -629,4 +633,22 @@
       expect(lantern_server.list_all_databases).to eq(["postgres", "db2"])
     end
   end
+
+  describe "#get_vm_image" do
+    it "returns default image" do
+      allow(described_class).to receive(:get_vm_image).and_call_original
+      api = instance_double(Hosting::GcpApis)
+      expect(Hosting::GcpApis).to receive(:new).and_return(api)
+      expect(api).to receive(:get_image).and_return(nil)
+      expect(described_class.get_vm_image("0.2.7", "0.1.5", "1")).to eq(Config.gcp_default_image)
+    end
+
+    it "returns custom image" do
+      allow(described_class).to receive(:get_vm_image).and_call_original
+      api = instance_double(Hosting::GcpApis)
+      expect(Hosting::GcpApis).to receive(:new).and_return(api)
+      expect(api).to receive(:get_image).and_return({"resource_name" => "custom-image"})
+      expect(described_class.get_vm_image("0.2.7", "0.1.5", "1")).to eq("custom-image")
+    end
+  end
 end

spec/prog/gcp_vm/nexus_spec.rb

@@ -12,7 +12,7 @@
 
   let(:st) { Strand.new }
   let(:gcp_vm) {
-    vm = GcpVm.new_with_id(family: "standard", cores: 1, name: "dummy-vm", arch: "x64", location: "us-central1", storage_size_gib: 50, boot_image: Config.lantern_gcp_image)
+    vm = GcpVm.new_with_id(family: "standard", cores: 1, name: "dummy-vm", arch: "x64", location: "us-central1", storage_size_gib: 50, boot_image: Config.gcp_default_image)
     vm
   }
   let(:prj) { Project.create_with_id(name: "default", provider: "gcp").tap { _1.associate_with_project(_1) } }

spec/prog/lantern/lantern_resource_nexus_spec.rb

@@ -30,6 +30,7 @@
   }
 
   before do
+    allow(LanternServer).to receive(:get_vm_image).and_return(Config.gcp_default_image)
     allow(nx).to receive(:lantern_resource).and_return(lantern_resource)
   end
 

spec/prog/lantern/lantern_server_nexus_spec.rb

@@ -37,6 +37,7 @@
   }
 
   before do
+    allow(LanternServer).to receive(:get_vm_image).and_return(Config.gcp_default_image)
     allow(nx).to receive(:lantern_server).and_return(lantern_server)
   end
 

spec/routes/api/project/lantern_spec.rb

@@ -4,9 +4,6 @@
 
 RSpec.describe Clover, "lantern" do
   let(:user) { create_account }
-
-  let(:project) { user.create_project_with_default_policy("project-1", provider: "gcp") }
-
   let(:pg) do
     st = Prog::Lantern::LanternResourceNexus.assemble(
       project_id: project.id,
@@ -19,6 +16,12 @@
     LanternResource[st.id]
   end
 
+  let(:project) { user.create_project_with_default_policy("project-1", provider: "gcp") }
+
+  before do
+    allow(LanternServer).to receive(:get_vm_image).and_return(Config.gcp_default_image)
+  end
+
   describe "unauthenticated" do
     it "not list" do
       get "/api/project/#{project.ubid}/lantern"

spec/routes/api/project/location/lantern_doctor_spec.rb

@@ -21,6 +21,10 @@
     LanternResource[st.id]
   end
 
+  before do
+    allow(LanternServer).to receive(:get_vm_image).and_return(Config.gcp_default_image)
+  end
+
   describe "authenticated" do
     before do
       login_api(user.email)

spec/routes/api/project/location/lantern_spec.rb

@@ -4,11 +4,6 @@
 
 RSpec.describe Clover, "lantern" do
   let(:user) { create_account }
-
-  let(:project) { user.create_project_with_default_policy("project-1", provider: "gcp") }
-
-  let(:project_wo_permissions) { user.create_project_with_default_policy("project-2", provider: "gcp", policy_body: []) }
-
   let(:pg) do
     st = Prog::Lantern::LanternResourceNexus.assemble(
       project_id: project.id,
@@ -20,7 +15,6 @@
     )
     LanternResource[st.id]
   end
-
   let(:pg_wo_pwermission) do
     st = Prog::Lantern::LanternResourceNexus.assemble(
       project_id: project_wo_permissions.id,
@@ -34,6 +28,14 @@
     LanternResource[st.id]
   end
 
+  let(:project) { user.create_project_with_default_policy("project-1", provider: "gcp") }
+
+  let(:project_wo_permissions) { user.create_project_with_default_policy("project-2", provider: "gcp", policy_body: []) }
+
+  before do
+    allow(LanternServer).to receive(:get_vm_image).and_return(Config.gcp_default_image)
+  end
+
   describe "unauthenticated" do
     before do
       Project.create_with_id(name: "default", provider: "gcp").tap { _1.associate_with_project(_1) }

spec/routes/api/project_spec.rb

@@ -7,6 +7,10 @@
 
   let(:project) { user.create_project_with_default_policy("project-1") }
 
+  before do
+    allow(LanternServer).to receive(:get_vm_image).and_return(Config.gcp_default_image)
+  end
+
   describe "unauthenticated" do
     it "not list" do
       get "/api/project"

spec/routes/web/project/lantern_doctor_spec.rb

@@ -4,9 +4,6 @@
 
 RSpec.describe Clover, "lantern" do
   let(:user) { create_account }
-  let(:project) { user.create_project_with_default_policy("project-1", provider: "gcp") }
-  let(:project_wo_permissions) { user.create_project_with_default_policy("project-2", policy_body: [], provider: "gcp") }
-
   let(:pg) do
     st = Prog::Lantern::LanternResourceNexus.assemble(
       project_id: project.id,
@@ -18,6 +15,12 @@
     )
     LanternResource[st.id]
   end
+  let(:project) { user.create_project_with_default_policy("project-1", provider: "gcp") }
+  let(:project_wo_permissions) { user.create_project_with_default_policy("project-2", policy_body: [], provider: "gcp") }
+
+  before do
+    allow(LanternServer).to receive(:get_vm_image).and_return(Config.gcp_default_image)
+  end
 
   describe "authenticated" do
     before do

spec/routes/web/project/lantern_spec.rb

@@ -4,9 +4,6 @@
 
 RSpec.describe Clover, "lantern" do
   let(:user) { create_account }
-  let(:project) { user.create_project_with_default_policy("project-1", provider: "gcp") }
-  let(:project_wo_permissions) { user.create_project_with_default_policy("project-2", policy_body: [], provider: "gcp") }
-
   let(:pg) do
     st = Prog::Lantern::LanternResourceNexus.assemble(
       project_id: project.id,
@@ -18,7 +15,6 @@
     )
     LanternResource[st.id]
   end
-
   let(:pg_wo_pwermission) do
     st = Prog::Lantern::LanternResourceNexus.assemble(
       project_id: project_wo_permissions.id,
@@ -31,6 +27,12 @@
 
     LanternResource[st.id]
   end
+  let(:project) { user.create_project_with_default_policy("project-1", provider: "gcp") }
+  let(:project_wo_permissions) { user.create_project_with_default_policy("project-2", policy_body: [], provider: "gcp") }
+
+  before do
+    allow(LanternServer).to receive(:get_vm_image).and_return(Config.gcp_default_image)
+  end
 
   describe "unauthenticated" do
     it "can not list without login" do