Fix backup listing bug for lantern timeline

lib/hosting/gcp_apis.rb

@@ -236,10 +236,10 @@ def resize_vm_disk(zone, disk_source, storage_size_gib)
     wait_for_operation(zone, data["id"])
   end
 
-  def list_objects(bucket, prefix)
+  def list_objects(bucket, pattern)
     connection = Excon.new("https://storage.googleapis.com", headers: @host[:headers])
-    query = {prefix: prefix}
-    puts "Query is #{query}"
+    query = {matchGlob: pattern, delimiter: "/"}
+
     response = connection.get(path: "/storage/v1/b/#{bucket}/o", query: query, expects: [200, 400])
     Hosting::GcpApis.check_errors(response)
     data = JSON.parse(response.body)

model/lantern/lantern_timeline.rb

@@ -76,17 +76,24 @@ def take_manual_backup
 
   def backups
     blob_storage_client
-      .list_objects(Config.lantern_backup_bucket, "#{ubid}/basebackups_005/")
-      .select { _1[:key].end_with?("backup_stop_sentinel.json") }
+      .list_objects(Config.lantern_backup_bucket, "#{ubid}/basebackups_005/*_backup_stop_sentinel.json")
   end
 
   def backups_with_metadata
     storage_client = blob_storage_client
-    backups
-      .map {
-        metadata = storage_client.get_json_object(Config.lantern_backup_bucket, _1[:key])
-        {**_1, compressed_size: metadata["CompressedSize"], uncompressed_size: metadata["UncompressedSize"]}
-      }
+    mutex = Mutex.new
+    thread_count = 8
+    backup_list = backups
+    results = []
+    Array.new(thread_count) {
+      Thread.new(backup_list, results) do |backup_list, results|
+        while (backup = mutex.synchronize { backup_list.pop })
+          metadata = storage_client.get_json_object(Config.lantern_backup_bucket, backup[:key])
+          mutex.synchronize { results << {**backup, compressed_size: metadata["CompressedSize"], uncompressed_size: metadata["UncompressedSize"]} }
+        end
+      end
+    }.each(&:join)
+    results
   end
 
   def get_backup_label(key)

spec/lib/hosting/gcp_apis_spec.rb

@@ -179,22 +179,22 @@
     describe "#list_objects" do
       it "throws error from response" do
         stub_request(:post, "https://oauth2.googleapis.com/token").to_return(status: 200, body: JSON.dump({}), headers: {"Content-Type" => "application/json"})
-        stub_request(:get, "https://storage.googleapis.com/storage/v1/b/test/o?prefix=test").to_return(status: 200, body: JSON.dump({error: {errors: [{message: "test error"}]}}), headers: {"Content-Type" => "application/json"})
+        stub_request(:get, "https://storage.googleapis.com/storage/v1/b/test/o?delimiter=/&matchGlob=test").to_return(status: 200, body: JSON.dump({error: {errors: [{message: "test error"}]}}), headers: {"Content-Type" => "application/json"})
         api = described_class.new
         expect { api.list_objects("test", "test") }.to raise_error "test error"
       end
 
       it "returns empty array" do
         stub_request(:post, "https://oauth2.googleapis.com/token").to_return(status: 200, body: JSON.dump({}), headers: {"Content-Type" => "application/json"})
-        stub_request(:get, "https://storage.googleapis.com/storage/v1/b/test/o?prefix=test").to_return(status: 200, body: JSON.dump({}), headers: {"Content-Type" => "application/json"})
+        stub_request(:get, "https://storage.googleapis.com/storage/v1/b/test/o?delimiter=/&matchGlob=test").to_return(status: 200, body: JSON.dump({}), headers: {"Content-Type" => "application/json"})
         api = described_class.new
         expect(api.list_objects("test", "test")).to eq([])
       end
 
       it "maps object names" do
         stub_request(:post, "https://oauth2.googleapis.com/token").to_return(status: 200, body: JSON.dump({}), headers: {"Content-Type" => "application/json"})
         t = Time.new
-        stub_request(:get, "https://storage.googleapis.com/storage/v1/b/test/o?prefix=test").to_return(status: 200, body: JSON.dump({items: [{name: "test1", updated: t}, {name: "test2", updated: t}]}), headers: {"Content-Type" => "application/json"})
+        stub_request(:get, "https://storage.googleapis.com/storage/v1/b/test/o?delimiter=/&matchGlob=test").to_return(status: 200, body: JSON.dump({items: [{name: "test1", updated: t}, {name: "test2", updated: t}]}), headers: {"Content-Type" => "application/json"})
         api = described_class.new
         t_str = Time.new JSON.parse(JSON.dump(t))
         expect(api.list_objects("test", "test")).to eq([{key: "test1", last_modified: t_str}, {key: "test2", last_modified: t_str}])

spec/model/lantern/lantern_timeline_spec.rb

@@ -88,9 +88,9 @@
   end
 
   describe "#backups" do
-    it "returns empty array" do
+    it "returns backup list" do
       gcp_api = instance_double(Hosting::GcpApis)
-      expect(gcp_api).to receive(:list_objects).with(Config.lantern_backup_bucket, "pvr1mcnhzd8p0qwwa00tr5cvex/basebackups_005/").and_return([{key: "1_backup_stop_sentinel.json"}, {key: "2_backup_stop_sentinel.jzon"}, {key: "3_backup_stop_sentinel.json"}])
+      expect(gcp_api).to receive(:list_objects).with(Config.lantern_backup_bucket, "pvr1mcnhzd8p0qwwa00tr5cvex/basebackups_005/*_backup_stop_sentinel.json").and_return([{key: "1_backup_stop_sentinel.json"}, {key: "3_backup_stop_sentinel.json"}])
       expect(lantern_timeline).to receive(:blob_storage_client).and_return(gcp_api)
       expect(lantern_timeline.backups).to eq([{key: "1_backup_stop_sentinel.json"}, {key: "3_backup_stop_sentinel.json"}])
     end
@@ -99,11 +99,10 @@
   describe "#backups_with_metadata" do
     it "returns backups with metadata about size" do
       gcp_api = instance_double(Hosting::GcpApis)
-      expect(gcp_api).to receive(:list_objects).with(Config.lantern_backup_bucket, "pvr1mcnhzd8p0qwwa00tr5cvex/basebackups_005/").and_return([{key: "1_backup_stop_sentinel.json"}, {key: "2_backup_stop_sentinel.json"}])
+      expect(gcp_api).to receive(:list_objects).with(Config.lantern_backup_bucket, "pvr1mcnhzd8p0qwwa00tr5cvex/basebackups_005/*_backup_stop_sentinel.json").and_return([{key: "1_backup_stop_sentinel.json"}])
       expect(gcp_api).to receive(:get_json_object).with(Config.lantern_backup_bucket, "1_backup_stop_sentinel.json").and_return({"CompressedSize" => 10, "UncompressedSize" => 20})
-      expect(gcp_api).to receive(:get_json_object).with(Config.lantern_backup_bucket, "2_backup_stop_sentinel.json").and_return({"CompressedSize" => 10, "UncompressedSize" => 20})
       expect(lantern_timeline).to receive(:blob_storage_client).and_return(gcp_api).at_least(:once)
-      expect(lantern_timeline.backups_with_metadata).to eq([{key: "1_backup_stop_sentinel.json", compressed_size: 10, uncompressed_size: 20}, {key: "2_backup_stop_sentinel.json", compressed_size: 10, uncompressed_size: 20}])
+      expect(lantern_timeline.backups_with_metadata).to eq([{key: "1_backup_stop_sentinel.json", compressed_size: 10, uncompressed_size: 20}])
     end
   end