From aeb832966e0e45209a066dfeddf03b41b261ee28 Mon Sep 17 00:00:00 2001 From: Andre Andreassa Date: Thu, 2 Nov 2023 03:21:16 +0000 Subject: [PATCH] feat(cloud-storage): support match_glob for Object.list Introduce `match_glob` parameter to `Bucket.files, with required changes to the service method and storing instance variables for pagination in `File::List`. Updating gem to Ruby v2.7 as current supported version. More info at https://cloud.google.com/storage/docs/json_api/v1/objects/list --- ...try-conformance-test-against-emulator.yaml | 2 +- .../google-cloud-storage.gemspec | 2 +- .../lib/google/cloud/storage/bucket.rb | 12 ++- .../lib/google/cloud/storage/file/list.rb | 10 +- .../lib/google/cloud/storage/service.rb | 2 + .../test/google/cloud/storage/bucket_test.rb | 93 +++++++++++-------- .../test/google/cloud/storage/file_test.rb | 4 +- .../google/cloud/storage/lazy/bucket_test.rb | 93 +++++++++++-------- .../cloud/storage/project_anonymous_test.rb | 2 +- .../test/google/cloud/storage/retry/utils.rb | 2 +- 10 files changed, 136 insertions(+), 86 deletions(-) diff --git a/.github/workflows/storage-retry-conformance-test-against-emulator.yaml b/.github/workflows/storage-retry-conformance-test-against-emulator.yaml index ecce644fb026..7ba3945d1a2a 100644 --- a/.github/workflows/storage-retry-conformance-test-against-emulator.yaml +++ b/.github/workflows/storage-retry-conformance-test-against-emulator.yaml @@ -26,7 +26,7 @@ jobs: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4 - uses: ruby/setup-ruby@v1 with: - ruby-version: '2.6' + ruby-version: '3.2' - run: ruby --version - run: bundle install - run: bundle exec rake conformance diff --git a/google-cloud-storage/google-cloud-storage.gemspec b/google-cloud-storage/google-cloud-storage.gemspec index 84d836884e54..0ceed18fa112 100644 --- a/google-cloud-storage/google-cloud-storage.gemspec +++ b/google-cloud-storage/google-cloud-storage.gemspec @@ -16,7 +16,7 @@ Gem::Specification.new do |gem| ["OVERVIEW.md", "AUTHENTICATION.md", "LOGGING.md", "CONTRIBUTING.md", "TROUBLESHOOTING.md", "CHANGELOG.md", "CODE_OF_CONDUCT.md", "LICENSE", ".yardopts"] gem.require_paths = ["lib"] - gem.required_ruby_version = ">= 2.5" + gem.required_ruby_version = ">= 2.7" gem.add_dependency "google-cloud-core", "~> 1.6" gem.add_dependency "google-apis-iamcredentials_v1", "~> 0.1" diff --git a/google-cloud-storage/lib/google/cloud/storage/bucket.rb b/google-cloud-storage/lib/google/cloud/storage/bucket.rb index 59b45ed62e11..58988b275ca3 100644 --- a/google-cloud-storage/lib/google/cloud/storage/bucket.rb +++ b/google-cloud-storage/lib/google/cloud/storage/bucket.rb @@ -1299,6 +1299,9 @@ def delete if_metageneration_match: nil, if_metageneration_not_match: nil # `prefixes` are omitted. # @param [String] token A previously-returned page token representing # part of the larger set of results to view. + # @param [String] match_glob A glob pattern used to filter results returned in items (e.g. `foo*bar`). + # The string value must be UTF-8 encoded. See: + # https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-object-glob # @param [Integer] max Maximum number of items plus prefixes to return. # As duplicate prefixes are omitted, fewer total results may be # returned than requested. The default value of this parameter is @@ -1334,14 +1337,17 @@ def delete if_metageneration_match: nil, if_metageneration_not_match: nil # end # def files prefix: nil, delimiter: nil, token: nil, max: nil, - versions: nil + versions: nil, match_glob: nil ensure_service! gapi = service.list_files name, prefix: prefix, delimiter: delimiter, token: token, max: max, versions: versions, - user_project: user_project + user_project: user_project, + match_glob: match_glob File::List.from_gapi gapi, service, name, prefix, delimiter, max, - versions, user_project: user_project + versions, + user_project: user_project, + match_glob: match_glob end alias find_files files diff --git a/google-cloud-storage/lib/google/cloud/storage/file/list.rb b/google-cloud-storage/lib/google/cloud/storage/file/list.rb index 204874e02c17..1d1f0ce30a16 100644 --- a/google-cloud-storage/lib/google/cloud/storage/file/list.rb +++ b/google-cloud-storage/lib/google/cloud/storage/file/list.rb @@ -83,10 +83,12 @@ def next token: @token, max: @max, versions: @versions, - user_project: @user_project + user_project: @user_project, + match_glob: @match_glob File::List.from_gapi gapi, @service, @bucket, @prefix, @delimiter, @max, @versions, - user_project: @user_project + user_project: @user_project, + match_glob: @match_glob end ## @@ -163,7 +165,8 @@ def all request_limit: nil, &block # Google::Apis::StorageV1::Objects object. def self.from_gapi gapi_list, service, bucket = nil, prefix = nil, delimiter = nil, max = nil, versions = nil, - user_project: nil + user_project: nil, + match_glob: nil files = new(Array(gapi_list.items).map do |gapi_object| File.from_gapi gapi_object, service, user_project: user_project end) @@ -176,6 +179,7 @@ def self.from_gapi gapi_list, service, bucket = nil, prefix = nil, files.instance_variable_set :@max, max files.instance_variable_set :@versions, versions files.instance_variable_set :@user_project, user_project + files.instance_variable_set :@match_glob, match_glob files end diff --git a/google-cloud-storage/lib/google/cloud/storage/service.rb b/google-cloud-storage/lib/google/cloud/storage/service.rb index fda2d3d33738..a1e80272390d 100644 --- a/google-cloud-storage/lib/google/cloud/storage/service.rb +++ b/google-cloud-storage/lib/google/cloud/storage/service.rb @@ -349,6 +349,7 @@ def delete_notification bucket_name, notification_id, user_project: nil, options # Retrieves a list of files matching the criteria. def list_files bucket_name, delimiter: nil, max: nil, token: nil, prefix: nil, versions: nil, user_project: nil, + match_glob: nil, options: {} execute do service.list_objects \ @@ -356,6 +357,7 @@ def list_files bucket_name, delimiter: nil, max: nil, token: nil, page_token: token, prefix: prefix, versions: versions, user_project: user_project(user_project), + match_glob: match_glob, options: options end end diff --git a/google-cloud-storage/test/google/cloud/storage/bucket_test.rb b/google-cloud-storage/test/google/cloud/storage/bucket_test.rb index 3950a73ab21e..c27e66757461 100644 --- a/google-cloud-storage/test/google/cloud/storage/bucket_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/bucket_test.rb @@ -639,7 +639,7 @@ mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(num_files), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -659,7 +659,7 @@ mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(num_files), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -677,7 +677,7 @@ it "lists files with prefix set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, nil, ["/prefix/path1/", "/prefix/path2/"]), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: "/prefix/", versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: "/prefix/", versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -698,7 +698,7 @@ it "lists files with delimiter set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, nil, ["/prefix/path1/", "/prefix/path2/"]), - [bucket.name], delimiter: "/", max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: "/", max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -716,10 +716,29 @@ end end + it "lists files with match_glob set" do + mock = Minitest::Mock.new + mock.expect :list_objects, list_files_gapi(2), + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: "/foo/**/bar/", options: {} + + bucket.service.mocked_service = mock + + files = bucket.files match_glob: "/foo/**/bar/" + + mock.verify + + _(files.count).must_equal 2 + files.each do |file| + _(file).must_be_kind_of Google::Cloud::Storage::File + _(file.user_project).must_be :nil? + end + + end + it "lists files with max set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: 3, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: 3, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -739,7 +758,7 @@ it "lists files with versions set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: true, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: true, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -759,7 +778,7 @@ mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(num_files), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: "test", options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: "test", match_glob: nil, options: {} bucket_user_project.service.mocked_service = mock @@ -777,9 +796,9 @@ it "paginates files" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -807,9 +826,9 @@ it "paginates files with next? and next" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -836,9 +855,9 @@ it "paginates files with next? and next and prefix set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: "/prefix/", versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: "/prefix/", versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: "/prefix/", versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: "/prefix/", versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -865,9 +884,9 @@ it "paginates files with next? and next and delimiter set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: "/", max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: "/", max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: "/", max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: "/", max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -894,9 +913,9 @@ it "paginates files with next? and next and max set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: 3, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: 3, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: 3, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: 3, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -923,9 +942,9 @@ it "paginates files with next? and next and versions set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: true, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: true, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: true, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: true, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -952,9 +971,9 @@ it "paginates files with user_project set to true" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: "test", options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: "test", match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: "test", options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: "test", match_glob: nil, options: {} bucket_user_project.service.mocked_service = mock @@ -982,9 +1001,9 @@ it "paginates files with all" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -1002,9 +1021,9 @@ it "paginates files with all and prefix set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: "/prefix/", versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: "/prefix/", versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: "/prefix/", versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: "/prefix/", versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -1022,9 +1041,9 @@ it "paginates files with all and delimiter set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: "/", max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: "/", max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: "/", max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: "/", max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -1042,9 +1061,9 @@ it "paginates files with all and max set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: 3, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: 3, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: 3, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: 3, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -1062,9 +1081,9 @@ it "paginates files with all and versions set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: true, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: true, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: true, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: true, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -1082,9 +1101,9 @@ it "paginates files with all using Enumerator" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(3, "second_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -1102,9 +1121,9 @@ it "paginates files with all and request_limit set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(3, "second_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -1122,9 +1141,9 @@ it "paginates files with all and user_project set to true" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket_user_project.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: "test", options: {} + [bucket_user_project.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: "test", match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(3, "second_page_token"), - [bucket_user_project.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: "test", options: {} + [bucket_user_project.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: "test", match_glob: nil, options: {} bucket_user_project.service.mocked_service = mock diff --git a/google-cloud-storage/test/google/cloud/storage/file_test.rb b/google-cloud-storage/test/google/cloud/storage/file_test.rb index a6ca3c012d9a..d00487bb3421 100644 --- a/google-cloud-storage/test/google/cloud/storage/file_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/file_test.rb @@ -1489,7 +1489,7 @@ def file_user_project.sleep *args mock.expect :get_object, Google::Apis::StorageV1::Object.from_json(random_file_hash(bucket.name, file_name, generations[0]).to_json), [bucket.name, file_name], **get_object_args mock.expect :list_objects, Google::Apis::StorageV1::Objects.new(kind: "storage#objects", items: file_gapis), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: file_name, versions: true, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: file_name, versions: true, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock file.service.mocked_service = mock @@ -1515,7 +1515,7 @@ def file_user_project.sleep *args mock.expect :get_object, Google::Apis::StorageV1::Object.from_json(random_file_hash(bucket_user_project.name, file_name, generations[0]).to_json), [bucket_user_project.name, file_name], **get_object_args(user_project: "test") mock.expect :list_objects, Google::Apis::StorageV1::Objects.new(kind: "storage#objects", items: file_gapis), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: file_name, versions: true, user_project: "test", options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: file_name, versions: true, user_project: "test", match_glob: nil, options: {} bucket_user_project.service.mocked_service = mock file.service.mocked_service = mock diff --git a/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb b/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb index 29ff912473b7..2ed94d34c9ae 100644 --- a/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb @@ -369,7 +369,7 @@ mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(num_files), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -389,7 +389,7 @@ mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(num_files), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock files = bucket.find_files @@ -406,7 +406,7 @@ it "lists files with prefix set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, nil, ["/prefix/path1/", "/prefix/path2/"]), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: "/prefix/", versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: "/prefix/", versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -427,7 +427,7 @@ it "lists files with delimiter set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, nil, ["/prefix/path1/", "/prefix/path2/"]), - [bucket.name], delimiter: "/", max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: "/", max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock files = bucket.files delimiter: "/" @@ -444,10 +444,29 @@ end end + it "lists files with match_glob set" do + mock = Minitest::Mock.new + mock.expect :list_objects, list_files_gapi(2), + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: "/foo/**/bar/", options: {} + + bucket.service.mocked_service = mock + + files = bucket.files match_glob: "/foo/**/bar/" + + mock.verify + + _(files.count).must_equal 2 + files.each do |file| + _(file).must_be_kind_of Google::Cloud::Storage::File + _(file.user_project).must_be :nil? + end + + end + it "lists files with max set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: 3, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: 3, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -467,7 +486,7 @@ it "lists files with versions set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: true, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: true, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -487,7 +506,7 @@ mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(num_files), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: "test", options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: "test", match_glob: nil, options: {} bucket_user_project.service.mocked_service = mock @@ -505,9 +524,9 @@ it "paginates files" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -535,9 +554,9 @@ it "paginates files with next? and next" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock first_files = bucket.files @@ -563,9 +582,9 @@ it "paginates files with next? and next and prefix set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: "/prefix/", versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: "/prefix/", versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: "/prefix/", versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: "/prefix/", versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -592,9 +611,9 @@ it "paginates files with next? and next and delimiter set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: "/", max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: "/", max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: "/", max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: "/", max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -621,9 +640,9 @@ it "paginates files with next? and next and max set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: 3, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: 3, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: 3, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: 3, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock first_files = bucket.files max: 3 @@ -649,9 +668,9 @@ it "paginates files with next? and next and versions set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: true, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: true, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: true, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: true, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -678,9 +697,9 @@ it "paginates files with user_project set to true" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: "test", options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: "test", match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: "test", options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: "test", match_glob: nil, options: {} bucket_user_project.service.mocked_service = mock @@ -708,9 +727,9 @@ it "paginates files with all" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock files = bucket.files.all.to_a @@ -727,9 +746,9 @@ it "paginates files with all and prefix set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: "/prefix/", versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: "/prefix/", versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: "/prefix/", versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: "/prefix/", versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -747,9 +766,9 @@ it "paginates files with all and delimiter set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: "/", max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: "/", max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: "/", max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: "/", max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -767,9 +786,9 @@ it "paginates files with all and max set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: 3, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: 3, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: 3, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: 3, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -787,9 +806,9 @@ it "paginates files with all and versions set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: true, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: true, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(2), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: true, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: true, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -807,9 +826,9 @@ it "paginates files with all using Enumerator" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(3, "second_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -827,9 +846,9 @@ it "paginates files with all and request_limit set" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(3, "second_page_token"), - [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, options: {} + [bucket.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} bucket.service.mocked_service = mock @@ -847,9 +866,9 @@ it "paginates files with all and user_project set to true" do mock = Minitest::Mock.new mock.expect :list_objects, list_files_gapi(3, "next_page_token"), - [bucket_user_project.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: "test", options: {} + [bucket_user_project.name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: "test", match_glob: nil, options: {} mock.expect :list_objects, list_files_gapi(3, "second_page_token"), - [bucket_user_project.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: "test", options: {} + [bucket_user_project.name], delimiter: nil, max_results: nil, page_token: "next_page_token", prefix: nil, versions: nil, user_project: "test", match_glob: nil, options: {} bucket_user_project.service.mocked_service = mock diff --git a/google-cloud-storage/test/google/cloud/storage/project_anonymous_test.rb b/google-cloud-storage/test/google/cloud/storage/project_anonymous_test.rb index 7850aff8e346..cd0ac5410abe 100644 --- a/google-cloud-storage/test/google/cloud/storage/project_anonymous_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/project_anonymous_test.rb @@ -63,7 +63,7 @@ def stub.list_buckets *args mock = Minitest::Mock.new mock.expect :get_bucket, find_bucket_gapi(bucket_name), [bucket_name], **get_bucket_args mock.expect :list_objects, list_files_gapi(num_files), - [bucket_name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, options: {} + [bucket_name], delimiter: nil, max_results: nil, page_token: nil, prefix: nil, versions: nil, user_project: nil, match_glob: nil, options: {} anonymous_storage.service.mocked_service = mock bucket = anonymous_storage.bucket bucket_name diff --git a/google-cloud-storage/test/google/cloud/storage/retry/utils.rb b/google-cloud-storage/test/google/cloud/storage/retry/utils.rb index 04b6f806f04a..4d3d90007ddf 100644 --- a/google-cloud-storage/test/google/cloud/storage/retry/utils.rb +++ b/google-cloud-storage/test/google/cloud/storage/retry/utils.rb @@ -167,7 +167,7 @@ def self.get_bucket_policy client, _preconditions, **resources def self.insert_bucket client, _preconditions, **resources bucket_name = "new-bucket-" + Time.now.to_i.to_s + "-" + SecureRandom.hex(4) - new_bucket = Google::Apis::StorageV1::Bucket.new({name: bucket_name}) + new_bucket = Google::Apis::StorageV1::Bucket.new name: bucket_name client.insert_bucket(new_bucket) end