From c76569ddb745de9fc50197c9c3ba2058b56c38c2 Mon Sep 17 00:00:00 2001 From: Stan Lo Date: Wed, 15 Jan 2025 17:41:40 +0000 Subject: [PATCH] Add Tapioca compiler for JobIteration It takes the job's `build_enumerator` method and generates the signatures for the job's corresponding `perform_later` and `perform_now` methods. --- Gemfile | 2 + Gemfile.lock | 33 ++ lib/tapioca/dsl/compilers/job_iteration.rb | 120 +++++++ .../dsl/compilers/job_iteration_test.rb | 326 ++++++++++++++++++ 4 files changed, 481 insertions(+) create mode 100644 lib/tapioca/dsl/compilers/job_iteration.rb create mode 100644 test/tapioca/dsl/compilers/job_iteration_test.rb diff --git a/Gemfile b/Gemfile index 94f9e46c..a2a7ce94 100644 --- a/Gemfile +++ b/Gemfile @@ -37,3 +37,5 @@ gem "csv" # required for Ruby 3.4+ # for unit testing optional sorbet support gem "sorbet-runtime" + +gem "tapioca" diff --git a/Gemfile.lock b/Gemfile.lock index 99e75ffc..bf7c6456 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -35,12 +35,14 @@ GEM tzinfo (~> 2.0) ast (2.4.2) base64 (0.2.0) + benchmark (0.4.0) bigdecimal (3.1.9) coderay (1.1.3) concurrent-ruby (1.3.4) connection_pool (2.4.1) csv (3.3.2) drb (2.2.1) + erubi (1.13.1) globalid (1.2.1) activesupport (>= 6.1) i18n (1.14.6) @@ -54,10 +56,12 @@ GEM mono_logger (1.1.2) multi_json (1.15.0) mutex_m (0.2.0) + netrc (0.11.0) parallel (1.25.1) parser (3.3.3.0) ast (~> 2.4.1) racc + prism (1.3.0) pry (0.15.2) coderay (~> 1.1) method_source (~> 1.0) @@ -65,6 +69,9 @@ GEM rack (3.1.8) rainbow (3.1.1) rake (13.2.1) + rbi (0.2.3) + prism (~> 1.0) + sorbet-runtime (>= 0.5.9204) redis (5.3.0) redis-client (>= 0.22.0) redis-client (0.23.0) @@ -102,12 +109,37 @@ GEM redis-client (>= 0.19.0) sinatra (1.0) rack (>= 1.0) + sorbet (0.5.11460) + sorbet-static (= 0.5.11460) sorbet-runtime (0.5.11460) + sorbet-static (0.5.11460-universal-darwin) + sorbet-static-and-runtime (0.5.11460) + sorbet (= 0.5.11460) + sorbet-runtime (= 0.5.11460) + spoom (1.5.0) + erubi (>= 1.10.0) + prism (>= 0.28.0) + sorbet-static-and-runtime (>= 0.5.10187) + thor (>= 0.19.2) + tapioca (0.16.7) + benchmark + bundler (>= 2.2.25) + netrc (>= 0.11.0) + parallel (>= 1.21.0) + rbi (~> 0.2) + sorbet-static-and-runtime (>= 0.5.11087) + spoom (>= 1.2.0) + thor (>= 1.2.0) + yard-sorbet + thor (1.3.2) timeout (0.4.1) tzinfo (2.0.6) concurrent-ruby (~> 1.0) unicode-display_width (2.5.0) yard (0.9.37) + yard-sorbet (0.9.0) + sorbet-runtime + yard PLATFORMS ruby @@ -127,6 +159,7 @@ DEPENDENCIES rubocop-shopify sidekiq sorbet-runtime + tapioca yard BUNDLED WITH diff --git a/lib/tapioca/dsl/compilers/job_iteration.rb b/lib/tapioca/dsl/compilers/job_iteration.rb new file mode 100644 index 00000000..6dcfae4e --- /dev/null +++ b/lib/tapioca/dsl/compilers/job_iteration.rb @@ -0,0 +1,120 @@ +# typed: strict +# frozen_string_literal: true + +return unless defined?(JobIteration::Iteration) + +module Tapioca + module Dsl + module Compilers + class JobIteration < Compiler + extend T::Sig + + ConstantType = type_member { { fixed: T.class_of(::JobIteration::Iteration) } } + + sig { override.void } + def decorate + return unless constant.instance_methods(false).include?(:build_enumerator) + + root.create_path(constant) do |job| + method = constant.instance_method(:build_enumerator) + constant_name = name_of(constant) + expanded_parameters = compile_method_parameters_to_rbi(method).reject do |typed_param| + typed_param.param.name == "cursor" + end + + job.create_method( + "perform_later", + parameters: perform_later_parameters(expanded_parameters, constant_name), + return_type: "T.any(#{constant_name}, FalseClass)", + class_method: true, + ) + + job.create_method( + "perform_now", + parameters: expanded_parameters, + return_type: "void", + class_method: true, + ) + + job.create_method( + "perform", + parameters: expanded_parameters, + return_type: "void", + class_method: false, + ) + end + end + + private + + sig do + params( + parameters: T::Array[RBI::TypedParam], + constant_name: T.nilable(String), + ).returns(T::Array[RBI::TypedParam]) + end + def perform_later_parameters(parameters, constant_name) + if ::Gem::Requirement.new(">= 7.0").satisfied_by?(::ActiveJob.gem_version) + parameters.reject! { |typed_param| RBI::BlockParam === typed_param.param } + parameters + [create_block_param( + "block", + type: "T.nilable(T.proc.params(job: #{constant_name}).void)", + )] + else + parameters + end + end + + def compile_method_parameters_to_rbi(method_def) + signature = signature_of(method_def) + method_def = signature.nil? ? method_def : signature.method + method_types = parameters_types_from_signature(method_def, signature) + + parameters = T.let(method_def.parameters, T::Array[[Symbol, T.nilable(Symbol)]]) + + parameters.each_with_index.flat_map do |(type, name), index| + fallback_arg_name = "_arg#{index}" + + name = name ? name.to_s : fallback_arg_name + name = fallback_arg_name unless valid_parameter_name?(name) + method_type = T.must(method_types[index]) + + case type + when :req + if signature && (type_value = signature.arg_types[index][1]) && type_value.is_a?(T::Types::FixedHash) + type_value.types.map do |key, value| + create_kw_param(key.to_s, type: value.to_s) + end + else + create_param(name, type: method_type) + end + when :opt + create_opt_param(name, type: method_type, default: "T.unsafe(nil)") + when :rest + create_rest_param(name, type: method_type) + when :keyreq + create_kw_param(name, type: method_type) + when :key + create_kw_opt_param(name, type: method_type, default: "T.unsafe(nil)") + when :keyrest + create_kw_rest_param(name, type: method_type) + when :block + create_block_param(name, type: method_type) + else + raise "Unknown type `#{type}`." + end + end + end + + class << self + extend T::Sig + + sig { override.returns(T::Enumerable[Module]) } + def gather_constants + all_classes.select { |c| c < ::JobIteration::Iteration } + end + end + end + end + end +end diff --git a/test/tapioca/dsl/compilers/job_iteration_test.rb b/test/tapioca/dsl/compilers/job_iteration_test.rb new file mode 100644 index 00000000..02034217 --- /dev/null +++ b/test/tapioca/dsl/compilers/job_iteration_test.rb @@ -0,0 +1,326 @@ +# typed: strict +# frozen_string_literal: true + +require "test_helper" +require "tapioca/internal" +require "tapioca/helpers/test/dsl_compiler" +require "tapioca/dsl/compilers/job_iteration" + +module Tapioca + module Dsl + module Compilers + class JobIterationTest < Minitest::Test + extend T::Sig + extend Tapioca::Helpers::Test::Template + include Tapioca::Helpers::Test::DslCompiler + + def setup + require "job-iteration" + require "tapioca/dsl/compilers/job_iteration" + use_dsl_compiler(Tapioca::Dsl::Compilers::JobIteration) + end + + def test_gathers_constants_only_for_jobs_that_include_job_iteration + add_ruby_file("job.rb", <<~RUBY) + class FooJob < ActiveJob::Base + end + + class BarJob < ActiveJob::Base + include JobIteration::Iteration + end + RUBY + + assert_includes(gathered_constants, "BarJob") + refute_includes(gathered_constants, "FooJob") + end + + def test_generates_an_empty_rbi_file_if_there_is_no_build_enumerator_method + add_ruby_file("job.rb", <<~RUBY) + class NotifyJob < ActiveJob::Base + include JobIteration::Iteration + end + RUBY + + expected = <<~RBI + # typed: strong + RBI + + assert_equal(expected, rbi_for(:NotifyJob)) + end + + def test_generates_correct_rbi_file_for_job_with_build_enumerator_method + add_ruby_file("job.rb", <<~RUBY) + class NotifyJob < ActiveJob::Base + include JobIteration::Iteration + + def build_enumerator(user_id, cursor:) + # ... + end + end + RUBY + + expected = template(<<~RBI) + # typed: strong + + class NotifyJob + sig { params(user_id: T.untyped).void } + def perform(user_id); end + + class << self + sig { params(user_id: T.untyped, block: T.nilable(T.proc.params(job: NotifyJob).void)).returns(T.any(NotifyJob, FalseClass)) } + def perform_later(user_id, &block); end + + sig { params(user_id: T.untyped).void } + def perform_now(user_id); end + end + end + RBI + assert_equal(expected, rbi_for(:NotifyJob)) + end + + def test_generates_correct_rbi_file_for_job_with_build_enumerator_method_with_keyword_parameter + add_ruby_file("job.rb", <<~RUBY) + class NotifyJob < ActiveJob::Base + include JobIteration::Iteration + + def build_enumerator(user_id:, profile_id:, cursor:) + # ... + end + end + RUBY + + expected = template(<<~RBI) + # typed: strong + + class NotifyJob + sig { params(user_id: T.untyped, profile_id: T.untyped).void } + def perform(user_id:, profile_id:); end + + class << self + sig { params(user_id: T.untyped, profile_id: T.untyped, block: T.nilable(T.proc.params(job: NotifyJob).void)).returns(T.any(NotifyJob, FalseClass)) } + def perform_later(user_id:, profile_id:, &block); end + + sig { params(user_id: T.untyped, profile_id: T.untyped).void } + def perform_now(user_id:, profile_id:); end + end + end + RBI + assert_equal(expected, rbi_for(:NotifyJob)) + end + + def test_generates_correct_rbi_file_for_job_with_build_enumerator_method_signature + add_ruby_file("job.rb", <<~RUBY) + class NotifyJob < ActiveJob::Base + include JobIteration::Iteration + + extend T::Sig + sig { params(user_id: Integer, cursor: T.untyped).returns(T::Array[T.untyped]) } + def build_enumerator(user_id, cursor:) + # ... + end + end + RUBY + + expected = template(<<~RBI) + # typed: strong + + class NotifyJob + sig { params(user_id: Integer).void } + def perform(user_id); end + + class << self + sig { params(user_id: Integer, block: T.nilable(T.proc.params(job: NotifyJob).void)).returns(T.any(NotifyJob, FalseClass)) } + def perform_later(user_id, &block); end + + sig { params(user_id: Integer).void } + def perform_now(user_id); end + end + end + RBI + assert_equal(expected, rbi_for(:NotifyJob)) + end + + def test_generates_correct_rbi_file_for_job_with_build_enumerator_method_signature_with_keyword_parameter + add_ruby_file("job.rb", <<~RUBY) + class NotifyJob < ActiveJob::Base + include JobIteration::Iteration + + extend T::Sig + sig { params(user_id: Integer, profile_id: Integer, cursor: T.untyped).returns(T::Array[T.untyped]) } + def build_enumerator(user_id:, profile_id:, cursor:) + # ... + end + end + RUBY + + expected = template(<<~RBI) + # typed: strong + + class NotifyJob + sig { params(user_id: Integer, profile_id: Integer).void } + def perform(user_id:, profile_id:); end + + class << self + sig { params(user_id: Integer, profile_id: Integer, block: T.nilable(T.proc.params(job: NotifyJob).void)).returns(T.any(NotifyJob, FalseClass)) } + def perform_later(user_id:, profile_id:, &block); end + + sig { params(user_id: Integer, profile_id: Integer).void } + def perform_now(user_id:, profile_id:); end + end + end + RBI + assert_equal(expected, rbi_for(:NotifyJob)) + end + + def test_generates_correct_rbi_file_for_job_with_build_enumerator_method_with_multiple_parameters + add_ruby_file("job.rb", <<~RUBY) + class NotifyJob < ActiveJob::Base + include JobIteration::Iteration + + extend T::Sig + sig { params(user_id: Integer, name: String, cursor: T.untyped).returns(T::Array[T.untyped]) } + def build_enumerator(user_id, name, cursor:) + # ... + end + end + RUBY + + expected = template(<<~RBI) + # typed: strong + + class NotifyJob + sig { params(user_id: Integer, name: String).void } + def perform(user_id, name); end + + class << self + sig { params(user_id: Integer, name: String, block: T.nilable(T.proc.params(job: NotifyJob).void)).returns(T.any(NotifyJob, FalseClass)) } + def perform_later(user_id, name, &block); end + + sig { params(user_id: Integer, name: String).void } + def perform_now(user_id, name); end + end + end + RBI + assert_equal(expected, rbi_for(:NotifyJob)) + end + + def test_generates_correct_rbi_file_for_job_with_build_enumerator_method_with_aliased_hash_parameter + add_ruby_file("job.rb", <<~RUBY) + class NotifyJob < ActiveJob::Base + include JobIteration::Iteration + + Params = T.type_alias { { user_id: Integer, name: String } } + + extend T::Sig + sig { params(params: Params, cursor: T.untyped).returns(T::Array[T.untyped]) } + def build_enumerator(params, cursor:) + # ... + end + end + RUBY + + expected = template(<<~RBI) + # typed: strong + + class NotifyJob + sig { params(user_id: Integer, name: String).void } + def perform(user_id:, name:); end + + class << self + sig { params(user_id: Integer, name: String, block: T.nilable(T.proc.params(job: NotifyJob).void)).returns(T.any(NotifyJob, FalseClass)) } + def perform_later(user_id:, name:, &block); end + + sig { params(user_id: Integer, name: String).void } + def perform_now(user_id:, name:); end + end + end + RBI + assert_equal(expected, rbi_for(:NotifyJob)) + end + + def test_generates_correct_rbi_file_for_job_with_build_enumerator_method_with_nested_hash_parameter + add_ruby_file("job.rb", <<~RUBY) + class ResourceType; end + class Locale; end + + class NotifyJob < ActiveJob::Base + include JobIteration::Iteration + + extend T::Sig + sig do + params( + params: { shop_id: Integer, resource_types: T::Array[ResourceType], locale: Locale, metadata: T.nilable(String) }, + cursor: T.untyped + ).returns(T::Array[T.untyped]) + end + def build_enumerator(params, cursor:) + # ... + end + end + RUBY + + expected = template(<<~RBI) + # typed: strong + + class NotifyJob + sig { params(shop_id: Integer, resource_types: T::Array[ResourceType], locale: Locale, metadata: T.nilable(String)).void } + def perform(shop_id:, resource_types:, locale:, metadata:); end + + class << self + sig { params(shop_id: Integer, resource_types: T::Array[ResourceType], locale: Locale, metadata: T.nilable(String), block: T.nilable(T.proc.params(job: NotifyJob).void)).returns(T.any(NotifyJob, FalseClass)) } + def perform_later(shop_id:, resource_types:, locale:, metadata:, &block); end + + sig { params(shop_id: Integer, resource_types: T::Array[ResourceType], locale: Locale, metadata: T.nilable(String)).void } + def perform_now(shop_id:, resource_types:, locale:, metadata:); end + end + end + RBI + assert_equal(expected, rbi_for(:NotifyJob)) + end + + def test_generates_correct_rbi_file_for_job_with_build_enumerator_method_with_complex_hash_parameter + add_ruby_file("job.rb", <<~RUBY) + class NotifyJob < ActiveJob::Base + include JobIteration::Iteration + + extend T::Sig + sig do + params( + params: { + shop_ids: T.any(Integer, T::Array[Integer]), + profile_ids: T.any(Integer, T::Array[Integer]), + extension_ids: T.any(Integer, T::Array[Integer]), + foo: Symbol, + bar: String + }, + cursor: T.untyped + ).returns(T::Array[T.untyped]) + end + def build_enumerator(params, cursor:) + # ... + end + end + RUBY + + expected = template(<<~RBI) + # typed: strong + + class NotifyJob + sig { params(shop_ids: T.any(Integer, T::Array[Integer]), profile_ids: T.any(Integer, T::Array[Integer]), extension_ids: T.any(Integer, T::Array[Integer]), foo: Symbol, bar: String).void } + def perform(shop_ids:, profile_ids:, extension_ids:, foo:, bar:); end + + class << self + sig { params(shop_ids: T.any(Integer, T::Array[Integer]), profile_ids: T.any(Integer, T::Array[Integer]), extension_ids: T.any(Integer, T::Array[Integer]), foo: Symbol, bar: String, block: T.nilable(T.proc.params(job: NotifyJob).void)).returns(T.any(NotifyJob, FalseClass)) } + def perform_later(shop_ids:, profile_ids:, extension_ids:, foo:, bar:, &block); end + + sig { params(shop_ids: T.any(Integer, T::Array[Integer]), profile_ids: T.any(Integer, T::Array[Integer]), extension_ids: T.any(Integer, T::Array[Integer]), foo: Symbol, bar: String).void } + def perform_now(shop_ids:, profile_ids:, extension_ids:, foo:, bar:); end + end + end + RBI + assert_equal(expected, rbi_for(:NotifyJob)) + end + end + end + end +end