From ad29f1159a2b8b8eeee91427f7437fb06437b3c9 Mon Sep 17 00:00:00 2001 From: Justin Littman Date: Mon, 16 Oct 2023 16:50:09 -0400 Subject: [PATCH] Adds connection pool for redis. Increases timeout for redis. --- Gemfile | 1 + Gemfile.lock | 1 + app/jobs/concerns/unique_job.rb | 10 ++-------- config/initializers/redis.rb | 7 +++++++ config/initializers/sidekiq.rb | 7 +++++-- config/settings.yml | 4 +++- 6 files changed, 19 insertions(+), 11 deletions(-) create mode 100644 config/initializers/redis.rb diff --git a/Gemfile b/Gemfile index 91dcdbd67..1b9905177 100644 --- a/Gemfile +++ b/Gemfile @@ -3,6 +3,7 @@ source 'https://rubygems.org' # general Ruby/Rails gems gem 'aws-sdk-s3', '~> 1.17' gem 'committee' # Validates HTTP requests/responses per OpenAPI specification +gem 'connection_pool' # Used for redis gem 'config' # Settings to manage configs on different instances gem 'honeybadger' # for error reporting / tracking / notifications gem 'jbuilder', '~> 2.5' # Build JSON APIs with ease. diff --git a/Gemfile.lock b/Gemfile.lock index 91f0025a1..8a524a40a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -461,6 +461,7 @@ DEPENDENCIES capybara committee config + connection_pool debug dlss-capistrano dor-event-client (~> 1.0) diff --git a/app/jobs/concerns/unique_job.rb b/app/jobs/concerns/unique_job.rb index 1d7d716f7..80372d8d5 100644 --- a/app/jobs/concerns/unique_job.rb +++ b/app/jobs/concerns/unique_job.rb @@ -72,12 +72,6 @@ def lock_timeout 3600 end - def redis_connection - Redis.new(url: Settings.redis_url).tap do |conn| - yield conn if block_given? - end - end - # @return [String] the key for locking this job/payload combination, e.g. 'lock:MySpecificJob-bt821jk7040;1' def queue_lock_key(*args) # Changes in ActiveModel object don't result in new lock (which they do when just calling to_s). @@ -90,7 +84,7 @@ def before_enqueue_lock(*) now = Time.now.to_i new_expiry_time = now + lock_timeout + 1 - redis_connection do |conn| + REDIS.with do |conn| # return true if we successfully acquired the lock # "Set key to hold string value if key does not exist" (otherwise no-op) -- https://redis.io/commands/setnx if conn.setnx(key, new_expiry_time) @@ -123,7 +117,7 @@ def before_enqueue_lock(*) def clear_lock(*) key = queue_lock_key(*) Rails.logger.info("clearing lock for #{key}...") - redis_connection do |conn| + REDIS.with do |conn| conn.del(key).tap do |del_result| Rails.logger.info("...cleared lock for #{key} (del_result=#{del_result})") end diff --git a/config/initializers/redis.rb b/config/initializers/redis.rb new file mode 100644 index 000000000..112440a3f --- /dev/null +++ b/config/initializers/redis.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +pool_size = ENV.fetch('RAILS_MAX_THREADS', 5) + +REDIS = ConnectionPool.new(size: pool_size) do + Redis.new(url: Settings.redis_url, timeout: Settings.redis_timeout) +end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index c2a01dce2..66c491a38 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,12 +1,15 @@ # frozen_string_literal: true +# See separate initializer for redis used by unique job. + +# Note the increased timeouts to try to address Redis timeouts Sidekiq.configure_server do |config| - config.redis = { url: Settings.redis_url } + config.redis = { url: Settings.redis_url, network_timeout: Settings.redis_timeout, pool_timeout: Settings.redis_timeout } # For Sidekiq Pro config.super_fetch! end Sidekiq.configure_client do |config| - config.redis = { url: Settings.redis_url } + config.redis = { url: Settings.redis_url, network_timeout: Settings.redis_timeout, pool_timeout: Settings.redis_timeout } end Sidekiq::Client.reliable_push! unless Rails.env.test? diff --git a/config/settings.yml b/config/settings.yml index e1bc37972..57375ecaf 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -59,6 +59,7 @@ rabbitmq: password: guest redis_url: redis://localhost:6379/ +redis_timeout: 5 # seconds honeybadger_checkins: moab_to_catalog: xyzzy @@ -68,4 +69,5 @@ honeybadger_checkins: slow_queries: enable: false - threshold: 500 \ No newline at end of file + threshold: 500 +