Merge branch 'release/v5.2.0' into master

.circleci/config.yml

@@ -28,7 +28,7 @@ workflows:
  debian: buster
  matrix:
  parameters:
- ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2' ]
+ ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2', '3.0.0' ]
  # - test:
  # database: postgresql
  # sphinx_version: 3.3.1
@@ -62,7 +62,7 @@ workflows:
  debian: buster
  matrix:
  parameters:
- ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2' ]
+ ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2', '3.0.0' ]
  - test:
  name: "Manticore 3.5.4 with PostgreSQL"
  database: postgresql
@@ -71,7 +71,7 @@ workflows:
  debian: buster
  matrix:
  parameters:
- ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2' ]
+ ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2', '3.0.0' ]
 
 jobs:
  test:
@@ -115,6 +115,8 @@ jobs:
  command: |
  if [ "<< parameters.ruby >>" == "2.7.2" ]; then
  export BUNDLER_VERSION=2.1.4
+ elif [ "<< parameters.ruby >>" == "3.0.0" ]; then
+ export BUNDLER_VERSION=2.1.4
  else
  export BUNDLER_VERSION=1.17.3
  fi

Appraisals

@@ -15,18 +15,18 @@ appraise 'rails_5_0' do
  gem 'jdbc-mysql', '~> 5.1.36', :platform => :jruby
  gem 'activerecord-jdbcmysql-adapter', '~> 50.0', :platform => :jruby
  gem 'activerecord-jdbcpostgresql-adapter', '~> 50.0', :platform => :jruby
-end if RUBY_PLATFORM != "java" || ENV["SPHINX_VERSION"].to_f > 2.1
+end if (RUBY_PLATFORM != "java" || ENV["SPHINX_VERSION"].to_f > 2.1) && RUBY_VERSION.to_f < 3.0
 
 appraise 'rails_5_1' do
  gem 'rails', '~> 5.1.0'
  gem 'mysql2', '~> 0.4.0', :platform => :ruby
-end if RUBY_PLATFORM != 'java'
+end if RUBY_PLATFORM != 'java' && RUBY_VERSION.to_f < 3.0
 
 appraise 'rails_5_2' do
  gem 'rails', '~> 5.2.0'
  gem 'mysql2', '~> 0.5.0', :platform => :ruby
  gem 'pg', '~> 1.0', :platform => :ruby
-end if RUBY_PLATFORM != 'java'
+end if RUBY_PLATFORM != 'java' && RUBY_VERSION.to_f < 3.0
 
 appraise 'rails_6_0' do
  gem 'rails', '~> 6.0.0'

CHANGELOG.markdown

@@ -2,6 +2,20 @@
 
 All notable changes to this project (at least, from v3.0.0 onwards) are documented in this file.
 
+## 5.2.0 - 2021-06-12
+
+[Release Notes](https://github.com/pat/thinking-sphinx/releases/tag/v5.2.0)
+
+### Added
+
+* Confirmed support for Ruby 3.0.
+* Orphaned records in real-time indices can now be cleaned up without running `rails ts:rebuild`. Disabled by default, can be enabled by setting `real_time_tidy` to true per environment in `config/thinking_sphinx.yml` (and will need `ts:rebuild` to restructure indices upon initial deploy). More details in [#1192](https://github.com/pat/thinking-sphinx/pull/1192).
+
+### Fixed
+
+* Avoid loading ActiveRecord during Rails initialisation so app configuration can still have an impact ([@jdelStrother](https://github.com/jdelStrother) in [#1194](https://github.com/pat/thinking-sphinx/pull/1194)).
+* Remove `app/indices` (in both the Rails app and engines) from Rails' eager load paths, which was otherwise leading to indices being loaded more than once. (See [#1191](https://github.com/pat/thinking-sphinx/issues/1191) and [#1195](https://github.com/pat/thinking-sphinx/issues/1195)).
+
 ## 5.1.0 - 2020-12-28
 
 [Release Notes](https://github.com/pat/thinking-sphinx/releases/tag/v5.1.0)

README.textile

@@ -1,6 +1,6 @@
 h1. Thinking Sphinx
 
-Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v5.1.0.
+Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v5.2.0.
 
 h2. Upgrading
 
@@ -14,7 +14,7 @@ It's a gem, so install it like you would any other gem. You will also need to sp
 
 <pre><code>gem 'mysql2', '~> 0.4', :platform => :ruby
 gem 'jdbc-mysql', '~> 5.1.35', :platform => :jruby
-gem 'thinking-sphinx', '~> 5.1'</code></pre>
+gem 'thinking-sphinx', '~> 5.2'</code></pre>
 
 The MySQL gems mentioned are required for connecting to Sphinx, so please include it even when you're using PostgreSQL for your database.
 
@@ -29,18 +29,18 @@ h2. Requirements
 The current release of Thinking Sphinx works with the following versions of its dependencies:
 
 |_. Library |_. Minimum |_. Tested Against |
-| Ruby | v2.4 | v2.4, v2.5, v2.6, v2.7 |
-| Sphinx | v2.2.11 | v2.2.11, v3.2.1 |
-| Manticore | v2.8 | v2.8, v3.4 |
-| ActiveRecord | v4.2 | v4.2..v6.0 |
+| Ruby | v2.4 | v2.4, v2.5, v2.6, v2.7, v3.0 |
+| Sphinx | v2.2.11 | v2.2.11, v3.3.1 |
+| Manticore | v2.8 | v2.8, v3.5 |
+| ActiveRecord | v4.2 | v4.2..v6.1 |
 
 It _might_ work with older versions of Ruby, but it's highly recommended to update to a supported release.
 
-It should also work with JRuby, but the test environment on Travis CI has been timing out, hence that's not actively tested against at the moment.
+It should also work with JRuby, but the test environment for that in CI has been unreliable, hence that's not actively tested against at the moment.
 
 h3. Sphinx or Manticore
 
-Thinking Sphinx is currently built for Sphinx 2.2.11 or newer (though it'll likely work with 2.1.x releases), or Manticore v2.8+.
+Thinking Sphinx is currently built for Sphinx 2.2.11 or newer, or Manticore v2.8+.
 
 h3. Rails and ActiveRecord
 

lib/thinking_sphinx.rb

@@ -96,12 +96,15 @@ module Subscribers; end
 require 'thinking_sphinx/utf8'
 require 'thinking_sphinx/wildcard'
 # Extended
-require 'thinking_sphinx/active_record'
 require 'thinking_sphinx/deltas'
 require 'thinking_sphinx/distributed'
 require 'thinking_sphinx/logger'
 require 'thinking_sphinx/real_time'
 
-require 'thinking_sphinx/railtie' if defined?(Rails::Railtie)
+if defined?(Rails::Railtie)
+ require 'thinking_sphinx/railtie'
+else
+ require 'thinking_sphinx/active_record'
+end
 
 ThinkingSphinx.before_index_hooks << ThinkingSphinx::Hooks::GuardPresence

lib/thinking_sphinx/railtie.rb

@@ -7,6 +7,7 @@ class ThinkingSphinx::Railtie < Rails::Railtie
 
  initializer 'thinking_sphinx.initialisation' do
  ActiveSupport.on_load(:active_record) do
+ require 'thinking_sphinx/active_record'
  ActiveRecord::Base.include ThinkingSphinx::ActiveRecord::Base
  end
 
@@ -16,6 +17,10 @@ class ThinkingSphinx::Railtie < Rails::Railtie
  Rails.root.join("app", "indices").to_s
  )
  end
+
+ Rails.application.config.eager_load_paths -=
+ ThinkingSphinx::Configuration.instance.index_paths
+ Rails.application.config.eager_load_paths.freeze
  end
  end
 

lib/thinking_sphinx/real_time/index/template.rb

@@ -13,6 +13,10 @@ def apply
  add_attribute primary_key, :sphinx_internal_id, :bigint
  add_attribute class_column, :sphinx_internal_class, :string, :facet => true
  add_attribute 0, :sphinx_deleted, :integer
+
+ if tidying?
+ add_attribute -> (_) { Time.current.to_i }, :sphinx_updated_at, :timestamp
+ end
  end
 
  private
@@ -34,7 +38,15 @@ def class_column
  [:class, :name]
  end
 
+ def config
+ ThinkingSphinx::Configuration.instance
+ end
+
  def primary_key
  index.primary_key.to_sym
  end
+
+ def tidying?
+ config.settings["real_time_tidy"]
+ end
 end

lib/thinking_sphinx/real_time/populator.rb

@@ -7,6 +7,7 @@ def self.populate(index)
 
  def initialize(index)
  @index = index
+ @started_at = Time.current
  end
 
  def populate
@@ -17,12 +18,14 @@ def populate
  instrument 'populated', :instances => instances
  end
 
+ transcriber.clear_before(started_at) if configuration.settings["real_time_tidy"]
+
  instrument 'finish_populating'
  end
 
  private
 
- attr_reader :index
+ attr_reader :index, :started_at
 
  delegate :controller, :batch_size, :to => :configuration
  delegate :scope, :to => :index

lib/thinking_sphinx/real_time/transcriber.rb

@@ -5,6 +5,12 @@ def initialize(index)
  @index = index
  end
 
+ def clear_before(time)
+ execute <<~SQL.strip
+ DELETE FROM #{@index.name} WHERE sphinx_updated_at < #{time.to_i}
+ SQL
+ end
+
  def copy(*instances)
  items = instances.select { |instance|
  instance.persisted? && copy?(instance)

lib/thinking_sphinx/real_time/translator.rb

@@ -10,6 +10,7 @@ def initialize(object, column)
  end
 
  def call
+ return name.call(object) if name.is_a?(Proc)
  return name unless name.is_a?(Symbol)
  return result unless result.is_a?(String)
 

lib/thinking_sphinx/settings.rb

@@ -19,7 +19,8 @@ class ThinkingSphinx::Settings
  "binlog_path" => "tmp/binlog/ENVIRONMENT",
  "workers" => "threads",
  "mysql_encoding" => "utf8",
- "maximum_statement_length" => (2 ** 23) - 5
+ "maximum_statement_length" => (2 ** 23) - 5,
+ "real_time_tidy" => false
  }.freeze
 
  def self.call(configuration)

spec/thinking_sphinx/real_time/index_spec.rb

@@ -5,7 +5,9 @@
 describe ThinkingSphinx::RealTime::Index do
  let(:index) { ThinkingSphinx::RealTime::Index.new :user }
  let(:config) { double('config', :settings => {},
- :indices_location => 'location', :next_offset => 8) }
+ :indices_location => 'location', :next_offset => 8,
+ :index_set_class => index_set_class) }
+ let(:index_set_class) { double(:index_set_class, :reference_name => :user) }
 
  before :each do
  allow(ThinkingSphinx::Configuration).to receive_messages :instance => config
@@ -61,6 +63,16 @@
  it "has the internal deleted attribute by default" do
  expect(index.attributes.collect(&:name)).to include('sphinx_deleted')
  end
+
+ it "does not have an internal updated_at attribute by default" do
+ expect(index.attributes.collect(&:name)).to_not include('sphinx_updated_at')
+ end
+
+ it "has an internal updated_at attribute if real_time_tidy is true" do
+ config.settings["real_time_tidy"] = true
+
+ expect(index.attributes.collect(&:name)).to include('sphinx_updated_at')
+ end
  end
 
  describe '#delta?' do

thinking-sphinx.gemspec

@@ -5,7 +5,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
  s.name = 'thinking-sphinx'
- s.version = '5.1.0'
+ s.version = '5.2.0'
  s.platform = Gem::Platform::RUBY
  s.authors = ["Pat Allan"]
  s.email = ["pat@freelancing-gods.com"]