Meet Judge Judy, she is your AI powered SME

Gemfile.lock

@@ -180,6 +180,7 @@ GEM
     faraday-net_http (3.3.0)
       net-http
     ffi (1.17.0-aarch64-linux-gnu)
+    ffi (1.17.0-x86_64-darwin)
     ffi (1.17.0-x86_64-linux-gnu)
     font-awesome-sass (6.5.2)
       sassc (~> 2.0)
@@ -278,6 +279,8 @@ GEM
     nio4r (2.7.3)
     nokogiri (1.16.7-aarch64-linux)
       racc (~> 1.4)
+    nokogiri (1.16.7-x86_64-darwin)
+      racc (~> 1.4)
     nokogiri (1.16.7-x86_64-linux)
       racc (~> 1.4)
     numo-narray (0.9.2.1)
@@ -507,6 +510,7 @@ GEM
 
 PLATFORMS
   aarch64-linux
+  x86_64-darwin-22
   x86_64-linux
 
 DEPENDENCIES

app/controllers/ai_judges_controller.rb

@@ -0,0 +1,22 @@
+class AiJudgesController < ApplicationController
+  def create
+    @book = Book.find(params[:book_id])
+    @user = User.find(params[:user_id])
+
+    @ai_user = AiJudge.new(book: @book, user: @user)
+
+    if @ai_user.save
+      redirect_to @book, notice: 'User was successfully added as an AI Judge.'
+    else
+      render :new
+    end
+  end
+
+  def destroy
+    @ai_judge = User.find(params[:id])
+     @book = Book.find(params[:book_id])
+     @book.judge
+    @ai_user.destroy
+    redirect_to @ai_user.book, notice: 'AI Judge was successfully removed.'
+  end
+end

app/controllers/api/v1/bulk_ratings_controller.rb

@@ -15,11 +15,11 @@ def update
         if params[:doc_ids].present?
           params[:doc_ids].each do |doc_id|
             rating = @query.ratings.find_or_create_by doc_id: doc_id
-            rating.user = @current_user
 
             # rating.update rating_params
             rating.rating = pluck_out_just_rating_param
             rating.save
+            JudgementFromRatingJob.perform_later current_user, rating
           end
         end
 

app/controllers/api/v1/queries/ratings_controller.rb

@@ -8,10 +8,11 @@ class RatingsController < Api::V1::Queries::ApplicationController
 
         def update
           @rating = @query.ratings.find_or_create_by doc_id: @doc_id
-          @rating.user = @current_user
+          # @rating.user = @current_user
 
           if @rating.update rating_params
             Analytics::Tracker.track_rating_created_event current_user, @rating
+            JudgementFromRatingJob.perform_later current_user, @rating
             respond_with @rating
           else
             render json: @rating.errors, status: :bad_request

app/controllers/books_controller.rb

@@ -5,11 +5,11 @@ class BooksController < ApplicationController
   before_action :set_book,
                 only: [ :show, :edit, :update, :destroy, :combine, :assign_anonymous, :delete_ratings_by_assignee,
                         :reset_unrateable, :reset_judge_later, :delete_query_doc_pairs_below_position,
-                        :eric_steered_us_wrong ]
+                        :eric_steered_us_wrong, :run_judge_judy ]
   before_action :check_book,
                 only: [ :show, :edit, :update, :destroy, :combine, :assign_anonymous, :delete_ratings_by_assignee,
                         :reset_unrateable, :reset_judge_later, :delete_query_doc_pairs_below_position,
-                        :eric_steered_us_wrong ]
+                        :eric_steered_us_wrong, :run_judge_judy ]
 
   before_action :find_user, only: [ :reset_unrateable, :reset_judge_later, :delete_ratings_by_assignee ]
 
@@ -24,10 +24,6 @@ def index
   # rubocop:disable Metrics/MethodLength
   def show
     @count_of_anonymous_book_judgements = @book.judgements.where(user: nil).count
-    @count_of_anonymous_case_judgements = 0
-    @book.cases.each do |kase|
-      @count_of_anonymous_case_judgements += kase.ratings.where(user: nil).count
-    end
 
     @moar_judgements_needed = @book.judgements.where(user: current_user).count < @book.query_doc_pairs.count
     @cases = @book.cases
@@ -189,13 +185,17 @@ def combine
                   :alert => "Could not merge due to errors: #{@book.errors.full_messages.to_sentence}. #{query_doc_pair_count} query/doc pairs."
     end
   end
+
+  def run_judge_judy
+    RunJudgeJudyJob.perform_later(@book)
+    redirect_to book_path(@book), :notice => "Started #{@book.ai_judge.fullname} judging query/doc pairs."
+  end
   # rubocop:enable Metrics/AbcSize
   # rubocop:enable Metrics/MethodLength
   # rubocop:enable Metrics/CyclomaticComplexity
   # rubocop:enable Metrics/PerceivedComplexity
   # rubocop:enable Layout/LineLength
 
-  # rubocop:disable Metrics/MethodLength
   def assign_anonymous
     # assignee = @book.team.members.find_by(id: params[:assignee_id])
     assignee = User.find_by(id: params[:assignee_id])
@@ -209,17 +209,16 @@ def assign_anonymous
         judgement.save!
       end
     end
-    @book.cases.each do |kase|
-      kase.ratings.where(user: nil).find_each do |rating|
-        rating.user = assignee
-        rating.save!
-      end
-    end
+    # @book.cases.each do |kase|
+    #  kase.ratings.where(user: nil).find_each do |rating|
+    #    rating.user = assignee
+    #    rating.save!
+    #  end
+    # end
 
     UpdateCaseJob.perform_later @book
     redirect_to book_path(@book), :notice => "Assigned #{assignee.fullname} to ratings and judgements."
   end
-  # rubocop:enable Metrics/MethodLength
 
   def delete_ratings_by_assignee
     judgements_to_delete = @book.judgements.where(user: @user)

app/jobs/judgement_from_rating_job.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class JudgementFromRatingJob < ApplicationJob
+  queue_as :default
+
+  def perform user, rating
+    query = rating.query
+    book = query.case.book
+    if book
+      query_doc_pair = book.query_doc_pairs.find_or_create_by query_text: rating.query.query_text, doc_id: rating.doc_id
+
+      judgement = query_doc_pair.judgements.find_or_initialize_by(user: user)
+      judgement.rating = rating.rating
+      judgement.save!
+
+      RunJudgeJudyJob.perform_later book
+    end
+  end
+end

app/jobs/populate_book_job.rb

@@ -36,7 +36,7 @@ def perform user, book, kase
         # we are smart and just look up the correct user id from rating.user_id via the database, no API data needed.
         judgement = query_doc_pair.judgements.find_or_create_by user_id: rating.user_id
         judgement.rating = pair[:rating]
-        judgement.user = rating.user
+        judgement.user = User.find(pair[:user_id]) # rating.user
         judgement.save!
       end
 
@@ -52,6 +52,8 @@ def perform user, book, kase
     book.populate_file.purge
     book.save
 
+    RunJudgeJudyJob.perform_later book
+
     Analytics::Tracker.track_query_doc_pairs_bulk_updated_event user, book, is_book_empty
   end
   # rubocop:enable Security/MarshalLoad

app/jobs/run_judge_judy_job.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class RunJudgeJudyJob < ApplicationJob
+  queue_as :default
+  sidekiq_options log_level: :warn
+
+  def perform book
+    #judge = book.ai_judge
+    judge = nil
+    if judge # only run the job if we have a judge defined
+      loop do
+        query_doc_pair = SelectionStrategy.random_query_doc_based_on_strategy(book, judge)
+        break if query_doc_pair.nil?
+
+        judgement = Judgement.new(query_doc_pair: query_doc_pair, user: judge, updated_at: Time.zone.now)
+        judgement.rating = 4
+        judgement.explanation = "Eric writing code.  Judge is #{judge.email}"
+        judgement.save!
+
+        UpdateCaseJob.perform_later book
+      end
+    end
+  end
+end

app/models/book.rb

@@ -32,6 +32,15 @@ class Book < ApplicationRecord
   belongs_to :owner,
              class_name: 'User', optional: true
 
+  #belongs_to :ai_judge,
+  #           class_name: 'User', optional: true
+  # 
+  #has_many :users, dependent: :destroy
+  #has_many :ai_judges, through: :ai_judges
+  has_and_belongs_to_many :ai_judges,
+                          class_name:              'User',
+                          join_table: 'ai_judges'
+
   belongs_to :selection_strategy
   belongs_to :scorer
   has_many :query_doc_pairs, dependent: :destroy, autosave: true

app/models/user.rb

@@ -22,8 +22,10 @@
 #  locked_at                   :datetime
 #  name                        :string(255)
 #  num_logins                  :integer
+#  openai_key                  :string(255)
 #  password                    :string(120)
 #  profile_pic                 :string(4000)
+#  prompt                      :string(4000)
 #  reset_password_sent_at      :datetime
 #  reset_password_token        :string(255)
 #  stored_raw_invitation_token :string(255)
@@ -100,7 +102,7 @@ class User < ApplicationRecord
   has_many :shared_scorers,
            through: :teams,
            source:  :scorers
-
+  
   has_many :permissions,
            dependent: :destroy
 
@@ -118,6 +120,9 @@ class User < ApplicationRecord
            dependent:  :destroy
 
   has_many :announcements, foreign_key: 'author_id', dependent: :destroy, inverse_of: :author
+
+  #has_many :ai_judges, dependent: :destroy
+  #has_many :books, through: :ai_judges
 
   # Validations
 
@@ -208,6 +213,11 @@ def store_raw_invitation_token
 
   # Scopes
   # default_scope -> { includes(:permissions) }
+  scope :only_ai_judges, -> { where('`users`.`openai_key` IS NOT NULL') }
+
+  def ai_judge?
+    return !openai_key.nil?
+  end
 
   def num_queries
     queries.count

app/services/ratings_manager.rb

@@ -30,7 +30,6 @@ def sync_ratings_for_case kase
   # rubocop:disable Metrics/AbcSize
   # rubocop:disable Metrics/MethodLength
   # rubocop:disable Metrics/PerceivedComplexity
-  # rubocop:disable Metrics/CyclomaticComplexity
   def sync_judgements_to_ratings kase, query_doc_pair
     query = Query.find_or_initialize_by(case: kase, query_text: query_doc_pair.query_text)
 
@@ -53,7 +52,9 @@ def sync_judgements_to_ratings kase, query_doc_pair
       summed_rating = query_doc_pair.judgements.rateable.sum(&:rating)
 
       rating = Rating.find_or_initialize_by(query: query, doc_id: query_doc_pair.doc_id)
-      rating.user = query_doc_pair.judgements.last.user if rating.user.nil?
+      # I think the whole rating having a user was bogus becasue we have multiple raters
+      # on the book side, which kind of messes this us.
+      # rating.user = query_doc_pair.judgements.last.user if rating.user.nil?
       rating.rating = if @book.support_implicit_judgements?
                         summed_rating / count_of_judgements
                       else
@@ -70,5 +71,4 @@ def sync_judgements_to_ratings kase, query_doc_pair
   # rubocop:enable Metrics/AbcSize
   # rubocop:enable Metrics/MethodLength
   # rubocop:enable Metrics/PerceivedComplexity
-  # rubocop:enable Metrics/CyclomaticComplexity
 end

app/views/books/_form.html.erb

@@ -51,6 +51,33 @@
     <%= form.label :selection_strategy_id, class: 'form-label' %>
     <%= form.collection_select(:selection_strategy_id, SelectionStrategy.all, :id, :descriptive_name, { prompt: true }, required: true, class: 'form-control') %>
   </div>
+
+  <div class="mb-3">
+    <%= form.label :ai_judge_id, class: 'form-label' %>
+    <%
+    ai_judges = book.ai_judges
+    %>
+    <%= form.collection_select(:ai_judge_id, ai_judges, :id, :name, { include_blank: true, prompt: true }, required: false, class: 'form-control') %>
+    <div class="form-text">You can specify an AI powered Judge to evaluate this book.  This list is derived from Judges created for your teams, so you must associate the book with a team first!</div>
+  </div>
+
+  <div class="mb-3">
+    <%= form_with(url: book_ai_judges_path(@book), method: :post) do |form| %>
+      <%= select_tag 'user_id', options_from_collection_for_select(@book.teams.collect{|team| team.members.only_ai_judges }.flatten, :id, :name) %>
+      <%= form.submit 'Add AI Judge' %>
+    <% end %>
+  </div>
+
+  <h3>AI Judges</h3>
+  <ul>
+    <% @book.ai_judges.each do |user| %>
+      <li>
+        <%= user.name %>
+        <%= button_to 'Remove', book_ai_judge_path(@book, user), method: :delete, class: 'btn btn-danger btn-sm' %>
+      </li>
+    <% end %>
+  </ul>
+
   <div class="mb-3">
     <%= form.label :show_rank, class: 'form-check-label' %>
     <%= form.check_box :show_rank, class: 'form-check-input' %>