HYC-1840 - Updates for ruby 3 (#1124)

* Updates for ruby 3

* Add webrick dependency, which was removed from ruby core in 3

* Update docker compose and startup scripts so that web and sidekiq don't share gems anymore (symlinking didn't work across containers), get solr working again

* Updates to syntax and tests for ruby 3

* Boolean name fixes and ruby 3 param adjustment in test

* Update passenger and sidekiq

* Downgrade to match server

* Downgrade sidekiq back to 6
---------

.github/workflows/build.yml

@@ -83,10 +83,10 @@ jobs:
 
     strategy:
       matrix:
-        ruby-version: ['2.7.4']
+        ruby-version: ['3.0.4']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Ruby
     # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
     # change this to (see https://github.com/ruby/setup-ruby#versioning):
@@ -97,7 +97,7 @@ jobs:
     - name: Update rubygems
       run: |
         gem update --system 3.3.25
-        gem install bundler:2.2.33
+        gem install bundler:2.3.27
 
     # Run Rubocop as soon as gems are installed, so we fail early if there are issues
     - name: Run RuboCop

Dockerfile

@@ -15,6 +15,8 @@ CMD ["/usr/sbin/init"]
 
 FROM systemd-enabled
 
+COPY docker/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo
+
 # Install dependencies
 # Install compilers for gems & more dependencies - this section is > 1 GB so might see if we can shrink it down some
 # Should we remove git, since the mutagen files can't see the git directory?
@@ -25,15 +27,13 @@ FROM systemd-enabled
 RUN yum -y update \
 && yum -y install epel-release \
 && yum -y install centos-release-scl-rh centos-release-scl \
-&& yum -y --enablerepo=centos-sclo-rh install rh-ruby27 rh-ruby27-ruby-devel \
+&& yum -y install libyaml libyaml-devel \
+&& yum -y --enablerepo=centos-sclo-rh install rh-ruby30 rh-ruby30-ruby-devel \
 && yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm \
 && yum -y install gcc gcc-c++ zlib-devel devtoolset-8 postgresql14 libpq5-devel libxslt-devel \
 && yum -y install git libreoffice clamav-devel clamav clamav-update clamd redhat-lsb libXScrnSaver wget unzip \
-&& yum -y install epel-release \
 && yum -y install ghostscript GraphicsMagick \
-&& wget -q -P /tmp "https://dl.google.com/linux/direct/google-chrome-stable_current_x86_64.rpm" \
-&& yum -y localinstall /tmp/google-chrome-stable_current_x86_64.rpm \
-&& rm -f google-chrome-stable_current_x86_64.rpm \
+&& yum -y install chromium \
 && yum -y install rh-nodejs14 \
 && yum install -y python3 \
 && yum clean all && rm -rf /var/cache/yum \
@@ -42,7 +42,7 @@ RUN yum -y update \
 && unzip /tmp/fits-1.5.5.zip -d /fits/fits-1.5.5 \
 && rm -f /tmp/fits-1.5.5.zip \
 && echo "source scl_source enable devtoolset-8" >> /etc/bashrc \
-&& echo "source scl_source enable rh-ruby27" >> /etc/bashrc \
+&& echo "source scl_source enable rh-ruby30" >> /etc/bashrc \
 && scl enable rh-nodejs14 -- npm install yarn -g \
 && scl enable rh-nodejs14 -- yarn \
 && yum -y update ca-certificates
@@ -54,11 +54,10 @@ COPY docker/fits.xml /fits/fits-1.5.5/xml/fits.xml
 COPY Gemfile* /hyrax/
 WORKDIR /hyrax
 
-RUN scl enable devtoolset-8 rh-ruby27 -- gem install rubygems-update \
-&& scl enable devtoolset-8 rh-ruby27 -- update_rubygems \
-&& scl enable devtoolset-8 rh-ruby27 -- gem update --system \
-&& scl enable devtoolset-8 rh-ruby27 -- gem install bundler:2.2.33 \
-&& scl enable devtoolset-8 rh-ruby27 -- bundle install --jobs=3 --retry=3
+ENV BUNDLER_ALLOW_ROOT=1
+RUN scl enable devtoolset-8 rh-ruby30 -- gem install rubygems-update \
+&& scl enable devtoolset-8 rh-ruby30 -- update_rubygems \
+&& scl enable devtoolset-8 rh-ruby30 -- gem update --system
 
 # Create FTP directories
 RUN mkdir -p /opt/data/ftp/proquest && mkdir -p /opt/data/ftp/sage

Gemfile

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 source 'https://rubygems.org'
-ruby '~> 2.7.4'
+ruby '~> 3.0.2'
 
 git_source(:github) do |repo_name|
   repo_name = "#{repo_name}/#{repo_name}" unless repo_name.include?('/')
@@ -9,6 +9,7 @@ end
 
 gem 'browser', '~> 5.3', '>= 5.3.1'
 gem 'active-fedora', '~> 14.0'
+gem 'base64', '0.1.0' # Downgrade from 0.2 so that it will match the version being used by passenger on the server
 gem 'blacklight', git: 'https://github.com/UNC-Libraries/blacklight.git', branch: 'unc-development'
 gem 'blacklight_advanced_search', '~> 8.0.0.alpha2'
 gem 'blacklight_oai_provider', '7.0.2'
@@ -20,7 +21,7 @@ gem 'clamav-client', require: 'clamav/client'
 gem 'coffee-rails', '~> 5.0.0'
 gem 'devise', '~> 4.8.0'
 gem 'devise-guests', '~> 0.8.1'
-gem 'edtf-humanize', '2.0.1'
+gem 'edtf-humanize', '~> 2.1'
 # See https://github.com/rails/execjs#readme for more supported runtimes
 gem 'execjs', '2.8.1'
 gem 'httparty', '~>0.21.0'
@@ -32,10 +33,11 @@ gem 'jbuilder', '~> 2.11.2'
 # Use jquery as the JavaScript library
 gem 'jquery-rails', '~> 4.5.0'
 gem 'json-canonicalization', '0.3.1'
-gem 'json-ld', '< 3.2'
+gem 'json-ld', '~> 3.2'
 gem 'libv8', '~> 7.3'
 # linkeddata gem is released with rdf gems and should be the same version as rdf
 gem 'sparql', '3.2.5'
+gem 'ldp', '~> 1.1'
 gem 'linkeddata'
 gem 'logger', '1.5.3'
 gem 'loofah', '~>2.19.1'
@@ -45,7 +47,8 @@ gem 'nokogiri', '~>1.14.2'
 gem 'omniauth', '~> 2.0'
 gem 'omniauth-rails_csrf_protection'
 gem 'omniauth-shibboleth', '~> 1.3'
-gem 'passenger', '6.0.14', require: 'phusion_passenger/rack_handler'
+# Cannot use 6.0.23 because it adds rackup as a dependency, which leads to conflicts for now
+gem 'passenger', '6.0.22', require: 'phusion_passenger/rack_handler'
 # Use postgresql as the database for Active Record
 gem 'pg', '~> 1.3.5'
 # Bundle edge Rails instead: gem 'rails', github: 'rails/rails'
@@ -96,6 +99,7 @@ group :test do
   gem 'selenium-webdriver', '~> 4.8'
   gem 'shoulda-matchers', '~> 5.3.0'
   gem 'simplecov'
+  gem 'webrick'
   gem 'webdrivers', '~> 5.3', require: false
   gem 'webmock', '~> 3.14.0'
 end

Gemfile.lock

@@ -201,7 +201,7 @@ GEM
     bagit (0.4.6)
       docopt (~> 0.5.0)
       validatable (~> 1.6)
-    base64 (0.2.0)
+    base64 (0.1.0)
     bcp47 (0.3.3)
       i18n
     bcrypt (3.1.20)
@@ -383,7 +383,7 @@ GEM
       unicode-types (~> 1.8)
     edtf (3.1.1)
       activesupport (>= 3.0, < 8.0)
-    edtf-humanize (2.0.1)
+    edtf-humanize (2.2.0)
       activesupport (>= 4)
       edtf (>= 2.3, < 4)
       roman (~> 0.2.0)
@@ -534,13 +534,13 @@ GEM
       thor (>= 0.14, < 2.0)
     json (2.7.1)
     json-canonicalization (0.3.1)
-    json-ld (3.1.10)
+    json-ld (3.2.4)
       htmlentities (~> 4.3)
-      json-canonicalization (~> 0.2)
+      json-canonicalization (~> 0.3)
       link_header (~> 0.0, >= 0.0.8)
-      multi_json (~> 1.14)
-      rack (~> 2.0)
-      rdf (~> 3.1)
+      multi_json (~> 1.15)
+      rack (>= 2.2, < 4)
+      rdf (~> 3.2, >= 3.2.10)
     json-ld-preloaded (3.1.6)
       json-ld (~> 3.1)
       rdf (~> 3.1)
@@ -568,13 +568,14 @@ GEM
       rdf-xsd (~> 3.2)
       sparql (~> 3.2)
       sxp (~> 1.2)
-    ldp (1.0.3)
+    ldp (1.2.0)
       deprecation
-      faraday
+      faraday (>= 1)
       http_logger
-      json-ld
-      rdf (>= 1.1)
+      json-ld (~> 3.2)
+      rdf (~> 3.2)
       rdf-isomorphic
+      rdf-ldp
       rdf-turtle
       rdf-vocab (>= 0.8)
       slop
@@ -703,7 +704,7 @@ GEM
       ast (~> 2.4.1)
       racc
     parslet (2.0.0)
-    passenger (6.0.14)
+    passenger (6.0.22)
       rack
       rake (>= 0.8.1)
     pg (1.3.5)
@@ -770,6 +771,9 @@ GEM
       rdf (~> 3.2)
     rdf-json (3.2.0)
       rdf (~> 3.2)
+    rdf-ldp (0.1.0)
+      deprecation
+      rdf
     rdf-microdata (3.2.1)
       htmlentities (~> 4.3)
       nokogiri (~> 1.13)
@@ -1066,6 +1070,7 @@ GEM
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
+    webrick (1.8.2)
     websocket (1.2.10)
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
@@ -1079,6 +1084,7 @@ PLATFORMS
 
 DEPENDENCIES
   active-fedora (~> 14.0)
+  base64 (= 0.1.0)
   blacklight!
   blacklight_advanced_search (~> 8.0.0.alpha2)
   blacklight_oai_provider (= 7.0.2)
@@ -1093,7 +1099,7 @@ DEPENDENCIES
   database_cleaner
   devise (~> 4.8.0)
   devise-guests (~> 0.8.1)
-  edtf-humanize (= 2.0.1)
+  edtf-humanize (~> 2.1)
   execjs (= 2.8.1)
   factory_bot_rails (~> 6.2.0)
   fcrepo_wrapper (~> 0.9.0)
@@ -1105,7 +1111,8 @@ DEPENDENCIES
   jbuilder (~> 2.11.2)
   jquery-rails (~> 4.5.0)
   json-canonicalization (= 0.3.1)
-  json-ld (< 3.2)
+  json-ld (~> 3.2)
+  ldp (~> 1.1)
   libv8 (~> 7.3)
   linkeddata
   listen (~> 3.7.0)
@@ -1117,7 +1124,7 @@ DEPENDENCIES
   omniauth (~> 2.0)
   omniauth-rails_csrf_protection
   omniauth-shibboleth (~> 1.3)
-  passenger (= 6.0.14)
+  passenger (= 6.0.22)
   pg (~> 1.3.5)
   puma
   rails (~> 6.0)
@@ -1147,9 +1154,10 @@ DEPENDENCIES
   web-console (~> 3.7.0)
   webdrivers (~> 5.3)
   webmock (~> 3.14.0)
+  webrick
 
 RUBY VERSION
-   ruby 2.7.4p191
+   ruby 3.0.2p107
 
 BUNDLED WITH
    2.2.33

app/overrides/jobs/attach_files_to_work_job_override.rb

@@ -9,7 +9,7 @@ def perform(work, uploaded_files, **work_attributes)
       # [hyc-override] check all files for viruses
       virus_check!(uploaded_file)
     end
-    original_perform(work, uploaded_files, work_attributes)
+    original_perform(work, uploaded_files, **work_attributes)
   # [hyc-override] Log viruses
   rescue VirusDetectedError => error
     user = User.find_by_user_key(proxy_or_depositor(work))

app/services/tasks/dimensions_ingest_service.rb

@@ -158,7 +158,7 @@ def extract_pdf(publication)
         headers['Wiley-TDM-Client-Token'] = "#{@wiley_tdm_api_token}"
       else
         # Use the encoded linkout URL from dimensions otherwise
-        encoded_url = URI.encode(pdf_url)
+        encoded_url = URI::DEFAULT_PARSER.escape(pdf_url)
       end
       download_pdf(encoded_url, publication, headers)
     end

docker-compose.yml

@@ -42,7 +42,7 @@ services:
       - docker/.env
     volumes:
       - mount-code:/hyrax
-      - mount-gems:/hyc-gems
+      - mount-sidekiq-gems:/hyc-gems
       - fcrepo_data:/opt/fedora/
       - hyrax_data:/opt/hyrax
       - ftp_data:/opt/data/ftp
@@ -95,20 +95,22 @@ services:
   solr8:
     image: solr:8.11-slim
     volumes:
-      - solr8-data:/opt/solr/server/solr/configsets/hy-c
+      - ./solr/config:/opt/solr/server/solr/configsets/hy-c
+      - solr8-data:/var/solr/data
     ports:
       - "8983:8983"
     command: >
       bash -c "
-      precreate-core hydra-development /opt/solr/server/solr/configsets/hy-c;
-      precreate-core hydra-test /opt/solr/server/solr/configsets/hy-c;
+      precreate-core hydra-development /opt/solr/server/solr/configsets/hy-c &&
+      precreate-core hydra-test /opt/solr/server/solr/configsets/hy-c &&
       exec solr -f
       "
     networks:
       - hycdev
 volumes:
   mount-code:
   mount-gems:
+  mount-sidekiq-gems:
   solr8-data:
   clam:
   pg_data:
@@ -138,14 +140,3 @@ x-mutagen:
         permissions:
           defaultFileMode: 0644
           defaultDirectoryMode: 0755
-    #mount-gems:
-      #alpha: '../hyc-gems'
-      #beta: "volume://mount-gems"
-    # TODO: can we move these to simple volumes by auto-copying the configs over?
-    solr8-data:
-      alpha: './solr/config'
-      beta: 'volume://solr8-data'
-      configurationBeta:
-        permissions:
-          defaultOwner: 'id:8983'
-          defaultGroup: 'id:8983'

docker/CentOS-Base.repo

@@ -0,0 +1,54 @@
+#
+# The mirror system uses the connecting IP address of the client and the
+# update status of each mirror to pick mirrors that are updated to and
+# geographically close to the client.  You should use this for CentOS updates
+# unless you are manually picking other mirrors.
+#
+# If the mirrorlist= does not work for you, as a fall back you can try the
+# remarked out baseurl= line instead.
+#
+#
+
+[base]
+name=CentOS-$releasever - Base
+#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os&infra=$infra
+baseurl=http://vault.centos.org/centos/$releasever/os/$basearch/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+
+#released updates
+[updates]
+name=CentOS-$releasever - Updates
+#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=updates&infra=$infra
+baseurl=http://vault.centos.org/centos/$releasever/updates/$basearch/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+
+[centos-sclo-rh]
+name=CentOS-7.9.2009 - SCLo rh
+baseurl=https://vault.centos.org/7.9.2009/sclo/$basearch/rh/
+gpgcheck=0
+enabled=1
+
+[centos-sclo-sclo]
+name=CentOS-7.9.2009 - SCLo sclo
+baseurl=https://vault.centos.org/7.9.2009/sclo/$basearch/sclo/
+gpgcheck=0
+enabled=1
+
+#additional packages that may be useful
+[extras]
+name=CentOS-$releasever - Extras
+#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=extras&infra=$infra
+baseurl=http://vault.centos.org/centos/$releasever/extras/$basearch/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+
+#additional packages that extend functionality of existing packages
+[centosplus]
+name=CentOS-$releasever - Plus
+#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=centosplus&infra=$infra
+baseurl=http://vault.centos.org/centos/$releasever/centosplus/$basearch/
+gpgcheck=1
+enabled=0
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7