v9.3 Controller Changes:

- Callbacks added to API - Moved ACME validation IP to availability zones - Added Backups and Volume API endpoints - Added region and availability zone information in project and services api. - Added YJIT and JEMALLOC to controller container for better memory management. - Improvements to the development environment, including support for ARM. See full changelog (v9.1.2..v9.3.0) for full details of changes.
ComputeStacks · Apr 21, 2024 · d627b15 · d627b15
1 parent d957779
commit d627b15
Show file tree

Hide file tree

Showing 96 changed files with 1,679 additions and 488 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -7,16 +7,20 @@ workflow:
       when: never
     - when: always
 
-##
-# Set at job
-# CS_MINOR_VERSION
-# CS_MAJOR_VERSION
 variables:
+  DOCKER_TLS_CERTDIR: "/certs"
+  FF_NETWORK_PER_BUILD: "true"
   FULL_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-$CI_PIPELINE_ID
+  POSTGRES_DB: cstacks
+  POSTGRES_USER: cstacks
+  POSTGRES_PASSWORD: cstacks
+  REDIS_HOST: redis
+  DATABASE_URL: "postgres://cstacks:cstacks@postgres/cstacks"
+  CS_MINOR_VERSION: "9.3"
+  CS_MAJOR_VERSION: "9"
 
 default:
-  tags:
-    - shell
+  image: docker:25
   before_script:
     - echo "$CI_REGISTRY_PASSWORD" | docker login $CI_REGISTRY --username $CI_REGISTRY_USER --password-stdin
   after_script:
@@ -25,20 +29,15 @@ default:
 
 stages:
   - build
-  - deploy
 
 build:
   stage: build
-  before_script:
-    - 'docker run -d --rm --name cstacks-pg -e POSTGRES_USER=cstacks -e POSTGRES_DB=cstacks -e POSTGRES_PASSWORD=cstacks postgres:16'
-    - 'docker run -d --rm --name cstacks-redis --network "container:cstacks-pg" redis:alpine'
-  after_script:
-    - docker stop cstacks-redis cstacks-pg
-  script:
-    - 'docker build --network "container:cstacks-pg" --progress plain --build-arg github_user=$GITHUB_GEM_PULL_USER --build-arg github_token=$GITHUB_GEM_PULL_TOKEN -t $FULL_IMAGE .'
-deploy:
-  stage: deploy
+  services:
+    - docker:25-dind
+    - postgres:16
+    - redis:alpine
   script:
+    - "docker build --network=host --add-host=redis:$(getent hosts redis  | awk '{ print $1 }') --add-host=postgres:$(getent hosts postgres  | awk '{ print $1 }') --cache-from $CI_REGISTRY_IMAGE:$CS_MAJOR_VERSION --progress plain --build-arg db_url=$DATABASE_URL --build-arg redis_host=$REDIS_HOST -t $FULL_IMAGE ."
     - "docker tag $FULL_IMAGE $CI_REGISTRY_IMAGE:$CS_MINOR_VERSION"
     - "docker tag $FULL_IMAGE $CI_REGISTRY_IMAGE:$CS_MAJOR_VERSION"
     - "docker tag $FULL_IMAGE $CI_REGISTRY_IMAGE:latest"

diff --git a/.rubocop.yml b/.rubocop.yml
@@ -1,5 +1,15 @@
+Metrics/AbcSize:
+  Enabled: False
+Metrics/ClassLength:
+  Enabled: false
+Metrics/CyclomaticComplexity:
+  Enabled: false
 Metrics/LineLength:
   Max: 100
+Metrics/MethodLength:
+  Max: 100
+Metrics/PerceivedComplexity:
+  Enabled: false
 Style/TernaryParentheses:
   Enabled: false
 Layout/FirstParameterIndentation:
@@ -14,3 +24,4 @@ Layout/SpaceInsideParens:
   Enabled: false
 Layout/SpaceInsideArrayLiteralBrackets:
   Enabled: false
+
diff --git a/.ruby-version b/.ruby-version
@@ -1 +1 @@
-3.2.3
+3.3.0
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,57 @@
 # Change Log
 
+## v9.3.0
+
+- [FEATURE] Introducing Callbacks for the API. This allows you to receive a webhook from ComputeStacks when a api request is completed by a background worker.
+- [FEATURE] Move acme validation IP to a database field on the region. This allows different IPs for different regions.
+
+## v9.2.2
+
+- [FEATURE] Added volume and backups api.
+
+## v9.2.1
+
+- [FEATURE] `on_latest_image` Boolean field added to container and bastion api calls. If false, there is a new image on the node and a rebuild will use the new image.
+
+## v9.2.0
+
+**YJIT and JEMALLOC for ComputeStacks Production Environments**
+
+Please add the following to your `/etc/default/computestacks` file:
+
+```
+MALLOC_CONF=dirty_decay_ms:1000,narenas:2,background_thread:true,stats_print:false
+RUBY_YJIT_ENABLE=1
+```
+
+And download the latest `cstacks` helper script onto the controller with this command:
+
+```bash
+wget -O /usr/local/bin/cstacks https://raw.githubusercontent.com/ComputeStacks/ansible-install/main/roles/controller/files/cstacks.sh \
+  && chmod +x /usr/local/bin/cstacks
+```
+
+Please see[v92 Notes.md](./doc/v92 Notes.md) for more details.
+
+**Major Changes to the development environment**
+
+This release includes major changes to how our development environment is configured.
+
+- The controller, postgres, and redis are now run by calling `docker compose up -d` locally on your development machine. This means you'll need to have docker installed locally.
+- Our vagrant image is now much closer to a production compute node, in that it only runs the containers as a node and no longer runs any of the controller.
+- The controller's gemfile no longer pulls the gem from Github, but just does a git clone. This removes the requirement of having a github account to build the controller.
+
+See [DEV_SETUP.md](./doc/DEV_SETUP.md) for specific instructions.
+
+***
+
+* [CHANGE] Upgrade ruby to 3.3, and rails to 7.1, and fix deprecations and bugs introduced with this upgrade.
+* [CHANGE] Remove Rails.secrets in favor of environmental variables.
+* [CHANGE] Disable marketplace reporting. This project is on an indefinite hold.
+* [FIX] Updated monarx integration to resolve api issues.
+
+***
+
 ## v9.1.2
 
 * [FIX] Don't attempt node evacuation with local networking.

diff --git a/Dockerfile b/Dockerfile
@@ -1,25 +1,26 @@
-FROM ruby:3.2
+FROM ruby:3.3
 
 LABEL maintainer="https://github.com/ComputeStacks"
 LABEL org.opencontainers.image.authors="https://github.com/ComputeStacks"
 LABEL org.opencontainers.image.source="https://github.com/ComputeStacks/controller"
 LABEL org.opencontainers.image.url="https://github.com/ComputeStacks/controller"
 LABEL org.opencontainers.image.title="ComputeStacks Controller"
 
+ARG db_url="postgresql://cstacks:cstacks@localhost/cstacks?pool=30"
+ARG redis_host=localhost
+
+ENV DATABASE_URL=$db_url
+ENV REDIS_HOST=$redis_host
+
 ENV RACK_ENV=production
 ENV RAILS_ENV=production
 ENV SECRET_KEY_BASE=3a698257a1e32f1bb9b3fd861640c3b53cc9c57dd40b3fa360fed44d2e5da3fdb3351db2f8c881f2a04e6a7ca7e721de67d98061ffa7d394d3ad1c24ce9e09ec
 ENV USER_AUTH_SECRET=3a698257a1e32f1bb9b3fd861640c3b53cc9c57dd40b3fa360fed44d2e5da3fdb3351db2f8c881f2a04e6a7ca7e721de67d98061ffa7d394d3ad1c24ce9e09ec
 ENV LOCALE=en
 ENV CURRENCY=USD
-ENV DATABASE_URL="postgresql://cstacks:cstacks@localhost/cstacks?pool=30"
-ENV REDIS_HOST=localhost
 ENV RUBYOPT='-W:no-deprecated'
 ENV APP_ID=build
 
-ARG github_user
-ARG github_token
-
 RUN set -ex; \
     \
     apt-get update; \
@@ -62,14 +63,11 @@ WORKDIR /usr/src/app
 ADD Gemfile* /usr/src/app/
 ADD engines/ /usr/src/app/engines/
 
-RUN bundle config https://rubygems.pkg.github.com/ComputeStacks $github_user:$github_token \
-                && bundle config set without 'test development' \
-        ; \
-        cd /usr/src/app \
-                && if [ ! -f Gemfile ]; then mv Gemfile.common Gemfile; fi \
-                && bundle install \
-        ; \
-        bundle config --delete https://rubygems.pkg.github.com/computestacks/
+RUN bundle config set without 'test development' \
+      ; \
+      cd /usr/src/app \
+        && if [ ! -f Gemfile ]; then mv Gemfile.common Gemfile; fi \
+        && bundle install
 
 COPY lib/build/nginx.conf /etc/nginx/sites-enabled/default
 COPY lib/build/supervisord.conf /etc/supervisord.conf

diff --git a/Gemfile.common b/Gemfile.common
@@ -1,81 +1,81 @@
-ruby "~> 3.2"
+# vim syntax=ruby
+ruby "~> 3.3"
 
 source 'https://rubygems.org'
 
-git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
-gem 'timeout', '= 0.3.2'
+gem 'timeout', '~> 0.4'
 
-gem 'rails', '= 7.0.6'
-gem 'json', '= 2.6.3'
+gem 'rails', '~> 7.1'
+gem 'json', '~> 2.7'
 
 # Redis
 gem "hiredis", "< 1"
 gem "redis", "< 6"
 gem 'redis-client', '< 1' # sidekiq
 
-gem 'rack-attack', '~> 6.6'
+gem 'rack-attack', '~> 6.7'
 
 gem 'tzinfo-data' # No source of timezone data could be found. (TZInfo::DataSourceNotFound)
 
 gem 'rake', '= 13.0.6'
 gem 'sprockets', '= 4.2.0'
 gem 'sprockets-rails', '= 3.4.2'
 gem 'importmap-rails', '~> 1'
-gem 'pg', '= 1.5.3', platform: :ruby
+gem 'pg', '= 1.5.5', platform: :ruby
 
 gem 'httparty', '= 0.21.0' # @deprecated for httprb
-gem 'http', '= 5.1.1'
+gem 'http', '~> 5.2'
 
-gem 'geoip', '= 1.6.4'
+gem 'geoip', '~> 1'
 
-gem 'will_paginate', '= 3.3.1'
-gem 'api-pagination', '= 5.0.0'
+gem 'will_paginate', '~> 4'
+gem 'api-pagination', '~> 5'
 
-gem 'money-rails', '= 1.15.0'
-gem 'whois', '= 5.1.0'
+gem 'money-rails', '~> 1'
+gem 'whois', '~> 5'
 
 gem 'sidekiq', '< 8'
 gem 'sidekiq-unique-jobs', '< 9'
 
 # Logging
-gem 'lograge', '= 0.12.0'
+gem 'lograge', '~> 0.14'
 
 # Authentication
-gem 'devise', '= 4.9.2'
-gem 'bcrypt', '= 3.1.18'
+gem 'devise', '= 4.9.3'
+gem 'bcrypt', '= 3.1.20'
 
-gem "doorkeeper", "= 5.6.6"
+gem "doorkeeper", "= 5.6.9"
 
-gem 'rotp', '= 6.2.2'
-gem 'rqrcode', '= 2.1.2'
-gem 'webauthn', '= 3.0.0'
+gem 'rotp', '~> 6.3'
+gem 'rqrcode', '~> 2.2'
+gem 'webauthn', '~> 3.1'
 
 gem 'clockwork', '= 3.0.2'
 gem 'http_accept_language', '= 2.1.1'
 
 gem 'nokogiri', '~> 1'
 
-gem 'country_select', '= 8.0.1'
-gem 'responders', '= 3.1.0'
+gem 'country_select', '~> 9'
+gem 'responders', '~> 3.1'
 
 gem 'daemons', '= 1.4.1'
 gem 'net-smtp'
-gem 'slim', '= 5.1.0'
-# gem 'uglifier', '= 4.2.0'
+gem 'slim', '~> 5.2'
 
-gem 'chartkick', '= 5.0.2'
-gem 'groupdate', '= 6.2.1'
+gem 'chartkick', '~> 5'
+gem 'groupdate', '~> 6'
 
-gem 'ssh_data', '= 1.3.0' # Generate & Validate SSH Keys
-gem 'net-ssh', '= 7.1.0'
-gem 'ed25519', '= 1.3.0'
-gem 'bcrypt_pbkdf', '= 1.1.0'  # Support ed25519 ssh keys for net-ssh.
-gem 'highline', '= 2.1.0' # Hide PW entered into net-ssh from logs and output.
+gem 'ssh_data', '~> 1' # Generate & Validate SSH Keys
+gem 'net-ssh', '~> 7'
+gem 'ed25519', '~> 1.3'
+gem 'bcrypt_pbkdf', '~> 1.1'  # Support ed25519 ssh keys for net-ssh.
+gem 'highline', '~> 3' # Hide PW entered into net-ssh from logs and output.
 
 # Markdown Support
-gem 'github-markup', '= 4.0.1'
-gem 'redcarpet', '= 3.6.0', platform: :ruby
+gem 'github-markup', '~> 4.0'
+gem 'redcarpet', '~> 3.6', platform: :ruby
 
 
 gem "sentry-ruby"
@@ -86,26 +86,26 @@ gem 'sass-rails', '= 6.0.0'
 gem 'bootstrap-sass', '= 3.4.1'
 
 gem 'rabl', '= 0.16.1'
-gem 'oj', '= 3.14.3', platform: :ruby
-gem 'jwt', '= 2.7.0'
+gem 'oj', '= 3.16.3', platform: :ruby
+gem 'jwt', '= 2.8.0'
 
 # LetsEncrypt
-gem 'acme-client', '= 2.0.13'
+gem 'acme-client', '= 2.0.17'
 
 gem 'versioncake', '= 4.1.1'
 
 # Console
 gem 'pry'
 gem 'pry-rails'
 
-gem 'diplomat', '= 2.6.4'
+gem 'diplomat', '~> 2.6'
 
-gem 'fugit', '= 1.8.1'
-gem 'zaru', '= 1.0.0'
+gem 'fugit', '~> 1.9'
+gem 'zaru', '~> 1.0'
 
-gem 'acts-as-taggable-on', '= 9.0.1'
+gem 'acts-as-taggable-on', '~> 10.0'
 
-gem 'liquid', '= 5.4.0' # Used to generate variables in custom commands
+gem 'liquid', '~> 5.4' # Used to generate variables in custom commands
 
 ##
 # Used by CAA check to determine TLD for a domain.
@@ -139,15 +139,13 @@ group :test do
   gem 'webmock'
 end
 
-source "https://rubygems.pkg.github.com/computestacks" do
-  gem "autodns", "2.1.1"
-  gem "docker_registry", "0.3.4"
-  gem "docker_ssh", "0.7.1"
-  gem "docker_volume_local", "0.2.5"
-  gem "docker_volume_nfs", "0.2.7"
-  gem "pdns", "1.1.1"
-  gem "whmcs", "2.3.7"
-end
-
+# ComputeStack Gems
+gem 'autodns', github: 'ComputeStacks/autodns-ruby', ref: 'd045f63'
 gem 'docker-api', github: 'ComputeStacks/docker-api', ref: '499e609'
+gem 'docker_registry', github: 'ComputeStacks/docker-registry-ruby', ref: 'b69fb48'
+gem 'docker_ssh', github: 'ComputeStacks/docker-ssh-ruby', ref: '0b01ab8'
+gem 'docker_volume_local', github: 'ComputeStacks/docker-volume-local', ref: 'b6f066a'
+gem 'docker_volume_nfs', github: 'ComputeStacks/docker-volume-nfs', ref: '2614c82'
+gem 'pdns', github: 'ComputeStacks/powerdns-ruby', ref: '21a31b9'
+gem 'whmcs', github: 'ComputeStacks/whmcs-ruby', ref: '7dc4f4e'
 
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-9.1.2
+9.3.0
diff --git a/Vagrantfile.sample b/Vagrantfile.sample
@@ -10,7 +10,7 @@ Vagrant.configure("2") do |config|
   config.vm.provider :libvirt do |vm|
     vm.driver = "kvm"
     vm.title = "computestacks"
-    vm.description = "ComputeStacks Controller"
+    vm.description = "ComputeStacks Node"
     vm.memory = 4096
     vm.cpus = 4
     vm.cputopology sockets: "1", cores: "2", threads: "2"

diff --git a/app/controllers/admin/regions_controller.rb b/app/controllers/admin/regions_controller.rb
@@ -52,15 +52,15 @@ def region_create_params
     params.require(:region).permit(
       :name, :active, :fill_to, :metric_client_id, :loki_endpoint, :log_client_id, :loki_retries, :loki_batch_size,
       :volume_backend, :nfs_remote_host, :nfs_remote_path, :offline_window, :failure_count, :nfs_controller_ip,
-      :disable_oom, :pid_limit, :ulimit_nofile_hard, :ulimit_nofile_soft, :location_id, :p_net_size, :network_driver
+      :disable_oom, :pid_limit, :ulimit_nofile_hard, :ulimit_nofile_soft, :location_id, :p_net_size, :network_driver, :acme_server
     )
   end
 
   def region_update_params
     params.require(:region).permit(
       :active, :fill_to, :metric_client_id, :loki_endpoint, :log_client_id, :loki_retries, :loki_batch_size,
       :volume_backend, :nfs_remote_host, :nfs_remote_path, :offline_window, :failure_count, :nfs_controller_ip,
-      :disable_oom, :pid_limit, :ulimit_nofile_hard, :ulimit_nofile_soft, :p_net_size, :network_driver
+      :disable_oom, :pid_limit, :ulimit_nofile_hard, :ulimit_nofile_soft, :p_net_size, :network_driver, :acme_server
     )
   end