Update search-engine and API resource allocations

Search-engine: - Upgrade to Solr 8.11.2. - Update load tests with real world heavy load queries. - Refactor load tests to exercise the search-engine more. - Ensure each search query requests a random page within a particular page range. This helps to minimize the number of cache hits, and therefore exercise the heap and garbage collection settings of the search-engine. - Solr uses SOLR_JAVA_MEM to set the JAVA memory options, which overrides JAVA_OPTS. - Adjust CPU and memory allocation to minimize container throttling and thrashing under load. - Adjust health checks to reduce loading. API: - Adjust API resources to minimize container throttling and OOM issues during heavier load. Signed-off-by: Wade Barnes <wade@neoterictech.ca>
bcgov · Oct 25, 2023 · 37cf707 · 37cf707
1 parent 7b32833
commit 37cf707
Show file tree

Hide file tree

Showing 15 changed files with 99 additions and 70 deletions.
diff --git a/openshift/manage b/openshift/manage
@@ -1233,17 +1233,46 @@ function loadTest()
 
  echo
  echo
- for page in {1..5000}
+
+ range="248818-253818"
+ for count in {1..5000}
  do
- echo -e "\e[1A\e[KReading page ${page} ..."
- curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/topic?inactive=any&latest=any&ordering=first_effective_date&page=${page}&revoked=any&type_id=1"
- # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/search/credential/facets?page=${page}"
- # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/topic?page=${page}"
+ echo -e "\e[1A\e[KReading page ${count} ..."
+
+ if ((${count}%2)); then
+ # /api/v4
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/credential?format=json&ordering=-score&page=$(shuf -i ${range} -n 1)"
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/credential?format=json&ordering=revoked_date&page=$(shuf -i ${range} -n 1)"
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/credential?format=json&ordering=score&page=$(shuf -i ${range} -n 1)"
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/credential?format=json&credential_type_id=1&inactive=any&ordering=-effective_date&page=$(shuf -i ${range} -n 1)"
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/credential?format=json&ordering=-revoked_date&page=$(shuf -i ${range} -n 1)"
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/credential?format=json&credential_type_id=1&inactive=any&ordering=effective_date&page=$(shuf -i ${range} -n 1)"
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/topic?format=json&ordering=-effective_date&page=$(shuf -i ${range} -n 1)"
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/credential?ordering=-score&page=$(shuf -i ${range} -n 1)"
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/topic?format=json&ordering=-score&page=$(shuf -i ${range} -n 1)"
+ else
+ # /api/v3
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v3/search/topic?format=json&topic_id=$(shuf -i ${range} -n 1)"
+
+ # /api/v2
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/search/autocomplete?format=json&ordering=score&page=$(shuf -i ${range} -n 1)"
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/search/credential/facets?format=json&page=$(shuf -i ${range} -n 1)"
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/search/credential/topic/facets?format=json&id=$(shuf -i ${range} -n 1)"
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/search/credential?format=json&id=$(shuf -i ${range} -n 1)"
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/search/autocomplete?format=json&ordering=revoked_date&page=$(shuf -i ${range} -n 1)"
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/search/autocomplete?format=json&ordering=-score&page=$(shuf -i ${range} -n 1)"
+ curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/search/autocomplete?ordering=-score&page=$(shuf -i ${range} -n 1)"
+ fi
+
+ # Other
+ # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/topic/facets?q=Mayne+Island+School+Parent+Advisory+Council&inactive=&category:entity_type=&credential_type_id=&page=$(shuf -i ${range} -n 1)&revoked=false"
+ # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/topic?inactive=any&latest=any&ordering=first_effective_date&page=$(shuf -i ${range} -n 1)&revoked=any&type_id=1"
+ # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/topic?page=$(shuf -i ${range} -n 1)"
 
  # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/quickload"
- # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/topic?page=${page}"
- # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/topic?page=${page}"
- sleep 0.25
+ # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/topic?page=$(shuf -i ${range} -n 1)"
+ # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/topic?page=$(shuf -i ${range} -n 1)"
+ # sleep 0.25
  done
  )
 }

diff --git a/openshift/templates/api/api-deploy.bc.dev.param b/openshift/templates/api/api-deploy.bc.dev.param
@@ -53,8 +53,8 @@ SP_TRACKING_APP_ID=orgbook_api_dev
 # AGENT_HOST=agent
 # AGENT_ADMIN_PORT=8024
 # MIN_REPLICAS=2
-# MAX_REPLICAS=20
-# CPU_REQUEST=150m
-# CPU_LIMIT=300m
+# MAX_REPLICAS=12
+# CPU_REQUEST=300m
+# CPU_LIMIT=1000m
 # MEMORY_REQUEST=256Mi
-# MEMORY_LIMIT=512Mi
+# MEMORY_LIMIT=2Gi
diff --git a/openshift/templates/api/api-deploy.bc.param b/openshift/templates/api/api-deploy.bc.param
@@ -53,8 +53,8 @@ DEMO_SITE=True
 AGENT_HOST=agent
 AGENT_ADMIN_PORT=8024
 MIN_REPLICAS=2
-MAX_REPLICAS=20
-CPU_REQUEST=150m
-CPU_LIMIT=300m
+MAX_REPLICAS=12
+CPU_REQUEST=300m
+CPU_LIMIT=1000m
 MEMORY_REQUEST=256Mi
-MEMORY_LIMIT=512Mi
+MEMORY_LIMIT=2Gi
diff --git a/openshift/templates/api/api-deploy.bc.prod.param b/openshift/templates/api/api-deploy.bc.prod.param
@@ -53,8 +53,8 @@ DEMO_SITE=False
 # AGENT_HOST=agent
 # AGENT_ADMIN_PORT=8024
 MIN_REPLICAS=6
-MAX_REPLICAS=12
-CPU_REQUEST=300m
-CPU_LIMIT=1000m
-MEMORY_REQUEST=256Mi
-MEMORY_LIMIT=768Mi
+MAX_REPLICAS=10
+# CPU_REQUEST=300m
+# CPU_LIMIT=1000m
+# MEMORY_REQUEST=256Mi
+# MEMORY_LIMIT=2Gi
diff --git a/openshift/templates/api/api-deploy.bc.test.param b/openshift/templates/api/api-deploy.bc.test.param
@@ -53,8 +53,8 @@ SP_TRACKING_APP_ID=orgbook_api_test
 # AGENT_HOST=agent
 # AGENT_ADMIN_PORT=8024
 # MIN_REPLICAS=2
-# MAX_REPLICAS=20
-# CPU_REQUEST=150m
-# CPU_LIMIT=300m
+# MAX_REPLICAS=12
+# CPU_REQUEST=300m
+# CPU_LIMIT=1000m
 # MEMORY_REQUEST=256Mi
-# MEMORY_LIMIT=512Mi
+# MEMORY_LIMIT=2Gi
diff --git a/openshift/templates/search-engine/search-engine-base-build.yaml b/openshift/templates/search-engine/search-engine-base-build.yaml
@@ -72,7 +72,7 @@ parameters:
  displayName: Git Reference
  description: The git reference or branch.
  required: true
- value: "8.8.3"
+ value: "8.11.2"
  - name: SOURCE_CONTEXT_DIR
  displayName: Source Context Directory
  description: The source context directory.
@@ -87,7 +87,7 @@ parameters:
  displayName: Source Image Name
  description: The name of the Docker image to be used for the build. Overrides the FROM instruction in the Dockerfile.
  required: false
- value: solr:8.8.1
+ value: solr:8.11.2
  - name: OUTPUT_IMAGE_TAG
  displayName: Output Image Tag
  description: The tag given to the built image.

diff --git a/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.dev.param b/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.dev.param
@@ -22,9 +22,9 @@ TAG_NAME=dev
 # MOUNT_PATH=/var/solr/data
 # CORE_NAME=credential_registry
 # SOLR_LOG_LEVEL=WARN
-# LOG4J_FORMAT_MSG_NO_LOOKUPS=true
-# JAVA_OPTS=-Xms256m -Xmx512m
+# SOLR_JAVA_MEM=-Xms256m -Xmx1024m
+# GC_TUNE=-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75
 # CPU_REQUEST=100m
 # CPU_LIMIT=750m
 # MEMORY_REQUEST=256Mi
-# MEMORY_LIMIT=3000Mi
+# MEMORY_LIMIT=2000Mi
diff --git a/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.param b/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.param
@@ -22,9 +22,9 @@ PERSISTENT_VOLUME_CLASS=netapp-block-standard
 MOUNT_PATH=/var/solr/data
 CORE_NAME=credential_registry
 SOLR_LOG_LEVEL=WARN
-LOG4J_FORMAT_MSG_NO_LOOKUPS=true
-JAVA_OPTS=-Xms256m -Xmx512m
+SOLR_JAVA_MEM=-Xms256m -Xmx1024m
+GC_TUNE=-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75
 CPU_REQUEST=100m
 CPU_LIMIT=750m
 MEMORY_REQUEST=256Mi
-MEMORY_LIMIT=3000Mi
+MEMORY_LIMIT=2000Mi
diff --git a/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.prod.param b/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.prod.param
@@ -22,9 +22,9 @@ TAG_NAME=prod
 # MOUNT_PATH=/var/solr/data
 # CORE_NAME=credential_registry
 # SOLR_LOG_LEVEL=WARN
-# LOG4J_FORMAT_MSG_NO_LOOKUPS=true
-# JAVA_OPTS=-Xms256m -Xmx512m
+# SOLR_JAVA_MEM=-Xms256m -Xmx1024m
+# GC_TUNE=-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75
 # CPU_REQUEST=100m
 # CPU_LIMIT=750m
 # MEMORY_REQUEST=256Mi
-# MEMORY_LIMIT=3000Mi
+# MEMORY_LIMIT=2000Mi
diff --git a/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.test.param b/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.test.param
@@ -21,9 +21,9 @@ TAG_NAME=test
 # MOUNT_PATH=/var/solr/data
 # CORE_NAME=credential_registry
 # SOLR_LOG_LEVEL=WARN
-# LOG4J_FORMAT_MSG_NO_LOOKUPS=true
-# JAVA_OPTS=-Xms256m -Xmx512m
+# SOLR_JAVA_MEM=-Xms256m -Xmx1024m
+# GC_TUNE=-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75
 # CPU_REQUEST=100m
 # CPU_LIMIT=750m
 # MEMORY_REQUEST=256Mi
-# MEMORY_LIMIT=3000Mi
+# MEMORY_LIMIT=2000Mi
diff --git a/openshift/templates/search-engine/search-engine-deploy.bc.dev.param b/openshift/templates/search-engine/search-engine-deploy.bc.dev.param
@@ -22,9 +22,9 @@ TAG_NAME=dev
 # MOUNT_PATH=/var/solr/data
 # CORE_NAME=credential_registry
 # SOLR_LOG_LEVEL=WARN
-# LOG4J_FORMAT_MSG_NO_LOOKUPS=true
-# JAVA_OPTS=-Xms256m -Xmx512m
+# SOLR_JAVA_MEM=-Xms256m -Xmx1024m
+# GC_TUNE=-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75
 # CPU_REQUEST=100m
 # CPU_LIMIT=750m
 # MEMORY_REQUEST=256Mi
-# MEMORY_LIMIT=3000Mi
+# MEMORY_LIMIT=2000Mi
diff --git a/openshift/templates/search-engine/search-engine-deploy.bc.param b/openshift/templates/search-engine/search-engine-deploy.bc.param
@@ -22,9 +22,9 @@ PERSISTENT_VOLUME_CLASS=netapp-block-standard
 MOUNT_PATH=/var/solr/data
 CORE_NAME=credential_registry
 SOLR_LOG_LEVEL=WARN
-LOG4J_FORMAT_MSG_NO_LOOKUPS=true
-JAVA_OPTS=-Xms256m -Xmx512m
+SOLR_JAVA_MEM=-Xms256m -Xmx1024m
+GC_TUNE=-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75
 CPU_REQUEST=100m
 CPU_LIMIT=750m
 MEMORY_REQUEST=256Mi
-MEMORY_LIMIT=3000Mi
+MEMORY_LIMIT=2000Mi
diff --git a/openshift/templates/search-engine/search-engine-deploy.bc.prod.param b/openshift/templates/search-engine/search-engine-deploy.bc.prod.param
@@ -22,9 +22,9 @@ TAG_NAME=prod
 # MOUNT_PATH=/var/solr/data
 # CORE_NAME=credential_registry
 # SOLR_LOG_LEVEL=WARN
-# LOG4J_FORMAT_MSG_NO_LOOKUPS=true
-JAVA_OPTS=-Xms256m -Xmx1024m
-# CPU_REQUEST=100m
-# CPU_LIMIT=750m
-# MEMORY_REQUEST=256Mi
-# MEMORY_LIMIT=3000Mi
+SOLR_JAVA_MEM=-Xms2g -Xmx6g
+# GC_TUNE=-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75
+CPU_REQUEST=125m
+CPU_LIMIT=2000m
+MEMORY_REQUEST=2Gi
+MEMORY_LIMIT=8Gi
diff --git a/openshift/templates/search-engine/search-engine-deploy.bc.test.param b/openshift/templates/search-engine/search-engine-deploy.bc.test.param
@@ -22,9 +22,9 @@ TAG_NAME=test
 # MOUNT_PATH=/var/solr/data
 # CORE_NAME=credential_registry
 # SOLR_LOG_LEVEL=WARN
-# LOG4J_FORMAT_MSG_NO_LOOKUPS=true
-# JAVA_OPTS=-Xms256m -Xmx512m
+# SOLR_JAVA_MEM=-Xms256m -Xmx1024m
+# GC_TUNE=-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75
 # CPU_REQUEST=100m
 # CPU_LIMIT=750m
 # MEMORY_REQUEST=256Mi
-# MEMORY_LIMIT=3000Mi
+# MEMORY_LIMIT=2000Mi
diff --git a/openshift/templates/search-engine/search-engine-deploy.yaml b/openshift/templates/search-engine/search-engine-deploy.yaml
@@ -152,17 +152,17 @@ objects:
  protocol: TCP
  readinessProbe:
  failureThreshold: 5
- periodSeconds: 30
+ periodSeconds: 60
  initialDelaySeconds: 3
- timeoutSeconds: 20
+ timeoutSeconds: 40
  httpGet:
  path: /solr/${CORE_NAME}/admin/luke?wt=json&show=index&numTerms=0
  port: 8983
  livenessProbe:
  failureThreshold: 5
- periodSeconds: 60
+ periodSeconds: 90
  initialDelaySeconds: 30
- timeoutSeconds: 20
+ timeoutSeconds: 40
  httpGet:
  path: /solr/${CORE_NAME}/admin/luke?wt=json&show=index&numTerms=0
  port: 8983
@@ -174,10 +174,10 @@ objects:
  value: ${CORE_NAME}
  - name: SOLR_LOG_LEVEL
  value: ${SOLR_LOG_LEVEL}
- - name: LOG4J_FORMAT_MSG_NO_LOOKUPS
- value: ${LOG4J_FORMAT_MSG_NO_LOOKUPS}
- - name: JAVA_OPTS
- value: ${JAVA_OPTS}
+ - name: SOLR_JAVA_MEM
+ value: ${SOLR_JAVA_MEM}
+ - name: GC_TUNE
+ value: ${GC_TUNE}
  terminationMessagePath: /dev/termination-log
  terminationMessagePolicy: File
  imagePullPolicy: Always
@@ -294,16 +294,16 @@ parameters:
  description: Defines the loggin level for Solr.
  required: true
  value: WARN
- - name: LOG4J_FORMAT_MSG_NO_LOOKUPS
- displayName: LOG4J format msg no lookups
- description: Disable message lookups.
+ - name: SOLR_JAVA_MEM
+ displayName: Solr Java Options
+ description: Solr Java runtime options.
  required: true
- value: 'true'
- - name: JAVA_OPTS
- displayName: Java Options
- description: Java runtime options.
+ value: '-Xms512m -Xmx512m'
+ - name: GC_TUNE
+ displayName: Solr Java Garbage Collection Tuning Options
+ description: Solr Java Garbage Collection Tuning Options
  required: true
- value: '-Xms256m -Xmx512m'
+ value: '-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75'
 
  - name: CPU_REQUEST
  displayName: Resources CPU Request