fix(cli): Add error topic to pipeline inspect (#6117)

* leftover docs * add test during image build * add optimize disk space * set default for envars for docker cli * remove userid for now * add error topic
SeldonIO · Nov 27, 2024 · 75c6234 · 75c6234
1 parent a5c7a14
commit 75c6234
Show file tree

Hide file tree

Showing 11 changed files with 75 additions and 52 deletions.
diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml
@@ -28,12 +28,27 @@ jobs:
         run: make -C scheduler test
       - name: test-hodometer
         run: make -C hodometer test-hodometer
+      - name: test-components-tls
+        run: make -C components/tls test
+      - name: test-components-kafka
+        run: make -C components/kafka test
 
   docker:
     needs: test
     runs-on: ubuntu-latest
     if: github.repository == 'SeldonIO/seldon-core' # Do not run this on forks.
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          remove-dotnet: 'true'
+          remove-haskell: 'true'
+          remove-android: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
+          overprovision-lvm: 'true'
+          swap-size-mb: 1024
+
       - name: Checkout Git Commit
         uses: actions/checkout@v4
 

diff --git a/docs-gb/cli/seldon_experiment_start.md b/docs-gb/cli/seldon_experiment_start.md
@@ -16,6 +16,7 @@ seldon experiment start [flags]
       --authority string        authority (HTTP/2) or virtual host (HTTP/1)
   -f, --file-path string        experiment manifest file (YAML)
   -h, --help                    help for start
+      --force                   force control plane mode (load model, etc.)
       --scheduler-host string   seldon scheduler host (default "0.0.0.0:9004")
   -r, --show-request            show request
   -o, --show-response           show response (default true)

diff --git a/docs-gb/cli/seldon_experiment_stop.md b/docs-gb/cli/seldon_experiment_stop.md
@@ -14,10 +14,11 @@ seldon experiment stop <experimentName> [flags]
 
 ```
       --authority string        authority (HTTP/2) or virtual host (HTTP/1)
+  -f, --file-path string        experiment manifest file (YAML)
+      --force                   force control plane mode (load model, etc.)
   -h, --help                    help for stop
       --scheduler-host string   seldon scheduler host (default "0.0.0.0:9004")
-  -r, --show-request            show request
-  -o, --show-response           show response (default true)
+  -v, --verbose                 verbose output
 ```
 
 ### SEE ALSO

diff --git a/docs-gb/cli/seldon_model_load.md b/docs-gb/cli/seldon_model_load.md
@@ -15,6 +15,7 @@ seldon model load [flags]
 ```
       --authority string        authority (HTTP/2) or virtual host (HTTP/1)
   -f, --file-path string        model manifest file (YAML)
+      --force                   force control plane mode (load model, etc.)
   -h, --help                    help for load
       --scheduler-host string   seldon scheduler host (default "0.0.0.0:9004")
   -r, --show-request            show request

diff --git a/docs-gb/cli/seldon_model_unload.md b/docs-gb/cli/seldon_model_unload.md
@@ -14,10 +14,11 @@ seldon model unload <modelName> [flags]
 
 ```
       --authority string        authority (HTTP/2) or virtual host (HTTP/1)
+  -f, --file-path string        model manifest file (YAML)
+      --force                   force control plane mode (load model, etc.)
   -h, --help                    help for unload
       --scheduler-host string   seldon scheduler host (default "0.0.0.0:9004")
-  -r, --show-request            show request
-  -o, --show-response           show response (default true)
+  -v, --verbose                 verbose output
 ```
 
 ### SEE ALSO

diff --git a/docs-gb/cli/seldon_pipeline_inspect.md b/docs-gb/cli/seldon_pipeline_inspect.md
@@ -13,14 +13,16 @@ seldon pipeline inspect <expression> [flags]
 ### Options
 
 ```
-      --format string           inspect output format: raw or json. Default raw (default "raw")
-  -h, --help                    help for inspect
-      --kafka-broker string     kafka broker (default "0.0.0.0:9092")
-      --namespace string        Kubernetes namespace. Default default (default "default")
-      --offset int              message offset to start reading from, i.e. default 1 is the last message only (default 1)
-      --request-id string       request id to show, if not specified will be all messages in offset range
-      --scheduler-host string   seldon scheduler host (default "0.0.0.0:9004")
-      --verbose                 display more details, such as headers
+      --format string              inspect output format: raw or json. Default raw (default "raw")
+  -h, --help                       help for inspect
+      --kafka-broker string        kafka broker (default "0.0.0.0:9092")
+      --kafka-config-path string   path to kafka config file
+      --namespace string           Kubernetes namespace. Default default (default "default")
+      --offset int                 message offset to start reading from, i.e. default 1 is the last message only (default 1)
+      --request-id string          request id to show, if not specified will be all messages in offset range
+      --scheduler-host string      seldon scheduler host (default "0.0.0.0:9004")
+  -t, --truncate                   truncate data
+  -v, --verbose                    display more details, such as headers
 ```
 
 ### SEE ALSO

diff --git a/docs-gb/cli/seldon_pipeline_load.md b/docs-gb/cli/seldon_pipeline_load.md
@@ -13,12 +13,12 @@ seldon pipeline load [flags]
 ### Options
 
 ```
-      --authority string        authority (HTTP/2) or virtual host (HTTP/1)
+     --authority string        authority (HTTP/2) or virtual host (HTTP/1)
   -f, --file-path string        pipeline manifest file (YAML)
+      --force                   force control plane mode (load model, etc.)
   -h, --help                    help for load
       --scheduler-host string   seldon scheduler host (default "0.0.0.0:9004")
-  -r, --show-request            show request
-  -o, --show-response           show response (default true)
+  -v, --verbose                 verbose output
 ```
 
 ### SEE ALSO

diff --git a/docs-gb/cli/seldon_pipeline_unload.md b/docs-gb/cli/seldon_pipeline_unload.md
@@ -13,11 +13,12 @@ seldon pipeline unload <pipelineName> [flags]
 ### Options
 
 ```
-      --authority string        authority (HTTP/2) or virtual host (HTTP/1)
-  -h, --help                    help for unload
+     --authority string        authority (HTTP/2) or virtual host (HTTP/1)
+  -f, --file-path string        pipeline manifest file (YAML)
+      --force                   force control plane mode (load model, etc.)
+  -h, --help                    help for load
       --scheduler-host string   seldon scheduler host (default "0.0.0.0:9004")
-  -r, --show-request            show request
-  -o, --show-response           show response (default true)
+  -v, --verbose                 verbose output
 ```
 
 ### SEE ALSO

diff --git a/operator/Dockerfile.cli b/operator/Dockerfile.cli
@@ -21,8 +21,6 @@ COPY --from=certs /etc/ssl/certs/ca-bundle.crt /etc/pki/tls/certs/ca-bundle.crt
 COPY --from=certs /etc/ssl/certs/ca-bundle.crt /tmp/certs/kafka/broker/ca.crt
 RUN chmod -R 777 /tmp/certs/
 
-USER 1000
-
 WORKDIR /
 COPY --from=builder /workspace/operator/bin/seldon bin/seldon
 

diff --git a/operator/config/cli/seldon-cli.yaml b/operator/config/cli/seldon-cli.yaml
@@ -12,42 +12,18 @@ spec:
     imagePullPolicy: IfNotPresent
     name: seldon-cli
     env:
-    # - name: KAFKA_SECURITY_PROTOCOL
-    #   value: SSL
-    # - name: KAFKA_SASL_MECHANISM
-    #   value: SCRAM-SHA-512
-    # - name: KAFKA_CLIENT_TLS_ENDPOINT_IDENTIFICATION_ALGORITHM
-    #   value: ''
-    # - name: KAFKA_CLIENT_TLS_SECRET_NAME                       
-    #   value: seldon
-    # - name: KAFKA_CLIENT_TLS_KEY_LOCATION
-    #   value: /tmp/certs/kafka/client/user.key
-    # - name: KAFKA_CLIENT_TLS_CRT_LOCATION
-    #   value: /tmp/certs/kafka/client/user.crt
-    # - name: KAFKA_CLIENT_TLS_CA_LOCATION
-    #   value: /tmp/certs/kafka/client/ca.crt
-    # - name: KAFKA_CLIENT_SASL_USERNAME
-    #   value: seldon
-    # - name: KAFKA_CLIENT_SASL_SECRET_NAME
-    #   value: ''
-    # - name: KAFKA_CLIENT_SASL_PASSWORD_LOCATION
-    #   value: password
-    # - name: KAFKA_BROKER_TLS_SECRET_NAME
-    #   value: seldon-cluster-ca-cert
-    # - name: KAFKA_BROKER_TLS_CA_LOCATION                        
-    #   value: /tmp/certs/kafka/broker/ca.crt
     - name: KAFKA_SECURITY_PROTOCOL
-      value: PLAINTEXT
+      value: SSL
     - name: KAFKA_SASL_MECHANISM
       value: SCRAM-SHA-512
     - name: KAFKA_CLIENT_TLS_ENDPOINT_IDENTIFICATION_ALGORITHM
       value: ''
-    - name: KAFKA_CLIENT_TLS_SECRET_NAME
-      value: ''
+    - name: KAFKA_CLIENT_TLS_SECRET_NAME                       
+      value: seldon
     - name: KAFKA_CLIENT_TLS_KEY_LOCATION
-      value: /tmp/certs/kafka/client/tls.key
+      value: /tmp/certs/kafka/client/user.key
     - name: KAFKA_CLIENT_TLS_CRT_LOCATION
-      value: /tmp/certs/kafka/client/tls.crt
+      value: /tmp/certs/kafka/client/user.crt
     - name: KAFKA_CLIENT_TLS_CA_LOCATION
       value: /tmp/certs/kafka/client/ca.crt
     - name: KAFKA_CLIENT_SASL_USERNAME
@@ -57,9 +33,35 @@ spec:
     - name: KAFKA_CLIENT_SASL_PASSWORD_LOCATION
       value: password
     - name: KAFKA_BROKER_TLS_SECRET_NAME
-      value: ''
-    - name: KAFKA_BROKER_TLS_CA_LOCATION
+      value: seldon-cluster-ca-cert
+    - name: KAFKA_BROKER_TLS_CA_LOCATION                        
       value: /tmp/certs/kafka/broker/ca.crt
+    # - name: KAFKA_SECURITY_PROTOCOL
+    #   value: PLAINTEXT
+    # - name: KAFKA_SASL_MECHANISM
+    #   value: SCRAM-SHA-512
+    # - name: KAFKA_CLIENT_TLS_ENDPOINT_IDENTIFICATION_ALGORITHM
+    #   value: ''
+    # - name: KAFKA_CLIENT_TLS_SECRET_NAME
+    #   value: ''
+    # - name: KAFKA_CLIENT_TLS_KEY_LOCATION
+    #   value: /tmp/certs/kafka/client/tls.key
+    # - name: KAFKA_CLIENT_TLS_CRT_LOCATION
+    #   value: /tmp/certs/kafka/client/tls.crt
+    # - name: KAFKA_CLIENT_TLS_CA_LOCATION
+    #   value: /tmp/certs/kafka/client/ca.crt
+    # - name: KAFKA_CLIENT_SASL_USERNAME
+    #   value: seldon
+    # - name: KAFKA_CLIENT_SASL_SECRET_NAME
+    #   value: ''
+    # - name: KAFKA_CLIENT_SASL_PASSWORD_LOCATION
+    #   value: password
+    # - name: KAFKA_BROKER_TLS_SECRET_NAME
+    #   value: ''
+    # - name: KAFKA_BROKER_TLS_CA_LOCATION
+    #   value: /tmp/certs/kafka/broker/ca.crt
+
+    # kafka config path
     - name: SELDON_KAFKA_CONFIG_PATH
       value: /mnt/kafka/kafka.json
     # The following environment variables are used to configure the seldon-cli from the already existing environment variables

diff --git a/operator/pkg/cli/kafka.go b/operator/pkg/cli/kafka.go
@@ -138,6 +138,7 @@ func createPipelineInspectTopics(pipelineSpec string, response *scheduler.Pipeli
 		}
 		topics = append(topics, fmt.Sprintf("%s.%s.%s.%s.%s", topicPrefix, namespace, PipelineSpecifier, parts[0], InputsSpecifier))
 		topics = append(topics, fmt.Sprintf("%s.%s.%s.%s.%s", topicPrefix, namespace, PipelineSpecifier, parts[0], OutputsSpecifier))
+		topics = append(topics, fmt.Sprintf("%s.%s.errors.errors", topicPrefix, namespace)) // error topic
 		return &PipelineTopics{
 			pipeline: pipelineSpec,
 			topics:   topics,