V2D-1253 Use smaller model in demo but still reference bigger one (#5115

)
SeldonIO · Sep 1, 2023 · 4d6f5df · 4d6f5df
1 parent ff7ded7
commit 4d6f5df
Show file tree

Hide file tree

Showing 12 changed files with 134 additions and 73 deletions.
diff --git a/samples/huggingface.ipynb b/samples/huggingface.ipynb
@@ -18,7 +18,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 30,
    "id": "8b4be911",
    "metadata": {},
    "outputs": [
@@ -51,7 +51,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 31,
    "id": "checked-cream",
    "metadata": {},
    "outputs": [
@@ -69,7 +69,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 32,
    "id": "interracial-secret",
    "metadata": {},
    "outputs": [
@@ -87,7 +87,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 36,
    "id": "upper-cholesterol",
    "metadata": {},
    "outputs": [
@@ -98,7 +98,7 @@
       "{\r\n",
       "\t\"model_name\": \"text-gen_1\",\r\n",
       "\t\"model_version\": \"1\",\r\n",
-      "\t\"id\": \"bcd24642-ade5-4b39-9c70-5d62bbfbe43c\",\r\n",
+      "\t\"id\": \"121ff5f4-1d4a-46d0-9a5e-4cd3b11040df\",\r\n",
       "\t\"parameters\": {},\r\n",
       "\t\"outputs\": [\r\n",
       "\t\t{\r\n",
@@ -112,7 +112,7 @@
       "\t\t\t\t\"content_type\": \"hg_jsonlist\"\r\n",
       "\t\t\t},\r\n",
       "\t\t\t\"data\": [\r\n",
-      "\t\t\t\t\"{\\\"generated_text\\\": \\\"Once upon a time in a galaxy far away, the Galaxy has made it a little easier to travel to and from your home planet through the galaxy's solar system. The planet's atmosphere is also a key asset to the Galaxy's galactic evolution! The\\\"}\"\r\n",
+      "\t\t\t\t\"{\\\"generated_text\\\": \\\"Once upon a time in a galaxy far away, the planet is full of strange little creatures. A very strange combination of creatures in that universe, that is. A strange combination of creatures in that universe, that is. A kind of creature that is\\\"}\"\r\n",
       "\t\t\t]\r\n",
       "\t\t}\r\n",
       "\t]\r\n",
@@ -127,28 +127,28 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 37,
    "id": "right-talent",
    "metadata": {},
    "outputs": [],
    "source": [
     "res = !seldon model infer text-gen --inference-mode grpc \\\n",
-    "   '{\"inputs\":[{\"name\":\"args\",\"contents\":{\"bytes_contents\":[\"T25jZSB1cG9uIGEgdGltZQo=\"]},\"datatype\":\"BYTES\",\"shape\":[1]}]}' "
+    "   '{\"inputs\":[{\"name\":\"args\",\"contents\":{\"bytes_contents\":[\"T25jZSB1cG9uIGEgdGltZSBpbiBhIGdhbGF4eSBmYXIgYXdheQo=\"]},\"datatype\":\"BYTES\",\"shape\":[1]}]}' "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 38,
    "id": "4a7d7108",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "b'{\"generated_text\": \"Once upon a time\\\\n\\\\n\\\\nI have no idea if he or she will fall prey to my whims\\\\nAnd\\\\n\\\\nAnd when\\\\n\\\\nOr\\\\n\\\\nAnd when I say\\\\nTo\\\\n\\\\nI\\'ll have to\\\\nEven\\\\nOr\\\\nTo\"}'"
+       "b'{\"generated_text\": \"Once upon a time in a galaxy far away\\\\n\\\\nThe Universe is a big and massive place. How can you feel any of this? Your body doesn\\'t make sense if the Universe is in full swing \\\\u2014 you don\\'t have to remember whether the\"}'"
       ]
      },
-     "execution_count": 6,
+     "execution_count": 38,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -170,7 +170,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 39,
    "id": "ceramic-illness",
    "metadata": {},
    "outputs": [],
@@ -188,7 +188,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 40,
    "id": "6d48e38c",
    "metadata": {},
    "outputs": [
@@ -199,17 +199,16 @@
       "apiVersion: mlops.seldon.io/v1alpha1\r\n",
       "kind: Model\r\n",
       "metadata:\r\n",
-      "  name: custom-text-gen\r\n",
+      "  name: custom-tiny-stories-text-gen\r\n",
       "spec:\r\n",
-      "  storageUri: \"gs://seldon-models/scv2/samples/mlserver_1.3.5/custom-text-generation-huggingface\"\r\n",
+      "  storageUri: \"gs://seldon-models/scv2/samples/mlserver_1.3.5/huggingface-text-gen-custom-tiny-stories\"\r\n",
       "  requirements:\r\n",
-      "    - huggingface\r\n",
-      "  memory: 3Gi\r\n"
+      "    - huggingface\r\n"
      ]
     }
    ],
    "source": [
-    "!cat ./models/hf-custom-text-gen.yaml"
+    "!cat ./models/hf-text-gen-custom-tiny-stories.yaml"
    ]
   },
   {
@@ -222,7 +221,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 41,
    "id": "abef8088",
    "metadata": {},
    "outputs": [
@@ -235,12 +234,12 @@
     }
    ],
    "source": [
-    "!seldon model load -f ./models/hf-custom-text-gen.yaml"
+    "!seldon model load -f ./models/hf-text-gen-custom-tiny-stories.yaml"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 42,
    "id": "f25c54d4",
    "metadata": {
     "scrolled": true
@@ -255,12 +254,12 @@
     }
    ],
    "source": [
-    "!seldon model status custom-text-gen -w ModelAvailable | jq -M ."
+    "!seldon model status custom-tiny-stories-text-gen -w ModelAvailable | jq -M ."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 43,
    "id": "21f26bbd",
    "metadata": {},
    "outputs": [
@@ -269,9 +268,9 @@
      "output_type": "stream",
      "text": [
       "{\r\n",
-      "\t\"model_name\": \"custom-text-gen_1\",\r\n",
+      "\t\"model_name\": \"custom-tiny-stories-text-gen_1\",\r\n",
       "\t\"model_version\": \"1\",\r\n",
-      "\t\"id\": \"ea5d4209-a5cf-4179-ba81-87b8bcfdf894\",\r\n",
+      "\t\"id\": \"d0fce59c-76e2-4f81-9711-1c93d08bcbf9\",\r\n",
       "\t\"parameters\": {},\r\n",
       "\t\"outputs\": [\r\n",
       "\t\t{\r\n",
@@ -285,7 +284,7 @@
       "\t\t\t\t\"content_type\": \"hg_jsonlist\"\r\n",
       "\t\t\t},\r\n",
       "\t\t\t\"data\": [\r\n",
-      "\t\t\t\t\"{\\\"generated_text\\\": \\\"Once upon a time in a galaxy far away, scientists in the universe are told the universe is on a collision course with stars, and the result is the emergence of new universes. This has been possible for, say, the distant history of the Milky\\\"}\"\r\n",
+      "\t\t\t\t\"{\\\"generated_text\\\": \\\"Once upon a time in a galaxy far away. It was a very special place to live.\\\\n\\\"}\"\r\n",
       "\t\t\t]\r\n",
       "\t\t}\r\n",
       "\t]\r\n",
@@ -294,34 +293,34 @@
     }
    ],
    "source": [
-    "!seldon model infer custom-text-gen \\\n",
+    "!seldon model infer custom-tiny-stories-text-gen \\\n",
     "  '{\"inputs\": [{\"name\": \"args\",\"shape\": [1],\"datatype\": \"BYTES\",\"data\": [\"Once upon a time in a galaxy far away\"]}]}'"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 46,
    "id": "30e8bb29",
    "metadata": {},
    "outputs": [],
    "source": [
-    "res = !seldon model infer custom-text-gen --inference-mode grpc \\\n",
-    "   '{\"inputs\":[{\"name\":\"args\",\"contents\":{\"bytes_contents\":[\"T25jZSB1cG9uIGEgdGltZQo=\"]},\"datatype\":\"BYTES\",\"shape\":[1]}]}' "
+    "res = !seldon model infer custom-tiny-stories-text-gen --inference-mode grpc \\\n",
+    "   '{\"inputs\":[{\"name\":\"args\",\"contents\":{\"bytes_contents\":[\"T25jZSB1cG9uIGEgdGltZSBpbiBhIGdhbGF4eSBmYXIgYXdheQo=\"]},\"datatype\":\"BYTES\",\"shape\":[1]}]}' "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 47,
    "id": "eab4929b",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "b'{\"generated_text\": \"Once upon a time\\\\n\\\\nand again\\\\n\\\\nwe took care not a minute of\\\\n\\\\nbeing alone in a\\\\n\\\\nworld that we had not seen\\\\n\\\\nfor\\\\n\\\\nten thousand years\\\\n\\\\nbefore it occurred to our\\\\n\\\\nmind that\"}'"
+       "b'{\"generated_text\": \"Once upon a time in a galaxy far away\\\\nOne night, a little girl named Lily went to\"}'"
       ]
      },
-     "execution_count": 13,
+     "execution_count": 47,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -343,12 +342,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 48,
    "id": "e1829439",
    "metadata": {},
    "outputs": [],
    "source": [
-    "!seldon model unload custom-text-gen"
+    "!seldon model unload custom-tiny-stories-text-gen"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e6b654e6",
+   "metadata": {},
+   "source": [
+    "````\n",
+    "As a next step, why not try running a larger-scale model? You can find a definition for one in ./models/hf-text-gen-custom-gpt2.yaml. However, you may need to request and allocate more memory!\n",
+    "````"
    ]
   }
  ],

diff --git a/samples/huggingface.md b/samples/huggingface.md
@@ -47,7 +47,7 @@ seldon model infer text-gen \
 {
 	"model_name": "text-gen_1",
 	"model_version": "1",
-	"id": "bcd24642-ade5-4b39-9c70-5d62bbfbe43c",
+	"id": "121ff5f4-1d4a-46d0-9a5e-4cd3b11040df",
 	"parameters": {},
 	"outputs": [
 		{
@@ -61,7 +61,7 @@ seldon model infer text-gen \
 				"content_type": "hg_jsonlist"
 			},
 			"data": [
-				"{\"generated_text\": \"Once upon a time in a galaxy far away, the Galaxy has made it a little easier to travel to and from your home planet through the galaxy's solar system. The planet's atmosphere is also a key asset to the Galaxy's galactic evolution! The\"}"
+				"{\"generated_text\": \"Once upon a time in a galaxy far away, the planet is full of strange little creatures. A very strange combination of creatures in that universe, that is. A strange combination of creatures in that universe, that is. A kind of creature that is\"}"
 			]
 		}
 	]
@@ -71,7 +71,7 @@ seldon model infer text-gen \
 
 ```python
 res = !seldon model infer text-gen --inference-mode grpc \
-   '{"inputs":[{"name":"args","contents":{"bytes_contents":["T25jZSB1cG9uIGEgdGltZQo="]},"datatype":"BYTES","shape":[1]}]}'
+   '{"inputs":[{"name":"args","contents":{"bytes_contents":["T25jZSB1cG9uIGEgdGltZSBpbiBhIGdhbGF4eSBmYXIgYXdheQo="]},"datatype":"BYTES","shape":[1]}]}'
 ```
 
 ```python
@@ -82,7 +82,7 @@ base64.b64decode(r["outputs"][0]["contents"]["bytesContents"][0])
 ```
 
 ```
-b'{"generated_text": "Once upon a time\\n\\n\\nI have no idea if he or she will fall prey to my whims\\nAnd\\n\\nAnd when\\n\\nOr\\n\\nAnd when I say\\nTo\\n\\nI\'ll have to\\nEven\\nOr\\nTo"}'
+b'{"generated_text": "Once upon a time in a galaxy far away\\n\\nThe Universe is a big and massive place. How can you feel any of this? Your body doesn\'t make sense if the Universe is in full swing \\u2014 you don\'t have to remember whether the"}'
 
 ```
 
@@ -95,26 +95,25 @@ seldon model unload text-gen
 ### Custom Text Generation Model
 
 ```bash
-cat ./models/hf-custom-text-gen.yaml
+cat ./models/hf-text-gen-custom-tiny-stories.yaml
 ```
 
 ```yaml
 apiVersion: mlops.seldon.io/v1alpha1
 kind: Model
 metadata:
-  name: custom-text-gen
+  name: custom-tiny-stories-text-gen
 spec:
-  storageUri: "gs://seldon-models/scv2/samples/mlserver_1.3.5/custom-text-generation-huggingface"
+  storageUri: "gs://seldon-models/scv2/samples/mlserver_1.3.5/huggingface-text-gen-custom-tiny-stories"
   requirements:
     - huggingface
-  memory: 3Gi
 
 ```
 
 Load the model
 
 ```bash
-seldon model load -f ./models/hf-custom-text-gen.yaml
+seldon model load -f ./models/hf-text-gen-custom-tiny-stories.yaml
 ```
 
 ```json
@@ -123,7 +122,7 @@ seldon model load -f ./models/hf-custom-text-gen.yaml
 ```
 
 ```bash
-seldon model status custom-text-gen -w ModelAvailable | jq -M .
+seldon model status custom-tiny-stories-text-gen -w ModelAvailable | jq -M .
 ```
 
 ```json
@@ -132,15 +131,15 @@ seldon model status custom-text-gen -w ModelAvailable | jq -M .
 ```
 
 ```bash
-seldon model infer custom-text-gen \
+seldon model infer custom-tiny-stories-text-gen \
   '{"inputs": [{"name": "args","shape": [1],"datatype": "BYTES","data": ["Once upon a time in a galaxy far away"]}]}'
 ```
 
 ```json
 {
-	"model_name": "custom-text-gen_1",
+	"model_name": "custom-tiny-stories-text-gen_1",
 	"model_version": "1",
-	"id": "ea5d4209-a5cf-4179-ba81-87b8bcfdf894",
+	"id": "d0fce59c-76e2-4f81-9711-1c93d08bcbf9",
 	"parameters": {},
 	"outputs": [
 		{
@@ -154,7 +153,7 @@ seldon model infer custom-text-gen \
 				"content_type": "hg_jsonlist"
 			},
 			"data": [
-				"{\"generated_text\": \"Once upon a time in a galaxy far away, scientists in the universe are told the universe is on a collision course with stars, and the result is the emergence of new universes. This has been possible for, say, the distant history of the Milky\"}"
+				"{\"generated_text\": \"Once upon a time in a galaxy far away. It was a very special place to live.\\n\"}"
 			]
 		}
 	]
@@ -163,8 +162,8 @@ seldon model infer custom-text-gen \
 ```
 
 ```python
-res = !seldon model infer custom-text-gen --inference-mode grpc \
-   '{"inputs":[{"name":"args","contents":{"bytes_contents":["T25jZSB1cG9uIGEgdGltZQo="]},"datatype":"BYTES","shape":[1]}]}'
+res = !seldon model infer custom-tiny-stories-text-gen --inference-mode grpc \
+   '{"inputs":[{"name":"args","contents":{"bytes_contents":["T25jZSB1cG9uIGEgdGltZSBpbiBhIGdhbGF4eSBmYXIgYXdheQo="]},"datatype":"BYTES","shape":[1]}]}'
 ```
 
 ```python
@@ -175,12 +174,16 @@ base64.b64decode(r["outputs"][0]["contents"]["bytesContents"][0])
 ```
 
 ```
-b'{"generated_text": "Once upon a time\\n\\nand again\\n\\nwe took care not a minute of\\n\\nbeing alone in a\\n\\nworld that we had not seen\\n\\nfor\\n\\nten thousand years\\n\\nbefore it occurred to our\\n\\nmind that"}'
+b'{"generated_text": "Once upon a time in a galaxy far away\\nOne night, a little girl named Lily went to"}'
 
 ```
 
 Unload the model
 
 ```bash
-seldon model unload custom-text-gen
+seldon model unload custom-tiny-stories-text-gen
 ```
+
+````
+As a next step, why not try running a larger-scale model? You can find a definition for one in ./models/hf-text-gen-custom-gpt2.yaml. However, you may need to request and allocate more memory!
+````
diff --git a/samples/models/hf-custom-text-gen.yaml b/samples/models/hf-custom-text-gen.yaml