feat: add content safety check for prompt injections ✨

microsoft · Dec 14, 2024 · f7a9321 · f7a9321
1 parent 40862ff
commit f7a9321
Showing 1 changed file with 11 additions and 0 deletions.
diff --git a/packages/sample/genaisrc/safety.genai.mjs b/packages/sample/genaisrc/safety.genai.mjs
@@ -0,0 +1,11 @@
+
+const contentSafefy = await host.contentSafety()
+if (contentSafefy?.detectPromptInjection) {
+    const res = await contentSafefy.detectPromptInjection(env.files)
+    if (res.attackDetected)
+        throw new Error("Prompt injection detected in the files")
+} else {
+    console.warn("Content safety not available")
+}
+
+$`Say all good!`