From e286ba5efb320dca5f588be3a7aff9ee6fc07a33 Mon Sep 17 00:00:00 2001 From: daisyfaithauma Date: Thu, 12 Sep 2024 16:29:18 +0100 Subject: [PATCH 01/11] Logpush initial documentation --- .../docs/ai-gateway/observability/logpush.mdx | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 src/content/docs/ai-gateway/observability/logpush.mdx diff --git a/src/content/docs/ai-gateway/observability/logpush.mdx b/src/content/docs/ai-gateway/observability/logpush.mdx new file mode 100644 index 000000000000000..c14b4a6efb93876 --- /dev/null +++ b/src/content/docs/ai-gateway/observability/logpush.mdx @@ -0,0 +1,101 @@ +--- +pcx_content_type: reference +title: Logpush +sidebar: + badge: + text: Beta +--- + +import { Render } from "~/components"; + +AI Gateway allows you to securely push logs to an external storage location, where you can decrypt and process them. This guide explains how to set up Logpush for the AI Gateway, generate an RSA key pair for encryption, and decrypt the logs once they are received. +Logpush has a limit of 4 jobs. + + + +## Setting up Logpush + +To configure Logpush for AI Gateway, follow these steps: + +## 1. Generate an RSA key pair locally + +You need to generate a key pair to encrypt and decrypt the logs. This script will output your RSA privateKey and publicKey. Keep the private key secure, as it will be used to decrypt the logs. Below is a sample script to generate the keys using Node.js. + +```js title="JavaScript" +const crypto = require("crypto"); + +const { privateKey, publicKey } = crypto.generateKeyPairSync("rsa", { + modulusLength: 4096, + publicKeyEncoding: { + type: "spki", + format: "pem", + }, + privateKeyEncoding: { + type: "pkcs8", + format: "pem", + }, +}); + +console.log(publicKey); +console.log(privateKey); +``` + +## 2. Upload public key to gateway settings + +Once you have generated the key pair, upload the public key to your AI Gateway settings. This key will be used to encrypt your logs. + +## 3. Receive encrypted logs + +After configuring Logpush, logs will be sent encrypted using the public key you uploaded. To access the data, you will need to decrypt it using your private key. + +## 4. Decrypting logs + +To decrypt the encrypted log bodies and metadata from AI Gateway, you can use the following Node.js script: + +```js title="JavaScript" +const privateKey = `-----BEGIN RSA PRIVATE KEY----- +.... +-----END RSA PRIVATE KEY-----`; + +const crypto = require("crypto"); +const key = crypto.createPrivateKey(privateKey); + +const fs = require("fs"); +const zlib = require("zlib"); +const readline = require("readline"); + +function decryptBase64(key, string) { + const decryptedData = crypto.privateDecrypt( + { + key: key, + oaepHash: "SHA256", + }, + Buffer.from(string, "base64"), + ); + + return decryptedData.toString(); +} + +let lineReader = readline.createInterface({ + input: fs.createReadStream("my_log.log.gz").pipe(zlib.createGunzip()), +}); + +lineReader.on("line", (line) => { + line = JSON.parse(line); + + const { Metadata, RequestBody, ResponseBody, ...remaining } = line; + + console.log({ + ...remaining, + Metadata: decryptBase64(key, Metadata.data), + RequestBody: decryptBase64(key, RequestBody.data), + ResponseBody: decryptBase64(key, ResponseBody.data), + }); + console.log("--"); +}); +``` + +## Script Explanation + +The script reads the encrypted log file `(my_log.log.gz)`, decrypts the metadata, request body, and response body, and prints the decrypted data. +Ensure you replace the `privateKey` variable with your actual private RSA key that you generated in step 1. From 3b303d0f981e3aa179c6f90f536ce5b5a4e8bacd Mon Sep 17 00:00:00 2001 From: Kathy <153706637+kathayl@users.noreply.github.com> Date: Tue, 24 Sep 2024 15:18:31 +0100 Subject: [PATCH 02/11] Update logpush.mdx --- src/content/docs/ai-gateway/observability/logpush.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content/docs/ai-gateway/observability/logpush.mdx b/src/content/docs/ai-gateway/observability/logpush.mdx index c14b4a6efb93876..a1deb041e9434f2 100644 --- a/src/content/docs/ai-gateway/observability/logpush.mdx +++ b/src/content/docs/ai-gateway/observability/logpush.mdx @@ -8,7 +8,7 @@ sidebar: import { Render } from "~/components"; -AI Gateway allows you to securely push logs to an external storage location, where you can decrypt and process them. This guide explains how to set up Logpush for the AI Gateway, generate an RSA key pair for encryption, and decrypt the logs once they are received. +AI Gateway allows you to securely export logs to an external storage location, where you can decrypt and process them. This guide explains how to set up Logpush for AI Gateway, generate an RSA key pair for encryption, and decrypt the logs once they are received. Logpush has a limit of 4 jobs. From 83e34edbf1dd6e6cb3638fb48ae11c79c96293c8 Mon Sep 17 00:00:00 2001 From: Kathy <153706637+kathayl@users.noreply.github.com> Date: Tue, 24 Sep 2024 18:50:32 +0100 Subject: [PATCH 03/11] Update logpush.mdx --- src/content/docs/ai-gateway/observability/logpush.mdx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/content/docs/ai-gateway/observability/logpush.mdx b/src/content/docs/ai-gateway/observability/logpush.mdx index a1deb041e9434f2..09bc31c3bda1ee0 100644 --- a/src/content/docs/ai-gateway/observability/logpush.mdx +++ b/src/content/docs/ai-gateway/observability/logpush.mdx @@ -13,6 +13,12 @@ Logpush has a limit of 4 jobs. +:::note[Note] + +To export logs using Logpush, you must have logs turned on for the gateway or have overridden the gateway settings by using the `cf-aig-collect-log` request header. + +::: + ## Setting up Logpush To configure Logpush for AI Gateway, follow these steps: From 569a75512fa26752eeb3688f03ed71a06cd2aae5 Mon Sep 17 00:00:00 2001 From: daisyfaithauma Date: Wed, 25 Sep 2024 14:26:17 +0100 Subject: [PATCH 04/11] Added limits link --- src/content/docs/ai-gateway/observability/logpush.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/content/docs/ai-gateway/observability/logpush.mdx b/src/content/docs/ai-gateway/observability/logpush.mdx index 09bc31c3bda1ee0..d9b6670b86da546 100644 --- a/src/content/docs/ai-gateway/observability/logpush.mdx +++ b/src/content/docs/ai-gateway/observability/logpush.mdx @@ -9,13 +9,13 @@ sidebar: import { Render } from "~/components"; AI Gateway allows you to securely export logs to an external storage location, where you can decrypt and process them. This guide explains how to set up Logpush for AI Gateway, generate an RSA key pair for encryption, and decrypt the logs once they are received. -Logpush has a limit of 4 jobs. +Logpush has a limit of 4 jobs. If you have reached the [limit of logs stored for your plan](/ai-gateway/reference/limits/), logs will not be exported through Logpush. :::note[Note] -To export logs using Logpush, you must have logs turned on for the gateway or have overridden the gateway settings by using the `cf-aig-collect-log` request header. +To export logs using Logpush, you must have logs turned on for the gateway or have overridden the gateway settings by using the `cf-aig-collect-log` request header. ::: From 698dd626f9c32ffd4b8cb0253a06cd1117745a69 Mon Sep 17 00:00:00 2001 From: daisyfaithauma Date: Wed, 25 Sep 2024 15:09:12 +0100 Subject: [PATCH 05/11] Updated details --- .../docs/ai-gateway/observability/logpush.mdx | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/content/docs/ai-gateway/observability/logpush.mdx b/src/content/docs/ai-gateway/observability/logpush.mdx index d9b6670b86da546..c32ef9c56a47b27 100644 --- a/src/content/docs/ai-gateway/observability/logpush.mdx +++ b/src/content/docs/ai-gateway/observability/logpush.mdx @@ -8,7 +8,10 @@ sidebar: import { Render } from "~/components"; -AI Gateway allows you to securely export logs to an external storage location, where you can decrypt and process them. This guide explains how to set up Logpush for AI Gateway, generate an RSA key pair for encryption, and decrypt the logs once they are received. +AI Gateway allows you to securely export logs to an external storage location, where you can decrypt and process them. +You can toggle Logpush on and off in the [Cloudflare dashboard](https://dash.cloudflare.com) settings. + +This guide explains how to set up Logpush for AI Gateway, generate an RSA key pair for encryption, and decrypt the logs once they are received. Logpush has a limit of 4 jobs. If you have reached the [limit of logs stored for your plan](/ai-gateway/reference/limits/), logs will not be exported through Logpush. @@ -46,13 +49,21 @@ console.log(publicKey); console.log(privateKey); ``` +This script will output your RSA privateKey and publicKey. Keep the private key secure, as it will be used to decrypt the logs. + ## 2. Upload public key to gateway settings -Once you have generated the key pair, upload the public key to your AI Gateway settings. This key will be used to encrypt your logs. +Once you have generated the key pair, upload the public key to your AI Gateway settings. This key will be used to encrypt your logs. For more information on Logpush refer to [Logs](/logs/get-started/). + +:::note[Note] +You should enable Logpush in AI Gateway settings. This option only shows up when logs are enabled. +You can toggle on or off as needed. If you do not have logs turned on nothing will be saved. + +::: ## 3. Receive encrypted logs -After configuring Logpush, logs will be sent encrypted using the public key you uploaded. To access the data, you will need to decrypt it using your private key. +After configuring Logpush, logs will be sent encrypted using the public key you uploaded. To access the data, you will need to decrypt it using your private key. The logs will be sent to the object storage provider that you have selected. ## 4. Decrypting logs From e15cf05caecd8d59ff1dd01866ed335c33c23a6d Mon Sep 17 00:00:00 2001 From: daisyfaithauma Date: Wed, 25 Sep 2024 17:23:38 +0100 Subject: [PATCH 06/11] Add script to run file --- src/content/docs/ai-gateway/observability/logpush.mdx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/content/docs/ai-gateway/observability/logpush.mdx b/src/content/docs/ai-gateway/observability/logpush.mdx index c32ef9c56a47b27..75d2b3f28daee40 100644 --- a/src/content/docs/ai-gateway/observability/logpush.mdx +++ b/src/content/docs/ai-gateway/observability/logpush.mdx @@ -49,7 +49,13 @@ console.log(publicKey); console.log(privateKey); ``` -This script will output your RSA privateKey and publicKey. Keep the private key secure, as it will be used to decrypt the logs. +Run the script by executing the code below on your terminal. Where `file name` is the name of your JavaScript file. + +```bash +node {file name} +``` + +This script will output your RSA privateKey and publicKey on your terminal. Keep the private key secure, as it will be used to decrypt the logs. ## 2. Upload public key to gateway settings From 5c3c2b515936d50f2f2d63aae2533c28bb6c0a17 Mon Sep 17 00:00:00 2001 From: Kathy <153706637+kathayl@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:21:23 +0100 Subject: [PATCH 07/11] Update logpush.mdx --- .../docs/ai-gateway/observability/logpush.mdx | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/content/docs/ai-gateway/observability/logpush.mdx b/src/content/docs/ai-gateway/observability/logpush.mdx index 75d2b3f28daee40..2a8eca6254c6e38 100644 --- a/src/content/docs/ai-gateway/observability/logpush.mdx +++ b/src/content/docs/ai-gateway/observability/logpush.mdx @@ -12,13 +12,12 @@ AI Gateway allows you to securely export logs to an external storage location, w You can toggle Logpush on and off in the [Cloudflare dashboard](https://dash.cloudflare.com) settings. This guide explains how to set up Logpush for AI Gateway, generate an RSA key pair for encryption, and decrypt the logs once they are received. -Logpush has a limit of 4 jobs. If you have reached the [limit of logs stored for your plan](/ai-gateway/reference/limits/), logs will not be exported through Logpush. - +You can store up to 10 million logs per gateway. If your limit is reached, new logs will stop being saved and will not be exported through Logpush. To continue saving and exporting logs, you must delete older logs to free up space for new logs. Logpush has a limit of 4 jobs. :::note[Note] -To export logs using Logpush, you must have logs turned on for the gateway or have overridden the gateway settings by using the `cf-aig-collect-log` request header. +To export logs using Logpush, you must have logs turned on for the gateway. ::: @@ -59,19 +58,17 @@ This script will output your RSA privateKey and publicKey on your terminal. Keep ## 2. Upload public key to gateway settings -Once you have generated the key pair, upload the public key to your AI Gateway settings. This key will be used to encrypt your logs. For more information on Logpush refer to [Logs](/logs/get-started/). +Once you have generated the key pair, upload the public key to your AI Gateway settings. This key will be used to encrypt your logs. In order to enable Logpush, you will need logs enabled for that gateway. -:::note[Note] -You should enable Logpush in AI Gateway settings. This option only shows up when logs are enabled. -You can toggle on or off as needed. If you do not have logs turned on nothing will be saved. +## 3. Set up Logpush -::: +To set up Logpush, refer to [Logpush Get Started](/logs/get-started/). -## 3. Receive encrypted logs +## 4. Receive encrypted logs After configuring Logpush, logs will be sent encrypted using the public key you uploaded. To access the data, you will need to decrypt it using your private key. The logs will be sent to the object storage provider that you have selected. -## 4. Decrypting logs +## 5. Decrypting logs To decrypt the encrypted log bodies and metadata from AI Gateway, you can use the following Node.js script: @@ -118,6 +115,12 @@ lineReader.on("line", (line) => { }); ``` +Run the script by executing the code below on your terminal. Where `file name` is the name of your JavaScript file. + +```bash +node {file name} +``` + ## Script Explanation The script reads the encrypted log file `(my_log.log.gz)`, decrypts the metadata, request body, and response body, and prints the decrypted data. From 1f2ea219e8afa7b01e572028161e528f43870c1a Mon Sep 17 00:00:00 2001 From: daisyfaithauma Date: Thu, 26 Sep 2024 11:57:41 +0100 Subject: [PATCH 08/11] Update src/content/docs/ai-gateway/observability/logpush.mdx Replace Co-authored-by: Jun Lee --- src/content/docs/ai-gateway/observability/logpush.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content/docs/ai-gateway/observability/logpush.mdx b/src/content/docs/ai-gateway/observability/logpush.mdx index 2a8eca6254c6e38..d59590e91b9a70c 100644 --- a/src/content/docs/ai-gateway/observability/logpush.mdx +++ b/src/content/docs/ai-gateway/observability/logpush.mdx @@ -48,7 +48,7 @@ console.log(publicKey); console.log(privateKey); ``` -Run the script by executing the code below on your terminal. Where `file name` is the name of your JavaScript file. +Run the script by executing the below code on your terminal. Replace `file name` with the name of your JavaScript file. ```bash node {file name} From 0a7318eb610bc3d0add363b2fcae3232d85c736b Mon Sep 17 00:00:00 2001 From: daisyfaithauma Date: Thu, 26 Sep 2024 11:58:45 +0100 Subject: [PATCH 09/11] Update src/content/docs/ai-gateway/observability/logpush.mdx Duplicate Co-authored-by: Jun Lee --- src/content/docs/ai-gateway/observability/logpush.mdx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/content/docs/ai-gateway/observability/logpush.mdx b/src/content/docs/ai-gateway/observability/logpush.mdx index d59590e91b9a70c..b2465f5f70ea6b4 100644 --- a/src/content/docs/ai-gateway/observability/logpush.mdx +++ b/src/content/docs/ai-gateway/observability/logpush.mdx @@ -54,7 +54,6 @@ Run the script by executing the below code on your terminal. Replace `file name` node {file name} ``` -This script will output your RSA privateKey and publicKey on your terminal. Keep the private key secure, as it will be used to decrypt the logs. ## 2. Upload public key to gateway settings From c7b61ef2602249a000918a78d0b878021acaa035 Mon Sep 17 00:00:00 2001 From: daisyfaithauma Date: Thu, 26 Sep 2024 11:58:52 +0100 Subject: [PATCH 10/11] Update src/content/docs/ai-gateway/observability/logpush.mdx Co-authored-by: Jun Lee --- src/content/docs/ai-gateway/observability/logpush.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content/docs/ai-gateway/observability/logpush.mdx b/src/content/docs/ai-gateway/observability/logpush.mdx index b2465f5f70ea6b4..6356f493a35e070 100644 --- a/src/content/docs/ai-gateway/observability/logpush.mdx +++ b/src/content/docs/ai-gateway/observability/logpush.mdx @@ -67,7 +67,7 @@ To set up Logpush, refer to [Logpush Get Started](/logs/get-started/). After configuring Logpush, logs will be sent encrypted using the public key you uploaded. To access the data, you will need to decrypt it using your private key. The logs will be sent to the object storage provider that you have selected. -## 5. Decrypting logs +## 5. Decrypt logs To decrypt the encrypted log bodies and metadata from AI Gateway, you can use the following Node.js script: From 902fe610f354a7aa6ad1572a9b3d72377dc107db Mon Sep 17 00:00:00 2001 From: daisyfaithauma Date: Thu, 26 Sep 2024 11:59:46 +0100 Subject: [PATCH 11/11] Update src/content/docs/ai-gateway/observability/logpush.mdx Co-authored-by: Jun Lee --- src/content/docs/ai-gateway/observability/logpush.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content/docs/ai-gateway/observability/logpush.mdx b/src/content/docs/ai-gateway/observability/logpush.mdx index 6356f493a35e070..a25494c89e37d74 100644 --- a/src/content/docs/ai-gateway/observability/logpush.mdx +++ b/src/content/docs/ai-gateway/observability/logpush.mdx @@ -114,7 +114,7 @@ lineReader.on("line", (line) => { }); ``` -Run the script by executing the code below on your terminal. Where `file name` is the name of your JavaScript file. +Run the script by executing the below code on your terminal. Replace `file name` with the name of your JavaScript file. ```bash node {file name}