From a802fe3afb637851200c128db161f106a62c7d7d Mon Sep 17 00:00:00 2001 From: Arun Date: Tue, 14 Mar 2023 23:27:41 -0700 Subject: [PATCH 1/5] Environment variable based importing and provisioning for OpenAI Evals --- plugins/evals/api_key.go | 41 +++++++++++++++++++++++++++++++++++ plugins/evals/api_key_test.go | 41 +++++++++++++++++++++++++++++++++++ plugins/evals/oaieval.go | 25 +++++++++++++++++++++ plugins/evals/plugin.go | 22 +++++++++++++++++++ 4 files changed, 129 insertions(+) create mode 100644 plugins/evals/api_key.go create mode 100644 plugins/evals/api_key_test.go create mode 100644 plugins/evals/oaieval.go create mode 100644 plugins/evals/plugin.go diff --git a/plugins/evals/api_key.go b/plugins/evals/api_key.go new file mode 100644 index 000000000..33aa32ead --- /dev/null +++ b/plugins/evals/api_key.go @@ -0,0 +1,41 @@ +package evals + +import ( + "github.com/1Password/shell-plugins/sdk" + "github.com/1Password/shell-plugins/sdk/importer" + "github.com/1Password/shell-plugins/sdk/provision" + "github.com/1Password/shell-plugins/sdk/schema" + "github.com/1Password/shell-plugins/sdk/schema/credname" + "github.com/1Password/shell-plugins/sdk/schema/fieldname" +) + +func APIKey() schema.CredentialType { + return schema.CredentialType{ + Name: credname.APIKey, + DocsURL: sdk.URL("https://help.openai.com/en/articles/4936850-where-do-i-find-my-secret-api-key"), + ManagementURL: sdk.URL("https://platform.openai.com/account/api-keys"), + Fields: []schema.CredentialField{ + { + Name: fieldname.APIKey, + MarkdownDescription: "API Key used to authenticate to OpenAI platform.", + Secret: true, + Composition: &schema.ValueComposition{ + Length: 51, + Prefix: "sk-", + Charset: schema.Charset{ + Uppercase: true, + Lowercase: true, + Digits: true, + }, + }, + }, + }, + DefaultProvisioner: provision.EnvVars(defaultEnvVarMapping), + Importer: importer.TryAll( + importer.TryEnvVarPair(defaultEnvVarMapping), + )} +} + +var defaultEnvVarMapping = map[string]sdk.FieldName{ + "OPENAI_API_KEY": fieldname.APIKey, +} diff --git a/plugins/evals/api_key_test.go b/plugins/evals/api_key_test.go new file mode 100644 index 000000000..902413e32 --- /dev/null +++ b/plugins/evals/api_key_test.go @@ -0,0 +1,41 @@ +package evals + +import ( + "testing" + + "github.com/1Password/shell-plugins/sdk" + "github.com/1Password/shell-plugins/sdk/plugintest" + "github.com/1Password/shell-plugins/sdk/schema/fieldname" +) + +func TestAPIKeyProvisioner(t *testing.T) { + plugintest.TestProvisioner(t, APIKey().DefaultProvisioner, map[string]plugintest.ProvisionCase{ + "default": { + ItemFields: map[sdk.FieldName]string{ + fieldname.APIKey: "sk-ZJipjm9euld2QcpLIX3NCDUiLgexQkvdn0WGlKayAEXAMPLE", + }, + ExpectedOutput: sdk.ProvisionOutput{ + Environment: map[string]string{ + "OPENAI_API_KEY": "sk-ZJipjm9euld2QcpLIX3NCDUiLgexQkvdn0WGlKayAEXAMPLE", + }, + }, + }, + }) +} + +func TestAPIKeyImporter(t *testing.T) { + plugintest.TestImporter(t, APIKey().Importer, map[string]plugintest.ImportCase{ + "environment": { + Environment: map[string]string{ + "OPENAI_API_KEY": "sk-ZJipjm9euld2QcpLIX3NCDUiLgexQkvdn0WGlKayAEXAMPLE", + }, + ExpectedCandidates: []sdk.ImportCandidate{ + { + Fields: map[sdk.FieldName]string{ + fieldname.APIKey: "sk-ZJipjm9euld2QcpLIX3NCDUiLgexQkvdn0WGlKayAEXAMPLE", + }, + }, + }, + }, + }) +} diff --git a/plugins/evals/oaieval.go b/plugins/evals/oaieval.go new file mode 100644 index 000000000..b7c3d7d10 --- /dev/null +++ b/plugins/evals/oaieval.go @@ -0,0 +1,25 @@ +package evals + +import ( + "github.com/1Password/shell-plugins/sdk" + "github.com/1Password/shell-plugins/sdk/needsauth" + "github.com/1Password/shell-plugins/sdk/schema" + "github.com/1Password/shell-plugins/sdk/schema/credname" +) + +func OpenAIEvalsCLI() schema.Executable { + return schema.Executable{ + Name: "OpenAI Evals CLI", + Runs: []string{"oaieval"}, + DocsURL: sdk.URL("https://github.com/openai/evals/blob/main/README.md"), + NeedsAuth: needsauth.IfAll( + needsauth.NotForHelpOrVersion(), + needsauth.NotWithoutArgs(), + ), + Uses: []schema.CredentialUsage{ + { + Name: credname.APIKey, + }, + }, + } +} diff --git a/plugins/evals/plugin.go b/plugins/evals/plugin.go new file mode 100644 index 000000000..38d5411a2 --- /dev/null +++ b/plugins/evals/plugin.go @@ -0,0 +1,22 @@ +package evals + +import ( + "github.com/1Password/shell-plugins/sdk" + "github.com/1Password/shell-plugins/sdk/schema" +) + +func New() schema.Plugin { + return schema.Plugin{ + Name: "evals", + Platform: schema.PlatformInfo{ + Name: "OpenAI Evals", + Homepage: sdk.URL("https://github.com/openai/evals"), + }, + Credentials: []schema.CredentialType{ + APIKey(), + }, + Executables: []schema.Executable{ + OpenAIEvalsCLI(), + }, + } +} From 6f463bea93d95ee2eaec8ee16233ad90a393c03d Mon Sep 17 00:00:00 2001 From: Arun Date: Mon, 24 Apr 2023 00:27:38 -0700 Subject: [PATCH 2/5] Drop evals shell plugin and just configure oaieval as another executable within openai shell plugin --- plugins/evals/api_key.go | 41 ------------------- plugins/evals/api_key_test.go | 41 ------------------- plugins/evals/plugin.go | 22 ---------- plugins/{evals/oaieval.go => openai/evals.go} | 2 +- plugins/openai/plugin.go | 1 + 5 files changed, 2 insertions(+), 105 deletions(-) delete mode 100644 plugins/evals/api_key.go delete mode 100644 plugins/evals/api_key_test.go delete mode 100644 plugins/evals/plugin.go rename plugins/{evals/oaieval.go => openai/evals.go} (97%) diff --git a/plugins/evals/api_key.go b/plugins/evals/api_key.go deleted file mode 100644 index 33aa32ead..000000000 --- a/plugins/evals/api_key.go +++ /dev/null @@ -1,41 +0,0 @@ -package evals - -import ( - "github.com/1Password/shell-plugins/sdk" - "github.com/1Password/shell-plugins/sdk/importer" - "github.com/1Password/shell-plugins/sdk/provision" - "github.com/1Password/shell-plugins/sdk/schema" - "github.com/1Password/shell-plugins/sdk/schema/credname" - "github.com/1Password/shell-plugins/sdk/schema/fieldname" -) - -func APIKey() schema.CredentialType { - return schema.CredentialType{ - Name: credname.APIKey, - DocsURL: sdk.URL("https://help.openai.com/en/articles/4936850-where-do-i-find-my-secret-api-key"), - ManagementURL: sdk.URL("https://platform.openai.com/account/api-keys"), - Fields: []schema.CredentialField{ - { - Name: fieldname.APIKey, - MarkdownDescription: "API Key used to authenticate to OpenAI platform.", - Secret: true, - Composition: &schema.ValueComposition{ - Length: 51, - Prefix: "sk-", - Charset: schema.Charset{ - Uppercase: true, - Lowercase: true, - Digits: true, - }, - }, - }, - }, - DefaultProvisioner: provision.EnvVars(defaultEnvVarMapping), - Importer: importer.TryAll( - importer.TryEnvVarPair(defaultEnvVarMapping), - )} -} - -var defaultEnvVarMapping = map[string]sdk.FieldName{ - "OPENAI_API_KEY": fieldname.APIKey, -} diff --git a/plugins/evals/api_key_test.go b/plugins/evals/api_key_test.go deleted file mode 100644 index 902413e32..000000000 --- a/plugins/evals/api_key_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package evals - -import ( - "testing" - - "github.com/1Password/shell-plugins/sdk" - "github.com/1Password/shell-plugins/sdk/plugintest" - "github.com/1Password/shell-plugins/sdk/schema/fieldname" -) - -func TestAPIKeyProvisioner(t *testing.T) { - plugintest.TestProvisioner(t, APIKey().DefaultProvisioner, map[string]plugintest.ProvisionCase{ - "default": { - ItemFields: map[sdk.FieldName]string{ - fieldname.APIKey: "sk-ZJipjm9euld2QcpLIX3NCDUiLgexQkvdn0WGlKayAEXAMPLE", - }, - ExpectedOutput: sdk.ProvisionOutput{ - Environment: map[string]string{ - "OPENAI_API_KEY": "sk-ZJipjm9euld2QcpLIX3NCDUiLgexQkvdn0WGlKayAEXAMPLE", - }, - }, - }, - }) -} - -func TestAPIKeyImporter(t *testing.T) { - plugintest.TestImporter(t, APIKey().Importer, map[string]plugintest.ImportCase{ - "environment": { - Environment: map[string]string{ - "OPENAI_API_KEY": "sk-ZJipjm9euld2QcpLIX3NCDUiLgexQkvdn0WGlKayAEXAMPLE", - }, - ExpectedCandidates: []sdk.ImportCandidate{ - { - Fields: map[sdk.FieldName]string{ - fieldname.APIKey: "sk-ZJipjm9euld2QcpLIX3NCDUiLgexQkvdn0WGlKayAEXAMPLE", - }, - }, - }, - }, - }) -} diff --git a/plugins/evals/plugin.go b/plugins/evals/plugin.go deleted file mode 100644 index 38d5411a2..000000000 --- a/plugins/evals/plugin.go +++ /dev/null @@ -1,22 +0,0 @@ -package evals - -import ( - "github.com/1Password/shell-plugins/sdk" - "github.com/1Password/shell-plugins/sdk/schema" -) - -func New() schema.Plugin { - return schema.Plugin{ - Name: "evals", - Platform: schema.PlatformInfo{ - Name: "OpenAI Evals", - Homepage: sdk.URL("https://github.com/openai/evals"), - }, - Credentials: []schema.CredentialType{ - APIKey(), - }, - Executables: []schema.Executable{ - OpenAIEvalsCLI(), - }, - } -} diff --git a/plugins/evals/oaieval.go b/plugins/openai/evals.go similarity index 97% rename from plugins/evals/oaieval.go rename to plugins/openai/evals.go index b7c3d7d10..acdd56f07 100644 --- a/plugins/evals/oaieval.go +++ b/plugins/openai/evals.go @@ -1,4 +1,4 @@ -package evals +package openai import ( "github.com/1Password/shell-plugins/sdk" diff --git a/plugins/openai/plugin.go b/plugins/openai/plugin.go index 297e1345f..3439dcafc 100644 --- a/plugins/openai/plugin.go +++ b/plugins/openai/plugin.go @@ -17,6 +17,7 @@ func New() schema.Plugin { }, Executables: []schema.Executable{ OpenAICLI(), + OpenAIEvalsCLI(), }, } } From 1df163380d7845a644059956670f0532855cd873 Mon Sep 17 00:00:00 2001 From: Arun Date: Mon, 24 Apr 2023 00:29:36 -0700 Subject: [PATCH 3/5] Fix incorrect oaieval executable file name --- plugins/openai/{evals.go => oaieval.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename plugins/openai/{evals.go => oaieval.go} (100%) diff --git a/plugins/openai/evals.go b/plugins/openai/oaieval.go similarity index 100% rename from plugins/openai/evals.go rename to plugins/openai/oaieval.go From 3dc567407221db3247d749de2868c2fb73b95fb8 Mon Sep 17 00:00:00 2001 From: Arun Date: Mon, 24 Apr 2023 00:32:21 -0700 Subject: [PATCH 4/5] Support oaievalset as another executable within the openai shell plugin --- plugins/openai/oaievalset.go | 25 +++++++++++++++++++++++++ plugins/openai/plugin.go | 1 + 2 files changed, 26 insertions(+) create mode 100644 plugins/openai/oaievalset.go diff --git a/plugins/openai/oaievalset.go b/plugins/openai/oaievalset.go new file mode 100644 index 000000000..23c3b74aa --- /dev/null +++ b/plugins/openai/oaievalset.go @@ -0,0 +1,25 @@ +package openai + +import ( + "github.com/1Password/shell-plugins/sdk" + "github.com/1Password/shell-plugins/sdk/needsauth" + "github.com/1Password/shell-plugins/sdk/schema" + "github.com/1Password/shell-plugins/sdk/schema/credname" +) + +func OpenAIEvalSetCLI() schema.Executable { + return schema.Executable{ + Name: "OpenAI Eval Set CLI", + Runs: []string{"oaievalset"}, + DocsURL: sdk.URL("https://github.com/openai/evals/blob/main/README.md"), + NeedsAuth: needsauth.IfAll( + needsauth.NotForHelpOrVersion(), + needsauth.NotWithoutArgs(), + ), + Uses: []schema.CredentialUsage{ + { + Name: credname.APIKey, + }, + }, + } +} diff --git a/plugins/openai/plugin.go b/plugins/openai/plugin.go index 3439dcafc..17caee4e1 100644 --- a/plugins/openai/plugin.go +++ b/plugins/openai/plugin.go @@ -18,6 +18,7 @@ func New() schema.Plugin { Executables: []schema.Executable{ OpenAICLI(), OpenAIEvalsCLI(), + OpenAIEvalSetCLI(), }, } } From a1e226c69fe93fd62d0836c12af81b3fa6d80530 Mon Sep 17 00:00:00 2001 From: Arun Date: Tue, 25 Apr 2023 00:59:53 -0700 Subject: [PATCH 5/5] Update documentation URL for both oaieval and oaievalset --- plugins/openai/oaieval.go | 2 +- plugins/openai/oaievalset.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/openai/oaieval.go b/plugins/openai/oaieval.go index acdd56f07..bfb6acf3f 100644 --- a/plugins/openai/oaieval.go +++ b/plugins/openai/oaieval.go @@ -11,7 +11,7 @@ func OpenAIEvalsCLI() schema.Executable { return schema.Executable{ Name: "OpenAI Evals CLI", Runs: []string{"oaieval"}, - DocsURL: sdk.URL("https://github.com/openai/evals/blob/main/README.md"), + DocsURL: sdk.URL("https://github.com/openai/evals/blob/main/docs/run-evals.md"), NeedsAuth: needsauth.IfAll( needsauth.NotForHelpOrVersion(), needsauth.NotWithoutArgs(), diff --git a/plugins/openai/oaievalset.go b/plugins/openai/oaievalset.go index 23c3b74aa..32e22f3db 100644 --- a/plugins/openai/oaievalset.go +++ b/plugins/openai/oaievalset.go @@ -11,7 +11,7 @@ func OpenAIEvalSetCLI() schema.Executable { return schema.Executable{ Name: "OpenAI Eval Set CLI", Runs: []string{"oaievalset"}, - DocsURL: sdk.URL("https://github.com/openai/evals/blob/main/README.md"), + DocsURL: sdk.URL("https://github.com/openai/evals/blob/main/docs/run-evals.md"), NeedsAuth: needsauth.IfAll( needsauth.NotForHelpOrVersion(), needsauth.NotWithoutArgs(),