From 67ddd792cc88d499791f0659b651964e7cae6ef8 Mon Sep 17 00:00:00 2001 From: Beyang Liu Date: Wed, 18 Sep 2024 18:29:54 -0700 Subject: [PATCH 1/2] better errors --- agent/src/cli/command-bench/strategy-chat-context.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/agent/src/cli/command-bench/strategy-chat-context.ts b/agent/src/cli/command-bench/strategy-chat-context.ts index fd84020559d7..270f510cf3bf 100644 --- a/agent/src/cli/command-bench/strategy-chat-context.ts +++ b/agent/src/cli/command-bench/strategy-chat-context.ts @@ -53,7 +53,7 @@ async function runContextCommand(examples: Example[], outputFile: string): Promi const repoNames = targetRepoRevs.map(repoRev => repoRev.repoName) const repoIDNames = await graphqlClient.getRepoIds(repoNames, repoNames.length + 10) if (isError(repoIDNames)) { - throw repoIDNames + throw new Error(`getRepoIds failed for [${repoNames.join(',')}]: ${repoIDNames}`) } if (repoIDNames.length !== repoNames.length) { throw new Error( @@ -69,10 +69,10 @@ async function runContextCommand(examples: Example[], outputFile: string): Promi filePatterns: [], }) if (isError(resultsResp)) { - throw resultsResp + throw new Error(`contextSearch failed for [${repoNames.join(',')}]: ${resultsResp}`) } if (resultsResp === null) { - throw new Error('!!! null results') + throw new Error(`contextSearch failed for [${repoNames.join(',')}]: null results`) } const results = resultsResp ?? [] const actualContext: EvalContextItem[] = results.map(result => ({ From 358360a29cdec5c3b88ae2c3b5a77ab794913af3 Mon Sep 17 00:00:00 2001 From: Beyang Liu Date: Thu, 19 Sep 2024 14:28:15 -0700 Subject: [PATCH 2/2] cody bench chat-context-v2: adjust overlap threshold --- agent/src/cli/command-bench/strategy-chat-context.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/agent/src/cli/command-bench/strategy-chat-context.ts b/agent/src/cli/command-bench/strategy-chat-context.ts index 270f510cf3bf..f8537f5db3d6 100644 --- a/agent/src/cli/command-bench/strategy-chat-context.ts +++ b/agent/src/cli/command-bench/strategy-chat-context.ts @@ -97,7 +97,11 @@ async function runContextCommand(examples: Example[], outputFile: string): Promi await writeExamplesToCSV(outputFile, exampleOutputs) } -function contextOverlaps(parentStr: string, childStr: string, threshold = 0.2): boolean { +function contextOverlaps( + parentStr: string, + childStr: string, + threshold = { lines: 3, fraction: 0.2 } +): boolean { const parent = contextItemFromString(parentStr) const child = contextItemFromString(childStr) if (!parent || !child) { @@ -123,7 +127,7 @@ function contextOverlaps(parentStr: string, childStr: string, threshold = 0.2): const overlapLength = overlapEnd - overlapStart + 1 const parentLength = parent.endLine - parent.startLine + 1 - return overlapLength / parentLength >= threshold + return overlapLength / parentLength >= threshold.fraction || overlapLength >= threshold.lines } function computeRecall(