From 358360a29cdec5c3b88ae2c3b5a77ab794913af3 Mon Sep 17 00:00:00 2001 From: Beyang Liu Date: Thu, 19 Sep 2024 14:28:15 -0700 Subject: [PATCH] cody bench chat-context-v2: adjust overlap threshold --- agent/src/cli/command-bench/strategy-chat-context.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/agent/src/cli/command-bench/strategy-chat-context.ts b/agent/src/cli/command-bench/strategy-chat-context.ts index 270f510cf3bf..f8537f5db3d6 100644 --- a/agent/src/cli/command-bench/strategy-chat-context.ts +++ b/agent/src/cli/command-bench/strategy-chat-context.ts @@ -97,7 +97,11 @@ async function runContextCommand(examples: Example[], outputFile: string): Promi await writeExamplesToCSV(outputFile, exampleOutputs) } -function contextOverlaps(parentStr: string, childStr: string, threshold = 0.2): boolean { +function contextOverlaps( + parentStr: string, + childStr: string, + threshold = { lines: 3, fraction: 0.2 } +): boolean { const parent = contextItemFromString(parentStr) const child = contextItemFromString(childStr) if (!parent || !child) { @@ -123,7 +127,7 @@ function contextOverlaps(parentStr: string, childStr: string, threshold = 0.2): const overlapLength = overlapEnd - overlapStart + 1 const parentLength = parent.endLine - parent.startLine + 1 - return overlapLength / parentLength >= threshold + return overlapLength / parentLength >= threshold.fraction || overlapLength >= threshold.lines } function computeRecall(