Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Small improvements to cody bench context #5641

Merged
merged 2 commits into from
Sep 23, 2024
Merged

Small improvements to cody bench context #5641

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
67ddd79
better errors
beyang Sep 19, 2024
358360a
cody bench chat-context-v2: adjust overlap threshold
beyang Sep 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions agent/src/cli/command-bench/strategy-chat-context.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ async function runContextCommand(examples: Example[], outputFile: string): Promi
const repoNames = targetRepoRevs.map(repoRev => repoRev.repoName)
const repoIDNames = await graphqlClient.getRepoIds(repoNames, repoNames.length + 10)
if (isError(repoIDNames)) {
throw repoIDNames
throw new Error(`getRepoIds failed for [${repoNames.join(',')}]: ${repoIDNames}`)
}
if (repoIDNames.length !== repoNames.length) {
throw new Error(
Expand All @@ -69,10 +69,10 @@ async function runContextCommand(examples: Example[], outputFile: string): Promi
filePatterns: [],
})
if (isError(resultsResp)) {
throw resultsResp
throw new Error(`contextSearch failed for [${repoNames.join(',')}]: ${resultsResp}`)
}
if (resultsResp === null) {
throw new Error('!!! null results')
throw new Error(`contextSearch failed for [${repoNames.join(',')}]: null results`)
}
const results = resultsResp ?? []
const actualContext: EvalContextItem[] = results.map(result => ({
Expand All @@ -97,7 +97,11 @@ async function runContextCommand(examples: Example[], outputFile: string): Promi
await writeExamplesToCSV(outputFile, exampleOutputs)
}

function contextOverlaps(parentStr: string, childStr: string, threshold = 0.2): boolean {
function contextOverlaps(
parentStr: string,
childStr: string,
threshold = { lines: 3, fraction: 0.2 }
): boolean {
const parent = contextItemFromString(parentStr)
const child = contextItemFromString(childStr)
if (!parent || !child) {
Expand All @@ -123,7 +127,7 @@ function contextOverlaps(parentStr: string, childStr: string, threshold = 0.2):
const overlapLength = overlapEnd - overlapStart + 1
const parentLength = parent.endLine - parent.startLine + 1

return overlapLength / parentLength >= threshold
return overlapLength / parentLength >= threshold.fraction || overlapLength >= threshold.lines
}

function computeRecall(
Expand Down
Loading