Remove COT_ANSWER; can use built-in functions instead

robatwilliams · Dec 19, 2023 · 66a46db · 66a46db
1 parent c475245
commit 66a46db
Show file tree

Hide file tree

Showing 3 changed files with 1 addition and 91 deletions.
diff --git a/src/functions/functions.js b/src/functions/functions.js
@@ -115,36 +115,6 @@ function cost(completionsMatrix, pricesMatrix) {
   }, 0);
 }
 
-CustomFunctions.associate('COT_ANSWER', cotAnswer);
-function cotAnswer(completion, separator) {
-  validateIsCompletion(completion);
-
-  if (separator === null) {
-    // This default value must be kept in sync with documentation in the function metadata.
-    separator = '<!-- END CoT -->';
-  }
-
-  const choiceIndex = 0;
-  const choiceEntity =
-    completion.properties.response.properties.choices.elements[0][choiceIndex];
-  const choiceContent =
-    choiceEntity.properties.message.properties.content.basicValue;
-  const halves = choiceContent.split(separator, 2);
-
-  if (halves.length !== 2) {
-    throw new CustomFunctions.Error(
-      CustomFunctions.ErrorCode.invalidValue,
-      'Completion does not split into two by the separator',
-    );
-  }
-
-  const answer = halves[1];
-
-  // Models have a strong tendency to put a newline after the separator, and
-  // it's difficult to prompt GPT 3.5T to consistently do anything different.
-  return answer[0] === '\n' ? answer.substring(1) : answer;
-}
-
 function toEntityProperty(value) {
   if (value === null) {
     // There is no concept of null in Excel's data model.
@@ -192,6 +162,5 @@ if (typeof module === 'object') {
   module.exports = {
     chatComplete,
     cost,
-    cotAnswer,
   };
 }
diff --git a/src/functions/functions.json b/src/functions/functions.json
@@ -23,27 +23,6 @@
         "type": "any"
       }
     },
-    {
-      "description": "Extract the answer part from a Chain of Thought (CoT) completion",
-      "id": "COT_ANSWER",
-      "name": "COT_ANSWER",
-      "parameters": [
-        {
-          "name": "completion",
-          "description": "The completion to extract the answer from",
-          "type": "any"
-        },
-        {
-          "name": "separator",
-          "description": "The separator that denotes the end of the CoT reasoning and the beginning of the answer. The default separator is <!-- END CoT -->. Your completion prompt must instruct the model to produce this separator, by using wording such as 'Then state \"<!-- END CoT -->\" followed by your answer.'.",
-          "type": "string",
-          "optional": true
-        }
-      ],
-      "result": {
-        "type": "string"
-      }
-    },
     {
       "description": "Calculate the cost of the completions in the given cells",
       "id": "COST",

diff --git a/src/functions/functions.test.mjs b/src/functions/functions.test.mjs
@@ -1,7 +1,7 @@
 import assert from 'node:assert';
 import { describe, it } from 'node:test';
 import { makeCompletionEntity } from '../../testFramework/completionEntityStub.mjs';
-import { chatComplete, cost, cotAnswer } from './functions.js';
+import { chatComplete, cost } from './functions.js';
 import { makeCompletionResponse } from '../../testFramework/completionResponseStub.mjs';
 
 describe('CHAT_COMPLETE', () => {
@@ -201,44 +201,6 @@ describe('COST', () => {
   });
 });
 
-describe('COT_ANSWER', () => {
-  it('extracts the answer', () => {
-    const completion = makeCompletionEntity({
-      content:
-        'The color of the door has been explicitly stated in the provided statement.\n\n<!-- END CoT -->\nBlue',
-    });
-
-    assert.strictEqual(cotAnswer(completion, null), 'Blue');
-  });
-
-  it('uses a custom separator when given', () => {
-    const completion = makeCompletionEntity({
-      content:
-        'The color of the door has been explicitly stated in the provided statement.\n\n*** BEGIN ANSWER ***\nBlue',
-    });
-
-    assert.strictEqual(cotAnswer(completion, '*** BEGIN ANSWER ***'), 'Blue');
-  });
-
-  it('throws an error when the separator is not found', () => {
-    const completion = makeCompletionEntity({ content: 'Blue' });
-
-    assert.throws(() => cotAnswer(completion), {
-      code: '#VALUE!',
-      message: 'Completion does not split into two by the separator',
-    });
-  });
-
-  it('extracts the answer intact when it immediately follows the separator', () => {
-    const completion = makeCompletionEntity({
-      content:
-        'The color of the door has been explicitly stated in the provided statement.\n\n<!-- END CoT -->Blue',
-    });
-
-    assert.strictEqual(cotAnswer(completion, null), 'Blue');
-  });
-});
-
 async function mockResponseOk(body) {
   return new Response(JSON.stringify(body), {
     status: 200,