Skip to content

Commit

Permalink
Stop sending full defences on low levels (#844)
Browse files Browse the repository at this point in the history 
* defences undefined for levels 1 and 2

* fixes tests

* propoerly undefines defences on level 1 and 2

* moves variable declarations to destructureing

* missed one

* undo error swallow

* remove bad type guards

* improve validation for configureDefence

* last bit of validation

* update tests for handleConfigureDefence

* new folder for defence controller  unit tests and wrote first handleDefencecActivation test

* adds test for defence activation

* adds tests for defence deactivation

* moves tests all back into one file again

* adds tests for handleResetSingleDefence

* improve validation on handleGetDefenceStatus

* removes extraneous stuff from test objects

* adds tests to make sure defences aren't chngeable for level 1

* add loop to test levels 1 and 2 with same code

* adds remaining tests for defence controller integration

* handleResetSingleDefence takes level parameter now

* adds level to frontend call to reset defence config item. Types all the prop drillings properly

* some renamings and fill in gaps in tests

* adds another missing test

* tweaks

* fix test suite

* replace defence search with defences.some and replace cuurrentDefences === undefined check with not currentDefences

* change defenceId for generic falsy check

* improve guards for level and config

* improve guards for level and config

* correct test for configuring a config item

* updates error message when not allowed to activate or modify defences

* rescturecure defence controller integration tests

* change other tests to use test.each rather than foreach

* uses isValidLevel

* improve validateFilterConfig

* just pass qa defence rather than all defences

* fix tests

* uses type intersection to define QaLlmDefence

* only gedDefencesFrDTOs if it is defined
  • Loading branch information
@pmarsh-scottlogic
pmarsh-scottlogic authored Mar 12, 2024
1 parent ddcf704 commit 255f559
Show file tree
Hide file tree
Showing 24 changed files with 1,316 additions and 208 deletions.
25 changes: 14 additions & 11 deletions backend/src/controller/chatController.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { Response } from 'express';

import { defaultDefences } from '@src/defaultDefences';
import {
transformMessage,
detectTriggeredInputDefences,
Expand All @@ -21,7 +22,7 @@ import {
ChatInfoMessage,
chatInfoMessageTypes,
} from '@src/models/chatMessage';
import { Defence } from '@src/models/defence';
import { DEFENCE_ID, Defence, QaLlmDefence } from '@src/models/defence';
import { EmailInfo } from '@src/models/email';
import { LEVEL_NAMES } from '@src/models/level';
import { chatGptSendMessage } from '@src/openai';
Expand Down Expand Up @@ -94,8 +95,7 @@ async function handleChatWithoutDefenceDetection(
chatResponse: ChatHttpResponse,
currentLevel: LEVEL_NAMES,
chatModel: ChatModel,
chatHistory: ChatMessage[],
defences: Defence[]
chatHistory: ChatMessage[]
): Promise<LevelHandlerResponse> {
console.log(`User message: '${message}'`);

Expand All @@ -106,7 +106,6 @@ async function handleChatWithoutDefenceDetection(

const openAiReply = await chatGptSendMessage(
updatedChatHistory,
defences,
chatModel,
currentLevel
);
Expand Down Expand Up @@ -149,11 +148,15 @@ async function handleChatWithDefenceDetection(
}'`
);

const qaLlmDefence = defences.find(
(defence) => defence.id === DEFENCE_ID.QA_LLM
) as QaLlmDefence | undefined;

const openAiReplyPromise = chatGptSendMessage(
chatHistoryWithNewUserMessages,
defences,
chatModel,
currentLevel
currentLevel,
qaLlmDefence
);

// run input defence detection and chatGPT concurrently
Expand Down Expand Up @@ -240,14 +243,15 @@ async function handleChatToGPT(req: OpenAiChatRequest, res: Response) {
? req.session.chatModel
: defaultChatModel;

const defences =
req.session.levelState[currentLevel].defences ?? defaultDefences;

const currentChatHistory = setSystemRoleInChatHistory(
currentLevel,
req.session.levelState[currentLevel].defences,
defences,
req.session.levelState[currentLevel].chatHistory
);

const defences = [...req.session.levelState[currentLevel].defences];

let levelResult: LevelHandlerResponse;
try {
if (currentLevel < LEVEL_NAMES.LEVEL_3) {
Expand All @@ -256,8 +260,7 @@ async function handleChatToGPT(req: OpenAiChatRequest, res: Response) {
initChatResponse,
currentLevel,
chatModel,
currentChatHistory,
defences
currentChatHistory
);
} else {
levelResult = await handleChatWithDefenceDetection(
Expand Down
231 changes: 171 additions & 60 deletions backend/src/controller/defenceController.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ import {
resetDefenceConfig,
} from '@src/defence';
import { DefenceActivateRequest } from '@src/models/api/DefenceActivateRequest';
import { DefenceConfigResetRequest } from '@src/models/api/DefenceConfigResetRequest';
import { DefenceConfigItemResetRequest } from '@src/models/api/DefenceConfigResetRequest';
import { DefenceConfigureRequest } from '@src/models/api/DefenceConfigureRequest';
import { DefenceConfigItem } from '@src/models/defence';
import { LEVEL_NAMES } from '@src/models/level';
import { LEVEL_NAMES, isValidLevel } from '@src/models/level';

import { sendErrorResponse } from './handleError';

Expand All @@ -24,46 +24,99 @@ function configValueExceedsCharacterLimit(config: DefenceConfigItem[]) {
}

function handleDefenceActivation(req: DefenceActivateRequest, res: Response) {
const defenceId = req.body.defenceId;
const level = req.body.level;

if (defenceId && level !== undefined) {
// activate the defence
req.session.levelState[level].defences = activateDefence(
defenceId,
req.session.levelState[level].defences
);
res.status(200).send();
} else {
res.status(400).send();
const { defenceId, level } = req.body;

if (!defenceId) {
sendErrorResponse(res, 400, 'Missing defenceId');
return;
}

if (!Number.isFinite(level) || level === undefined) {
sendErrorResponse(res, 400, 'Missing level');
return;
}

if (!isValidLevel(level)) {
sendErrorResponse(res, 400, 'Invalid level');
return;
}

const currentDefences = req.session.levelState[level].defences;

if (!currentDefences) {
sendErrorResponse(res, 400, 'You cannot activate defences on this level');
return;
}

if (!currentDefences.some((defence) => defence.id === defenceId)) {
sendErrorResponse(res, 400, `Defence with id ${defenceId} not found`);
return;
}

req.session.levelState[level].defences = activateDefence(
defenceId,
currentDefences
);
res.status(200).send();
}

function handleDefenceDeactivation(req: DefenceActivateRequest, res: Response) {
// id of the defence
const defenceId = req.body.defenceId;
const level = req.body.level;
if (defenceId && level !== undefined) {
// deactivate the defence
req.session.levelState[level].defences = deactivateDefence(
defenceId,
req.session.levelState[level].defences
);
res.send();
} else {
res.status(400);
res.send();
const { defenceId, level } = req.body;

if (!defenceId) {
sendErrorResponse(res, 400, 'Missing defenceId');
return;
}

if (!Number.isFinite(level) || level === undefined) {
sendErrorResponse(res, 400, 'Missing level');
return;
}

if (!isValidLevel(level)) {
sendErrorResponse(res, 400, 'Invalid level');
return;
}

const currentDefences = req.session.levelState[level].defences;

if (!currentDefences) {
sendErrorResponse(res, 400, 'You cannot deactivate defences on this level');
return;
}

if (!currentDefences.some((defence) => defence.id === defenceId)) {
sendErrorResponse(res, 400, `Defence with id ${defenceId} not found`);
return;
}

req.session.levelState[level].defences = deactivateDefence(
defenceId,
currentDefences
);
res.status(200).send();
}

function handleConfigureDefence(req: DefenceConfigureRequest, res: Response) {
// id of the defence
const defenceId = req.body.defenceId;
const config = req.body.config;
const level = req.body.level;
const { defenceId, level, config } = req.body;

if (!defenceId) {
sendErrorResponse(res, 400, 'Missing defenceId');
return;
}

if (!Number.isFinite(level) || level === undefined) {
sendErrorResponse(res, 400, 'Missing level');
return;
}

if (!defenceId || !config || level === undefined) {
sendErrorResponse(res, 400, 'Missing defenceId, config or level');
if (!isValidLevel(level)) {
sendErrorResponse(res, 400, 'Invalid level');
return;
}

if (!config) {
sendErrorResponse(res, 400, 'Missing config');
return;
}

Expand All @@ -72,48 +125,106 @@ function handleConfigureDefence(req: DefenceConfigureRequest, res: Response) {
return;
}

// configure the defence
const currentDefences = req.session.levelState[level].defences;

if (!currentDefences || level !== LEVEL_NAMES.SANDBOX) {
sendErrorResponse(res, 400, 'You cannot configure defences on this level');
return;
}

const defence = currentDefences.find((defence) => defence.id === defenceId);

if (defence === undefined) {
sendErrorResponse(res, 400, `Defence with id ${defenceId} not found`);
return;
}

req.session.levelState[level].defences = configureDefence(
defenceId,
req.session.levelState[level].defences,
currentDefences,
config
);
res.send();
}

function handleResetSingleDefence(
req: DefenceConfigResetRequest,
function handleResetDefenceConfigItem(
req: DefenceConfigItemResetRequest,
res: Response
) {
const defenceId = req.body.defenceId;
const configId = req.body.configId;
const level = LEVEL_NAMES.SANDBOX; //configuration only available in sandbox
if (defenceId && configId) {
req.session.levelState[level].defences = resetDefenceConfig(
defenceId,
configId,
req.session.levelState[level].defences
const { defenceId, configItemId, level } = req.body;

if (!defenceId) {
sendErrorResponse(res, 400, 'Missing defenceId');
return;
}

if (!configItemId) {
sendErrorResponse(res, 400, 'Missing configId');
return;
}

if (!Number.isFinite(level) || level === undefined) {
sendErrorResponse(res, 400, 'Missing level');
return;
}

if (!isValidLevel(level)) {
sendErrorResponse(res, 400, 'Invalid level');
return;
}

const currentDefences = req.session.levelState[level].defences;

if (!currentDefences || level !== LEVEL_NAMES.SANDBOX) {
sendErrorResponse(
res,
400,
'You cannot reset defence config items on this level'
);
const updatedDefenceConfig: DefenceConfigItem | undefined =
req.session.levelState[level].defences
.find((defence) => defence.id === defenceId)
?.config.find((config) => config.id === configId);

if (updatedDefenceConfig) {
res.send(updatedDefenceConfig);
} else {
res.status(400);
res.send();
}
} else {
res.status(400);
res.send();
return;
}

const defence = currentDefences.find((defence) => defence.id === defenceId);

if (defence === undefined) {
sendErrorResponse(res, 400, `Defence with id ${defenceId} not found`);
return;
}

const configItem = defence.config.find(
(configItem) => configItem.id === configItemId
);

if (configItem === undefined) {
sendErrorResponse(
res,
400,
`Config item with id ${configItemId} not found for defence with id ${defenceId}`
);
return;
}

req.session.levelState[level].defences = resetDefenceConfig(
defenceId,
configItemId,
currentDefences
);

const updatedDefences = req.session.levelState[level].defences;
if (updatedDefences === undefined) {
sendErrorResponse(res, 500, 'Something went whacky');
return;
}
const updatedDefenceConfig = updatedDefences
.find((defence) => defence.id === defenceId)
?.config.find((config) => config.id === configItemId);

res.send(updatedDefenceConfig);
}

export {
handleDefenceActivation,
handleDefenceDeactivation,
handleConfigureDefence,
handleResetSingleDefence,
handleResetDefenceConfigItem,
};
6 changes: 4 additions & 2 deletions backend/src/models/api/DefenceConfigResetRequest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@ import {
DEFENCE_ID,
DefenceConfigItem,
} from '@src/models/defence';
import { LEVEL_NAMES } from '@src/models/level';

export type DefenceConfigResetRequest = Request<
export type DefenceConfigItemResetRequest = Request<
never,
DefenceConfigItem,
{
defenceId?: DEFENCE_ID;
configId?: DEFENCE_CONFIG_ITEM_ID;
configItemId?: DEFENCE_CONFIG_ITEM_ID;
level?: LEVEL_NAMES;
},
never
>;
11 changes: 10 additions & 1 deletion backend/src/models/defence.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,20 @@ type DefenceConfigItem = {
value: string;
};

type QaLlmDefence = Defence & {
id: DEFENCE_ID.QA_LLM;
};

type Defence = {
id: DEFENCE_ID;
config: DefenceConfigItem[];
isActive: boolean;
};

export { DEFENCE_ID };
export type { Defence, DefenceConfigItem, DEFENCE_CONFIG_ITEM_ID };
export type {
Defence,
DefenceConfigItem,
DEFENCE_CONFIG_ITEM_ID,
QaLlmDefence,
};
Loading

0 comments on commit 255f559

Please sign in to comment.