Skip to content

Commit

Permalink
Server: implement support for Pagination API on CMH (#390)
Browse files Browse the repository at this point in the history 
* Update /rest/variants/match request

* Do the pagination

* Update to support CMH

* Fix 2nd page fetch fail

* Add liftover for positions

* Fix requests
  • Loading branch information
@frewmack
frewmack authored Apr 15, 2024
1 parent 5ceb6e9 commit 52f973b
Show file tree
Hide file tree
Showing 6 changed files with 247 additions and 166 deletions.
221 changes: 114 additions & 107 deletions server/src/resolvers/getVariantsResolver/adapters/cmhAdapter.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import axios, { AxiosError, AxiosResponse } from 'axios';
import axios, { AxiosError } from 'axios';
import jwtDecode from 'jwt-decode';
import { URLSearchParams } from 'url';
import { v4 as uuidv4 } from 'uuid';
Expand All @@ -12,16 +12,21 @@ import {
VariantResponseFields,
G4RDFamilyQueryResult,
G4RDPatientQueryResult,
G4RDVariantQueryResult,
Disorder,
IndividualInfoFields,
PhenotypicFeaturesFields,
NonStandardFeature,
Feature,
PTVariantArray,
} from '../../../types';
import { getFromCache, putInCache } from '../../../utils/cache';
import { timeit, timeitAsync } from '../../../utils/timeit';
import resolveAssembly from '../utils/resolveAssembly';
import fetchPhenotipsVariants from '../utils/fetchPhenotipsVariants';
import fetchPhenotipsPatients from '../utils/fetchPhenotipsPatients';
import { QueryResponseError } from '../utils/queryResponseError';
import resolveChromosome from '../utils/resolveChromosome';
import { liftoverOne } from '../utils/liftOver';

/* eslint-disable camelcase */

Expand All @@ -45,8 +50,8 @@ const _getCMHNodeQuery = async ({
input: { gene: geneInput, variant },
}: QueryInput): Promise<VariantQueryResponse> => {
let CMHNodeQueryError: CMHNodeQueryError | null = null;
let CMHVariantQueryResponse: null | AxiosResponse<G4RDVariantQueryResult> = null;
let CMHPatientQueryResponse: null | AxiosResponse<G4RDPatientQueryResult> = null;
let CMHVariants: null | PTVariantArray = null;
let CMHPatientQueryResponse: null | G4RDPatientQueryResult[] = null;
const FamilyIds: null | Record<string, string> = {}; // <PatientId, FamilyId>
let Authorization = '';
try {
Expand All @@ -60,52 +65,57 @@ const _getCMHNodeQuery = async ({
source: SOURCE_NAME,
};
}
const url = `${process.env.CMH_URL}/rest/variants/match`;

if (!variant.assemblyId.includes('38')) {
// convert to GRCh38 if the position isn't in 38
const pos = resolveChromosome(geneInput.position);
const lifted = await liftoverOne(
{
chromosome: pos.chromosome,
start: Number(pos.start),
end: Number(pos.end),
},
'GRCh38',
variant.assemblyId
);
geneInput.position = `${pos.chromosome}:${lifted.start}-${lifted.end}`;
}

/* eslint-disable @typescript-eslint/no-unused-vars */
const { position, ...gene } = geneInput;
variant.assemblyId = 'GRCh38';
// the replacement
try {
CMHVariantQueryResponse = await axios.post<G4RDVariantQueryResult>(
url,
CMHVariants = await fetchPhenotipsVariants(
process.env.CMH_URL as string,
geneInput,
variant,
getAuthHeader,
{
gene,
variant,
},
{
headers: {
Authorization,
'Content-Type': 'application/json',
Accept: 'application/json',
'X-Gene42-Secret': `${process.env.CMH_GENE42_SECRET}`, //
},
'X-Gene42-Secret': `${process.env.CMH_GENE42_SECRET}`,
}
);

// Get patients info
if (CMHVariantQueryResponse) {
let individualIds = CMHVariantQueryResponse.data.results
.map(v => v.individual.individualId!)
.filter(Boolean); // Filter out undefined and null values.
if (CMHVariants && CMHVariants.length > 0) {
let individualIds = CMHVariants.flatMap(v => v.individualIds).filter(Boolean); // Filter out undefined and null values.

// Get all unique individual Ids.
individualIds = [...new Set(individualIds)];

if (individualIds.length > 0) {
const patientUrl = `${process.env.CMH_URL}/rest/patients/fetch?${individualIds
.map(id => `id=${id}`)
.join('&')}`;

CMHPatientQueryResponse = await axios.get<G4RDPatientQueryResult>(
new URL(patientUrl).toString(),
{
headers: {
Authorization,
'Content-Type': 'application/json',
Accept: 'application/json',
try {
CMHPatientQueryResponse = await fetchPhenotipsPatients(
process.env.CMH_URL!,
individualIds,
getAuthHeader,
{
'X-Gene42-Secret': `${process.env.CMH_GENE42_SECRET}`,
},
}
);
}
);
} catch (e) {
logger.error(JSON.stringify(e));
CMHPatientQueryResponse = [];
}

// Get Family Id for each patient.
const patientFamily = axios.create({
Expand Down Expand Up @@ -133,14 +143,18 @@ const _getCMHNodeQuery = async ({
}
}
} catch (e: any) {
if (e instanceof QueryResponseError) {
e.source = SOURCE_NAME;
}
logger.error(e);
logger.debug(JSON.stringify(e));
CMHNodeQueryError = e;
}

return {
data: transformCMHQueryResponse(
(CMHVariantQueryResponse?.data as G4RDVariantQueryResult) || [],
(CMHPatientQueryResponse?.data as G4RDPatientQueryResult) || [],
(CMHVariants as PTVariantArray) || [],
(CMHPatientQueryResponse as G4RDPatientQueryResult[]) || [],
FamilyIds
),
error: transformCMHNodeErrorResponse(CMHNodeQueryError),
Expand Down Expand Up @@ -204,99 +218,92 @@ export const transformCMHNodeErrorResponse: ErrorTransformer<CMHNodeQueryError>
const isObserved = (feature: Feature | NonStandardFeature) =>
feature.observed === 'yes' ? true : feature.observed === 'no' ? false : undefined;

export const transformCMHQueryResponse: ResultTransformer<G4RDVariantQueryResult> = timeit(
export const transformCMHQueryResponse: ResultTransformer<PTVariantArray> = timeit(
'transformCMHQueryResponse'
)(
(
variantResponse: G4RDVariantQueryResult,
variants: PTVariantArray,
patientResponse: G4RDPatientQueryResult[],
familyIds: Record<string, string>
) => {
const individualIdsMap = Object.fromEntries(patientResponse.map(p => [p.id, p]));

return (variantResponse.results || []).map(r => {
return (variants || []).flatMap(r => {
/* eslint-disable @typescript-eslint/no-unused-vars */
r.variant.assemblyId = resolveAssembly(r.variant.assemblyId);
const { individual, contactInfo } = r;
const { individualIds } = r;

const patient = individual.individualId ? individualIdsMap[individual.individualId] : null;
return individualIds.map(individualId => {
const patient = individualIdsMap[individualId];

let info: IndividualInfoFields = {};
let ethnicity: string = '';
let disorders: Disorder[] = [];
let phenotypicFeatures: PhenotypicFeaturesFields[] = individual.phenotypicFeatures || [];
const contactInfo: string = patient.contact
? patient.contact.map(c => c.name).join(', ')
: '';

if (patient) {
const candidateGene = (patient.genes ?? []).map(g => g.gene).join('\n');
const classifications = (patient.genes ?? []).map(g => g.status).join('\n');
const diagnosis = patient.clinicalStatus;
const solved = patient.solved ? patient.solved.status : '';
const clinicalStatus = patient.clinicalStatus;
disorders = patient.disorders.filter(({ label }) => label !== 'affected') as Disorder[];
ethnicity = Object.values(patient.ethnicity)
.flat()
.map(p => p.trim())
.join(', ');
info = {
solved,
candidateGene,
diagnosis,
classifications,
clinicalStatus,
disorders,
};
// variant response contains all phenotypic features listed,
// even if some of them are explicitly _not_ observed by clinician and recorded as such
if (individual.phenotypicFeatures !== null && individual.phenotypicFeatures !== undefined) {
let info: IndividualInfoFields = {};
let ethnicity: string = '';
let disorders: Disorder[] = [];
let phenotypicFeatures: PhenotypicFeaturesFields[] = [];

if (patient) {
const candidateGene = (patient.genes ?? []).map(g => g.gene).join('\n');
const classifications = (patient.genes ?? []).map(g => g.status).join('\n');
const diagnosis = patient.clinicalStatus;
const solved = patient.solved ? patient.solved.status : '';
const clinicalStatus = patient.clinicalStatus;
disorders = patient.disorders.filter(({ label }) => label !== 'affected') as Disorder[];
ethnicity = Object.values(patient.ethnicity)
.flat()
.map(p => p.trim())
.join(', ');
info = {
solved,
candidateGene,
diagnosis,
classifications,
clinicalStatus,
disorders,
};
// variant response contains all phenotypic features listed,
// even if some of them are explicitly _not_ observed by clinician and recorded as such
const features = [...(patient.features ?? []), ...(patient.nonstandard_features ?? [])];
const detailedFeatures = individual.phenotypicFeatures;
// build list of features the safe way
const detailedFeatureMap = Object.fromEntries(
detailedFeatures.map(feat => [feat.phenotypeId, feat])
);
const finalFeatures: PhenotypicFeaturesFields[] = features.map(feat => {
if (feat.id === undefined) {
return {
ageOfOnset: null,
dateOfOnset: null,
levelSeverity: null,
onsetType: null,
phenotypeId: feat.id,
phenotypeLabel: feat.label,
observed: isObserved(feat),
};
}
return {
...detailedFeatureMap[feat.id],
// ageOfOnset: null,
// dateOfOnset: null,
levelSeverity: null,
// onsetType: null,
phenotypeId: feat.id,
phenotypeLabel: feat.label,
observed: isObserved(feat),
};
});
phenotypicFeatures = finalFeatures;
}
}

const variant: VariantResponseFields = {
alt: r.variant.alt,
assemblyId: r.variant.assemblyId,
callsets: r.variant.callsets,
end: r.variant.end,
ref: r.variant.ref,
start: r.variant.start,
chromosome: r.variant.chromosome,
info: r.variant.info,
};
const variant: VariantResponseFields = {
alt: r.variant.alt,
assemblyId: r.variant.assemblyId,
callsets: r.variant.callsets,
end: r.variant.end,
ref: r.variant.ref,
start: r.variant.start,
chromosome: r.variant.chromosome,
info: r.variant.info,
};

let familyId: string = '';
if (individual.individualId) familyId = familyIds[individual.individualId];
const familyId: string = familyIds[individualId];

const individualResponseFields: IndividualResponseFields = {
...individual,
ethnicity,
info,
familyId,
phenotypicFeatures,
};
return { individual: individualResponseFields, variant, contactInfo, source: SOURCE_NAME };
const individualResponseFields: IndividualResponseFields = {
sex: patient.sex,
ethnicity,
info,
familyId,
phenotypicFeatures,
individualId,
};
return { individual: individualResponseFields, variant, contactInfo, source: SOURCE_NAME };
});
});
}
);
Expand Down
30 changes: 23 additions & 7 deletions server/src/resolvers/getVariantsResolver/adapters/g4rdAdapter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ import { timeit, timeitAsync } from '../../../utils/timeit';
import resolveAssembly from '../utils/resolveAssembly';
import fetchPhenotipsVariants from '../utils/fetchPhenotipsVariants';
import fetchPhenotipsPatients from '../utils/fetchPhenotipsPatients';
import { QueryResponseError } from '../utils/queryResponseError';
import { liftoverOne } from '../utils/liftOver';
import resolveChromosome from '../utils/resolveChromosome';

/* eslint-disable camelcase */

Expand Down Expand Up @@ -55,6 +58,22 @@ const _getG4rdNodeQuery = async ({
source: SOURCE_NAME,
};
}

if (variant.assemblyId.includes('38')) {
// convert to 37 if the position is in 38
const pos = resolveChromosome(geneInput.position);
const lifted = await liftoverOne(
{
chromosome: pos.chromosome,
start: Number(pos.start),
end: Number(pos.end),
},
'GRCh37',
variant.assemblyId
);
geneInput.position = `${pos.chromosome}:${lifted.start}-${lifted.end}`;
}

/* eslint-disable @typescript-eslint/no-unused-vars */
variant.assemblyId = 'GRCh37';
// For g4rd node, assemblyId is a required field as specified in this sample request:
Expand All @@ -71,7 +90,6 @@ const _getG4rdNodeQuery = async ({

// Get patients info
if (G4RDVariants && G4RDVariants.length > 0) {
logger.debug(`G4RDVariants length: ${G4RDVariants.length}`);
let individualIds = G4RDVariants.flatMap(v => v.individualIds).filter(Boolean); // Filter out undefined and null values.

// Get all unique individual Ids.
Expand All @@ -98,13 +116,8 @@ const _getG4rdNodeQuery = async ({
},
});

logger.debug('Begin fetching family IDs');

const familyResponses = await Promise.allSettled(
individualIds.map((id, i) => {
if (i % 50 === 0 || i === individualIds.length - 1) {
logger.debug(`Fetching family ${i + 1} of ${individualIds.length}`);
}
return patientFamily.get<G4RDFamilyQueryResult>(
new URL(`${process.env.G4RD_URL}/rest/patients/${id}/family`).toString()
);
Expand All @@ -131,6 +144,9 @@ const _getG4rdNodeQuery = async ({
};
}
} catch (e: any) {
if (e instanceof QueryResponseError) {
e.source = SOURCE_NAME;
}
logger.error(e);
G4RDNodeQueryError = e;
}
Expand Down Expand Up @@ -230,7 +246,7 @@ export const transformG4RDQueryResponse: ResultTransformer<PTVariantArray> = tim
const patient = individualIdsMap[individualId];

const contactInfo: string = patient.contact
? patient.contact.map(c => c.name).join(' ,')
? patient.contact.map(c => c.name).join(', ')
: '';

let info: IndividualInfoFields = {};
Expand Down
2 changes: 1 addition & 1 deletion server/src/resolvers/getVariantsResolver/getVariantsResolver.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import fetchCaddAnnotations from './utils/fetchCaddAnnotations';
import annotateCadd from './utils/annotateCadd';
import fetchGnomadAnnotations from './utils/fetchGnomadAnnotations';
import annotateGnomad from './utils/annotateGnomad';
import liftover from './utils/liftOver';
import { liftover } from './utils/liftOver';
import { QueryResponseError } from './utils/queryResponseError';
import getG4rdNodeQuery from './adapters/g4rdAdapter';
import { timeitAsync } from '../../utils/timeit';
Expand Down
Loading

0 comments on commit 52f973b

Please sign in to comment.