[Platform]: batch loading in variant page (#550)

packages/sections/src/constants.js

@@ -6,6 +6,9 @@ export const appCanonicalUrl = "https://platform.opentargets.org";
 
 // Chunk sizes for server side pagination/download.
 export const tableChunkSize = 100;
+export const table2HChunkSize = 200;
+export const table3HChunkSize = 300;
+export const table5HChunkSize = 500;
 export const downloaderChunkSize = 2500;
 export const sectionsBaseSizeQuery = 3500;
 export const sections5kSizeQuery = 5000;
@@ -107,3 +110,9 @@ export const variantConsequenceSource = {
       "The direction is inferred from the strongest effect across all the co-localising QTLs",
   },
 };
+
+export const initialResponse = {
+  data: null,
+  error: null,
+  loading: true,
+};

packages/sections/src/variant/GWASCredibleSets/Body.tsx

@@ -1,4 +1,3 @@
-import { useQuery } from "@apollo/client";
 import {
   Link,
   SectionItem,
@@ -8,9 +7,10 @@ import {
   Tooltip,
   ClinvarStars,
   OtScoreLinearBar,
+  useBatchQuery,
 } from "ui";
 import { Box, Chip } from "@mui/material";
-import { clinvarStarMap, naLabel, sectionsBaseSizeQuery } from "../../constants";
+import { clinvarStarMap, initialResponse, naLabel, table5HChunkSize } from "../../constants";
 import { definition } from ".";
 import Description from "./Description";
 import GWAS_CREDIBLE_SETS_QUERY from "./GWASCredibleSetsQuery.gql";
@@ -19,6 +19,8 @@ import { mantissaExponentComparator, variantComparator } from "../../utils/compa
 import PheWasPlot from "./PheWasPlot";
 import { faArrowRightToBracket } from "@fortawesome/free-solid-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { useEffect, useState } from "react";
+import { responseType } from "ui/src/types/response";
 
 type getColumnsType = {
   id: string;
@@ -225,13 +227,27 @@ type BodyProps = {
 function Body({ id, entity }: BodyProps) {
   const variables = {
     variantId: id,
-    size: sectionsBaseSizeQuery,
   };
 
-  const request = useQuery(GWAS_CREDIBLE_SETS_QUERY, {
-    variables,
+  const [request, setRequest] = useState<responseType>(initialResponse);
+
+  const getAllGwasData = useBatchQuery({
+    query: GWAS_CREDIBLE_SETS_QUERY,
+    variables: {
+      variantId: id,
+      size: table5HChunkSize,
+      index: 0,
+    },
+    dataPath: "data.variant.gwasCredibleSets",
+    size: table5HChunkSize,
   });
 
+  useEffect(() => {
+    getAllGwasData().then(r => {
+      setRequest(r);
+    });
+  }, []);
+
   return (
     <SectionItem
       definition={definition}

packages/sections/src/variant/GWASCredibleSets/GWASCredibleSetsQuery.gql

@@ -1,9 +1,9 @@
-query GWASCredibleSetsQuery($variantId: String!, $size: Int!) {
+query GWASCredibleSetsQuery($variantId: String!, $size: Int!, $index: Int!) {
   variant(variantId: $variantId) {
     id
     referenceAllele
     alternateAllele
-    gwasCredibleSets: credibleSets(studyTypes: [gwas], page: { size: $size, index: 0 }) {
+    gwasCredibleSets: credibleSets(studyTypes: [gwas], page: { size: $size, index: $index }) {
       count
       rows {
         studyLocusId

packages/sections/src/variant/QTLCredibleSets/Body.tsx

@@ -1,4 +1,3 @@
-import { useQuery } from "@apollo/client";
 import {
   Link,
   SectionItem,
@@ -7,14 +6,16 @@ import {
   OtTable,
   Tooltip,
   ClinvarStars,
+  useBatchQuery,
 } from "ui";
 import { Box, Chip } from "@mui/material";
-import { clinvarStarMap, naLabel, sectionsBaseSizeQuery } from "../../constants";
+import { clinvarStarMap, initialResponse, naLabel, table5HChunkSize } from "../../constants";
 import { definition } from ".";
 import Description from "./Description";
 import QTL_CREDIBLE_SETS_QUERY from "./QTLCredibleSetsQuery.gql";
 import { mantissaExponentComparator, variantComparator } from "../../utils/comparators";
-import { ReactNode } from "react";
+import { ReactNode, useEffect, useState } from "react";
+import { responseType } from "ui/src/types/response";
 import { faArrowRightToBracket } from "@fortawesome/free-solid-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 
@@ -173,7 +174,8 @@ function getColumns({ id, referenceAllele, alternateAllele }: getColumnsType) {
     {
       id: "confidence",
       label: "Fine-mapping confidence",
-      tooltip: "Fine-mapping confidence based on the quality of the linkage-desequilibrium information available and fine-mapping method",
+      tooltip:
+        "Fine-mapping confidence based on the quality of the linkage-desequilibrium information available and fine-mapping method",
       sortable: true,
       renderCell: ({ confidence }) => {
         if (!confidence) return naLabel;
@@ -209,13 +211,27 @@ type BodyProps = {
 function Body({ id, entity }: BodyProps): ReactNode {
   const variables = {
     variantId: id,
-    size: sectionsBaseSizeQuery,
   };
 
-  const request = useQuery(QTL_CREDIBLE_SETS_QUERY, {
-    variables,
+  const [request, setRequest] = useState<responseType>(initialResponse);
+
+  const getAllQtlData = useBatchQuery({
+    query: QTL_CREDIBLE_SETS_QUERY,
+    variables: {
+      variantId: id,
+      size: table5HChunkSize,
+      index: 0,
+    },
+    dataPath: "data.variant.qtlCredibleSets",
+    size: table5HChunkSize,
   });
 
+  useEffect(() => {
+    getAllQtlData().then(r => {
+      setRequest(r);
+    });
+  }, []);
+
   return (
     <SectionItem
       definition={definition}

packages/sections/src/variant/QTLCredibleSets/QTLCredibleSetsQuery.gql

@@ -1,11 +1,11 @@
-query QTLCredibleSetsQuery($variantId: String!, $size: Int!) {
+query QTLCredibleSetsQuery($variantId: String!, $size: Int!, $index: Int!) {
   variant(variantId: $variantId) {
     id
     referenceAllele
     alternateAllele
     qtlCredibleSets: credibleSets(
       studyTypes: [sqtl, pqtl, eqtl, tuqtl]
-      page: { size: $size, index: 0 }
+      page: { size: $size, index: $index }
     ) {
       count
       rows {

packages/ui/src/hooks/useBatchQuery.ts

@@ -0,0 +1,98 @@
+import _ from "lodash";
+
+import defaultClient from "../client";
+import { tableChunkSize } from "../constants";
+import { DocumentNode } from "@apollo/client";
+
+type useBatchQueryProps = {
+  query: DocumentNode;
+  variables: Record<string, unknown>;
+  dataPath: string;
+  size?: number;
+  client?: any;
+  rowField?: string;
+  countField?: string;
+};
+/**
+ * Provides a function to asynchronously batch-download a whole dataset from
+ * the backend.
+ *
+ * The function uses the parameter downloaderChunkSize from the configuration.js
+ * file to determine the size of the chunks to fetch.
+ *
+ * @param {import('graphql').DocumentNode} query Query to run to fetch the data.
+ * @param {import('apollo-client').QueryOptions} variables Variables object for the query.
+ * @param {string} dataPath Path where the data array, row count and cursor are inside the query's result.
+ * @param {string} [rowField=rows] field in dataPath containing the rows. Default: 'rows'.
+ * @param {string} [countField=count] field in dataPath containing the row count. Default: 'count'.
+ *
+ * @returns {Function} Function that will fetch the whole dataset.
+ */
+function useBatchQuery({
+  query,
+  variables,
+  dataPath,
+  client = defaultClient,
+  size = tableChunkSize,
+  rowField = "rows",
+  countField = "count",
+}: useBatchQueryProps): () => Promise<Record<string, unknown>[]> {
+  const rowPath = `${dataPath}.${rowField}`;
+  const countPath = `${dataPath}.${countField}`;
+
+  const getDataChunk = async (index: number) =>
+    client.query({
+      query,
+      variables: { ...variables, index, size },
+    });
+
+  return async function getWholeDataset() {
+    const chunkPromises = [];
+    let data: Array<Record<string, unknown>> = [];
+    let index = 0;
+
+    const firstChunk = await getDataChunk(index);
+
+    data = [...getRows(firstChunk, rowPath)];
+    index += 1;
+
+    const count = Math.ceil(_.get(firstChunk, countPath) / size);
+
+    while (index < count) {
+      chunkPromises.push(getDataChunk(index));
+      index += 1;
+    }
+
+    const remainingChunks = await Promise.all(chunkPromises);
+
+    remainingChunks.forEach(chunk => {
+      data = [...data, ...getRows(chunk, rowPath)];
+    });
+
+    const wholeData = setRows(firstChunk, rowPath, data);
+
+    return wholeData;
+  };
+}
+
+function getRows(data: Record<string, unknown>, dataPath: string) {
+  return _.get(data, dataPath, []);
+}
+
+function setRows(
+  res: Promise<Record<string, unknown>>,
+  dataPath: string,
+  rows: Record<string, unknown>[]
+): Promise<Record<string, unknown>> | null {
+  if (!res || !rows) return null;
+  const wholeRes = structuredClone(res);
+  let obj = wholeRes;
+  const dataPathSplit: string[] = dataPath.split(".");
+  for (let i = 0, len = dataPathSplit.length; i < len; i++) {
+    obj = obj[dataPathSplit[i]];
+  }
+  Object.assign(obj, rows);
+  return wholeRes;
+}
+
+export default useBatchQuery;

packages/ui/src/index.tsx

@@ -67,3 +67,7 @@ export { default as useBatchQuery } from "./hooks/useBatchQuery";
 export { default as useCursorBatchDownloader } from "./hooks/useCursorBatchDownloader";
 export { default as usePermissions } from "./hooks/usePermissions";
 export { default as useDebounce } from "./hooks/useDebounce";
+
+/* TYPES */
+
+// export { default as responseType } from "./types/response";

packages/ui/src/types/response.ts

@@ -0,0 +1,5 @@
+export type responseType = {
+  data: Record<string, unknown> | null;
+  error: Record<string, unknown> | null;
+  loading: boolean;
+};