From f6019ed13ce0ea5db0a911fac4063e0d33892604 Mon Sep 17 00:00:00 2001
From: fibonacci998 <fibonacci998@gmail.com>
Date: Fri, 27 Sep 2024 09:54:54 +0700
Subject: [PATCH] feat: optimize query account stat by split query

---
 ci/config.json.ci                             |  3 +-
 config.json                                   |  3 +-
 .../statistic/account_statistics.service.ts   | 52 +++++++++++++------
 3 files changed, 40 insertions(+), 18 deletions(-)

diff --git a/ci/config.json.ci b/ci/config.json.ci
index 5cc666cce..2ea9ebc32 100644
--- a/ci/config.json.ci
+++ b/ci/config.json.ci
@@ -264,7 +264,8 @@
   },
   "accountStatistics": {
     "numberOfTopRecords": 10,
-    "dayRange": [3, 15, 30]
+    "dayRange": [3, 15, 30],
+    "limitQuery": 5000
   },
   "dailyStatsJobs": {
     "jobPattern": "0 0 0 * * ?"
diff --git a/config.json b/config.json
index 3f82a3081..fadccb4a7 100644
--- a/config.json
+++ b/config.json
@@ -257,7 +257,8 @@
   },
   "accountStatistics": {
     "numberOfTopRecords": 10,
-    "dayRange": [3, 15, 30]
+    "dayRange": [3, 15, 30],
+    "limitQuery": 1000
   },
   "dailyStatsJobs": {
     "jobPattern": "0 0 0 * * ?"
diff --git a/src/services/evm/statistic/account_statistics.service.ts b/src/services/evm/statistic/account_statistics.service.ts
index 2b601c01e..76ad78d33 100644
--- a/src/services/evm/statistic/account_statistics.service.ts
+++ b/src/services/evm/statistic/account_statistics.service.ts
@@ -60,7 +60,6 @@ export default class EVMAccountStatisticsService extends BullableService {
   @QueueHandler({
     queueName: BULL_JOB_NAME.CRAWL_EVM_ACCOUNT_STATISTICS,
     jobName: BULL_JOB_NAME.CRAWL_EVM_ACCOUNT_STATISTICS,
-    // prefix: `horoscope-v2-${config.chainId}`,
   })
   public async handleJob(_payload: { date: string }): Promise<void> {
     const { date } = _payload;
@@ -147,12 +146,18 @@ export default class EVMAccountStatisticsService extends BullableService {
     const now = dayjs.utc().startOf('day').toDate();
 
     const { dayRange } = config.accountStatistics;
-    const [threeDayStat, fifteenDayStat, thirtyDayStat] = await Promise.all([
-      this.getStatsFromSpecificDaysAgo(dayRange[0], now),
-      this.getStatsFromSpecificDaysAgo(dayRange[1], now),
-      this.getStatsFromSpecificDaysAgo(dayRange[2], now),
-    ]);
-
+    const threeDayStat = await this.getStatsFromSpecificDaysAgo(
+      dayRange[0],
+      now
+    );
+    const fifteenDayStat = await this.getStatsFromSpecificDaysAgo(
+      dayRange[1],
+      now
+    );
+    const thirtyDayStat = await this.getStatsFromSpecificDaysAgo(
+      dayRange[2],
+      now
+    );
     const topAccounts = {
       three_days: this.calculateTop(threeDayStat),
       fifteen_days: this.calculateTop(fifteenDayStat),
@@ -252,15 +257,30 @@ export default class EVMAccountStatisticsService extends BullableService {
       .subtract(daysAgo, 'day')
       .startOf('day')
       .toDate();
-    const result = await AccountStatistics.query()
-      .select('address')
-      .sum('amount_sent as amount_sent')
-      .sum('amount_received as amount_received')
-      .sum('tx_sent as tx_sent')
-      .sum('gas_used as gas_used')
-      .where('date', '>=', startTime)
-      .andWhere('date', '<', endTime)
-      .groupBy('address');
+
+    const limit = config.accountStatistics.limitQuery;
+    let offset = 0;
+    const result: AccountStatistics[] = [];
+    while (1) {
+      this.logger.info(`${daysAgo} days ago offset: ${offset}`);
+      // eslint-disable-next-line no-await-in-loop
+      const chunk = await AccountStatistics.query()
+        .select('address')
+        .sum('amount_sent as amount_sent')
+        .sum('amount_received as amount_received')
+        .sum('tx_sent as tx_sent')
+        .sum('gas_used as gas_used')
+        .where('date', '>=', startTime)
+        .andWhere('date', '<', endTime)
+        .groupBy('address')
+        .limit(limit)
+        .offset(offset);
+      if (chunk.length === 0) break;
+      this.logger.info(`${daysAgo} days ago found chunk offset ${offset}`);
+      result.push(...chunk);
+      offset += limit;
+    }
+    this.logger.info(`${daysAgo} done`);
     return result;
   }