Feat/ibc app ics20 ( develop ) (#343)

* docs: cw721

* docs: cw20

* fix: update validator as UNSPECIFIED when not found onchain

* fix: remove token and delegator_shares when validator is UNRECOGNIZED

* feat: crawl ibc tao

* feat: add test for unrecognized validator

* feat: crawl genesis ibc tao

* feat: add migration to add index time to block, tx

* fix: code

* fix: code

* feat: ibc app

* feat: ibc app

* fix: allow whitelist query can have depth more than config (#312)

* feat: reindex cw20 service ( main ) (#277)

* feat: reindex cw20 service

* test: test cw20 reindexing

* fix: idiot

* refactor: code

* feat: api admin

* refactor: cw20/cw721 index and add relation ( main ) (#306)

* refactor: cw20 index and add relation

* refactor: code

* fix: duplicate latest migrate contract (#304)

* test: ibc app

* fix: review

* Feat/statistics (#272)

* feat: move statistic feature from v1 to v2 (not tested yet)

* feat: change job params so it can aggregate data at a specific date

* fix: query fields from exact table

* fix: query exact fields

* feat: add logger when done

* feat: finish statistic jobs (not tested)

* feat: create a separate interval job that gets the current date and create statistics jobs with date

* feat: update logic using dayjs and lodash lib

* feat: update cron jobs logic, move api actions to its service folder

* fix: query event from db and group it based on event_id and tx_id

* fix: just use a single job to query all daily data

* fix: move all queries in daily_stats job into a single Promise all

* fix: move dayjs.extend to after import statements

* fix: remove _start

* fix: only count native token in account_stats job

* feat: add api to sync stats from prev dates

* fix: support case when user just pass startDate to api

* fix: fix whitelist graphql

* fix: update column in account_statistic table, update bignum top_tx_sent

* fix: remove drop index in modify account statistic table

---------

Co-authored-by: AnDQK <doquockhanhan@gmail.com>
Co-authored-by: Phan Anh Tuan <fibonacci998@gmail.com>

* refactor: migrate cw721 activity missing smart contract event ( main ) (#322)

* refactor: migrate cw721 activity missing smart contract event

* feat: api

* test: test params

* test: test params

* test: test params

* test: test params

* test: test params

* test: test params

* refactor: code

* refactor: code

* refactor: code

* refactor: code

* refactor: code

* fix: ci

* feat: ibc app ics20

* Fix/account stat with feegrant (#337)

* fix: get use_feegrant_grantee if has, other get tx_fee_payer

* fix: remove code not used

* Update api_gateway.service.ts (#336)

* feat: ibc ics20 send

* feat: ibc ics20 send

* feat: ibc ics20 recv

* feat: ibc ics20 recv

* feat: handle ics20 ack

* feat: handle ics20 timeout

* test: test

* fix: code

* fix: code

* feat: ibc app ics20

* feat: ibc ics20 send

* feat: ibc ics20 send

* feat: ibc ics20 recv

* feat: ibc ics20 recv

* feat: handle ics20 ack

* feat: handle ics20 timeout

* test: test

* fix: code

* fix: code

* fix: cw721 activity missing from/to ( main ) (#294)

* fix: cw721 activity missing from/to

* fix: fill from/to cw721 activity

* fix: fill from/to cw721 activity

* fix: fill from/to cw721 activity

* fix: fill from/to cw721 activity

* refactor: code

* fix: code

* refactor: code

---------

Co-authored-by: Vu Ngoc Quang <quang.vn@outlook.com>

* feat: crawl ibc tao ( main ) (#278)

* feat: crawl ibc tao

* feat: crawl genesis ibc tao

* fix: code

* fix: code

* refactor: code

* fix: test

* refactor: code

* fix: merge code

* refactor: review code

* refactor: review code

* Fix/lint json file (#359)

* fix: display json on multiple lines

* fix: update launch.json to debug jest in 1 file; use moleculer.config.ts when debug service

* Feat/ibc app ( main ) (#321)

* feat: crawl ibc tao

* feat: crawl genesis ibc tao

* fix: code

* fix: code

* feat: ibc app

* feat: ibc app

* test: ibc app

* fix: review

* refactor: update status

* refactor: code

---------

Co-authored-by: Tuan Phan Anh <38557844+fibonacci998@users.noreply.github.com>
Co-authored-by: Phan Anh Tuan <fibonacci998@gmail.com>
Co-authored-by: Vu Ngoc Quang <quang.vn@outlook.com>
Co-authored-by: AnDQK <doquockhanhan@gmail.com>

ci/config.json.ci

@@ -186,6 +186,12 @@
     "millisecondRepeatJob": 2000,
     "blocksPerCall": 100
   },
+  "crawlIbcIcs20": {
+    "key": "crawlIbcIcs20",
+    "millisecondRepeatJob": 2000,
+    "blocksPerCall": 100,
+    "port": "transfer"
+  },
   "jobReassignMsgIndexToEvent": {
     "millisecondCrawl": 1000,
     "blocksPerCall": 100

config.json

@@ -186,6 +186,12 @@
     "millisecondRepeatJob": 2000,
     "blocksPerCall": 100
   },
+  "crawlIbcIcs20": {
+    "key": "crawlIbcIcs20",
+    "millisecondRepeatJob": 2000,
+    "blocksPerCall": 100,
+    "port": "transfer"
+  },
   "jobReassignMsgIndexToEvent": {
     "millisecondCrawl": 1000,
     "blocksPerCall": 100

migrations/20230823070516_ics20_model.ts

@@ -0,0 +1,21 @@
+import { Knex } from 'knex';
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.createTable('ibc_ics20', (table) => {
+    table.increments();
+    table.integer('ibc_message_id').notNullable().unique();
+    table.string('sender').index();
+    table.string('receiver').index().notNullable();
+    table.decimal('amount', 80, 0).notNullable();
+    table.string('denom').notNullable().index();
+    table.string('status').notNullable();
+    table.string('channel_id').notNullable().index();
+    table.foreign('ibc_message_id').references('ibc_message.id');
+    table.string('sequence_key').notNullable().index();
+    table.string('type').notNullable().index();
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists('ibc_ics20');
+}

src/common/constant.ts

@@ -76,6 +76,7 @@ export const BULL_JOB_NAME = {
   REINDEX_CW20_CONTRACT: 'reindex:cw20-contract',
   REINDEX_CW20_HISTORY: 'reindex:cw20-history',
   CRAWL_IBC_APP: 'crawl:ibc-app',
+  CRAWL_IBC_ICS20: 'crawl:ibc-ics20',
   JOB_REASSIGN_MSG_INDEX_TO_EVENT: 'job:reassign-msg-index-to-event',
   JOB_CREATE_COMPOSITE_INDEX_ATTR_PARTITION:
     'job:create-index-composite-attr-partition',
@@ -284,6 +285,10 @@ export const SERVICE = {
         path: 'v1.Cw20ReindexingService.reindexing',
       },
     },
+    CrawlIBCIcs20Service: {
+      key: 'CrawlIBCIcs20Service',
+      name: 'v1.CrawlIBCIcs20Service',
+    },
     ServicesManager: {
       key: 'ServicesManager',
       name: 'v1.ServicesManager',

src/models/event.ts

@@ -76,6 +76,13 @@ export class Event extends BaseModel {
     };
   }
 
+  getAttributesFrom(attributesType: string[]) {
+    return attributesType.map(
+      (attributeType) =>
+        this.attributes?.find((attr: any) => attr.key === attributeType)?.value
+    );
+  }
+
   static EVENT_TYPE = {
     STORE_CODE: 'store_code',
     SUBMIT_PROPOSAL: 'submit_proposal',

src/models/event_attribute.ts

@@ -114,6 +114,11 @@ export class EventAttribute extends BaseModel {
     SRC_CHANNEL: 'packet_src_channel',
     DST_PORT: 'packet_dst_port',
     DST_CHANNEL: 'packet_dst_channel',
+    DENOM: 'denom',
+    SUCCESS: 'success',
+    REFUND_RECEIVER: 'refund_receiver',
+    REFUND_DENOM: 'refund_denom',
+    REFUND_AMOUNT: 'refund_amount',
   };
 
   static ATTRIBUTE_COMPOSITE_KEY = {

src/models/ibc_ics20.ts

@@ -0,0 +1,93 @@
+/* eslint-disable import/no-cycle */
+import { Model } from 'objection';
+import BaseModel from './base';
+import { IbcMessage } from './ibc_message';
+import { IbcChannel } from './ibc_channel';
+
+export class IbcIcs20 extends BaseModel {
+  id!: number;
+
+  ibc_message_id!: number;
+
+  sender!: string;
+
+  receiver!: string;
+
+  amount!: string;
+
+  denom!: string;
+
+  status!: string;
+
+  channel_id!: string;
+
+  ibc_message!: IbcMessage;
+
+  sequence_key!: string;
+
+  type!: string;
+
+  static get tableName() {
+    return 'ibc_ics20';
+  }
+
+  static get jsonSchema() {
+    return {
+      type: 'object',
+      required: [
+        'receiver',
+        'amount',
+        'denom',
+        'ibc_message_id',
+        'channel_id',
+        'sequence_key',
+        'status',
+        'type',
+      ],
+      properties: {
+        receiver: { type: 'string' },
+        denom: { type: 'string' },
+        ibc_message_id: { type: 'number' },
+        amount: { type: 'string' },
+        channel_id: { type: 'string' },
+        sequence_key: { type: 'string' },
+        status: { type: 'string' },
+        type: { type: 'string' },
+      },
+    };
+  }
+
+  static get relationMappings() {
+    return {
+      ibc_message: {
+        relation: Model.BelongsToOneRelation,
+        modelClass: IbcMessage,
+        join: {
+          from: 'ibc_ics20.ibc_message_id',
+          to: 'ibc_message.id',
+        },
+      },
+      channel: {
+        relation: Model.BelongsToOneRelation,
+        modelClass: IbcChannel,
+        join: {
+          from: 'ibc_ics20.channel_id',
+          to: 'ibc_channel.channel_id',
+        },
+      },
+    };
+  }
+
+  static EVENT_TYPE = {
+    TIMEOUT: 'timeout',
+    FUNGIBLE_TOKEN_PACKET: 'fungible_token_packet',
+    DENOM_TRACE: 'denomination_trace',
+  };
+
+  static STATUS_TYPE = {
+    TIMEOUT: 'timeout',
+    ACK_ERROR: 'ack_error',
+    ACK_SUCCESS: 'ack_success',
+    ONGOING: 'ongoing',
+  };
+}

src/models/ibc_message.ts

@@ -3,6 +3,7 @@ import { Model } from 'objection';
 import BaseModel from './base';
 import { IbcChannel } from './ibc_channel';
 import { TransactionMessage } from './transaction_message';
+import config from '../../config.json' assert { type: 'json' };
 
 export class IbcMessage extends BaseModel {
   id!: number;
@@ -25,6 +26,8 @@ export class IbcMessage extends BaseModel {
 
   data!: any;
 
+  message!: TransactionMessage;
+
   static get tableName() {
     return 'ibc_message';
   }
@@ -90,4 +93,8 @@ export class IbcMessage extends BaseModel {
     ACKNOWLEDGE_PACKET: 'acknowledge_packet',
     TIMEOUT_PACKET: 'timeout_packet',
   };
+
+  static PORTS = {
+    ICS20: config.crawlIbcIcs20.port,
+  };
 }

src/models/index.ts

@@ -29,3 +29,4 @@ export * from './ibc_client';
 export * from './ibc_connection';
 export * from './ibc_channel';
 export * from './ibc_message';
+export * from './ibc_ics20';

src/models/transaction_message.ts

@@ -22,6 +22,8 @@ export class TransactionMessage extends BaseModel {
 
   parent_id!: number;
 
+  events!: Event[];
+
   static get tableName() {
     return 'transaction_message';
   }