Skip to content

Commit

Permalink
fix: improve crawl delegator job (#477)
Browse files Browse the repository at this point in the history 
* fix: improve crawl delegator job

* fix: remove unuse testcase

* fix: remove unuse testcase
  • Loading branch information
@matthew-nguyen-20032023
matthew-nguyen-20032023 authored Nov 16, 2023
1 parent 0586507 commit 91b78dc
Show file tree
Hide file tree
Showing 2 changed files with 110 additions and 169 deletions.
91 changes: 18 additions & 73 deletions src/services/crawl-validator/crawl_delegators.service.ts
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,16 @@
/* eslint-disable no-await-in-loop */
import { Service } from '@ourparentcenter/moleculer-decorators-extended';
import { ServiceBroker } from 'moleculer';
import Long from 'long';
import { fromBase64 } from '@cosmjs/encoding';
import BullableService, { QueueHandler } from '../../base/bullable.service';
import config from '../../../config.json' assert { type: 'json' };
import {
BULL_JOB_NAME,
getLcdClient,
IAuraJSClientFactory,
IPagination,
IValidatorDelegators,
SERVICE,
} from '../../common';
import { BlockCheckpoint, Delegator, Validator } from '../../models';
import knex from '../../common/utils/db_connection';
import { BlockCheckpoint, Validator } from '../../models';

@Service({
name: SERVICE.V1.CrawlDelegatorsService.key,
Expand All @@ -30,7 +26,6 @@ export default class CrawlDelegatorsService extends BullableService {
@QueueHandler({
queueName: BULL_JOB_NAME.CRAWL_DELEGATORS,
jobName: BULL_JOB_NAME.CRAWL_DELEGATORS,
// prefix: `horoscope-v2-${config.chainId}`,
})
public async handleJob(_payload: object): Promise<void> {
this.logger.info('Update validator delegators');
Expand Down Expand Up @@ -60,86 +55,36 @@ export default class CrawlDelegatorsService extends BullableService {
@QueueHandler({
queueName: BULL_JOB_NAME.CRAWL_VALIDATOR_DELEGATORS,
jobName: BULL_JOB_NAME.CRAWL_VALIDATOR_DELEGATORS,
// prefix: `horoscope-v2-${config.chainId}`,
})
public async handleJobCrawlValidatorDelegators(
_payload: IValidatorDelegators
): Promise<void> {
this._lcdClient = await getLcdClient();

const delegations: any[] = [];
let delegators: Delegator[] = [];

let resultCallApi;
let done = false;
const pagination: IPagination = {
limit: Long.fromInt(config.crawlDelegators.queryPageLimit),
const pagination = {
limit: 1,
countTotal: true,
};
const validatorDelegationInfo =
await this._lcdClient.auranw.cosmos.staking.v1beta1.validatorDelegations({
validatorAddr: _payload.address,
pagination,
});

while (!done) {
resultCallApi =
await this._lcdClient.auranw.cosmos.staking.v1beta1.validatorDelegations(
{
validatorAddr: _payload.address,
pagination,
}
);

delegations.push(...resultCallApi.delegation_responses);
if (resultCallApi.pagination.next_key === null) {
done = true;
} else {
pagination.key = fromBase64(resultCallApi.pagination.next_key);
}
}

if (delegations.length > 0) {
delegators = delegations.map((delegate) =>
Delegator.fromJson({
validator_id: _payload.id,
delegator_address: delegate.delegation.delegator_address,
amount: delegate.balance.amount,
})
);
}

const totalDelegator = Number(validatorDelegationInfo.pagination.total);
const latestBlock: BlockCheckpoint | undefined =
await BlockCheckpoint.query()
.where('job_name', BULL_JOB_NAME.CRAWL_BLOCK)
.first();

await knex.transaction(async (trx) => {
await Promise.all([
Delegator.query()
.insert(delegators)
.onConflict(['validator_id', 'delegator_address'])
.merge()
.transacting(trx)
.catch((error) => {
this.logger.error(
`Insert or update validator delegators error: ${_payload.address}`
);
this.logger.error(error);
}),
Delegator.query()
.delete(true)
.whereNotIn(
'delegator_address',
delegators.map((delegate) => delegate.delegator_address)
)
.andWhere('validator_id', _payload.id)
.transacting(trx),
Validator.query()
.patch({
delegators_count: delegations.length,
delegators_last_height: latestBlock
? latestBlock.height
: _payload.height,
})
.where('id', _payload.id)
.transacting(trx),
]);
});
await Validator.query()
.patch({
delegators_count: totalDelegator,
delegators_last_height: latestBlock
? latestBlock.height
: _payload.height,
})
.where('id', _payload.id);
}

public async _start() {
Expand Down
188 changes: 92 additions & 96 deletions test/unit/services/crawl-validator/crawl_delegators.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,8 @@ import {
SigningStargateClient,
assertIsDeliverTxSuccess,
} from '@cosmjs/stargate';
import _ from 'lodash';
import { BULL_JOB_NAME } from '../../../../src/common';
import { BlockCheckpoint, Delegator, Validator } from '../../../../src/models';
import { BlockCheckpoint, Validator } from '../../../../src/models';
import CrawlDelegatorsService from '../../../../src/services/crawl-validator/crawl_delegators.service';
import config from '../../../../config.json' assert { type: 'json' };
import network from '../../../../network.json' assert { type: 'json' };
Expand Down Expand Up @@ -124,28 +123,25 @@ export default class CrawlDelegatorsTest {
height: validator?.delegators_last_height ?? 0,
});

const [updatedValidator, delegators] = await Promise.all([
Validator.query().first(),
Delegator.query(),
]);
const updatedValidator = await Validator.query().first();

expect(updatedValidator?.delegators_count).toEqual(2);
expect(updatedValidator?.delegators_last_height).toEqual(3967500);

expect(
_.omit(
delegators.find(
(del) =>
del.delegator_address ===
'aura1phaxpevm5wecex2jyaqty2a4v02qj7qmvkxyqk'
),
['id']
)
).toEqual({
validator_id: updatedValidator?.id,
delegator_address: 'aura1phaxpevm5wecex2jyaqty2a4v02qj7qmvkxyqk',
amount: '100000000',
});
// expect(
// _.omit(
// delegators.find(
// (del) =>
// del.delegator_address ===
// 'aura1phaxpevm5wecex2jyaqty2a4v02qj7qmvkxyqk'
// ),
// ['id']
// )
// ).toEqual({
// validator_id: updatedValidator?.id,
// delegator_address: 'aura1phaxpevm5wecex2jyaqty2a4v02qj7qmvkxyqk',
// amount: '100000000',
// });

result = await client.undelegateTokens(
'aura1qwexv7c6sm95lwhzn9027vyu2ccneaqa7c24zk',
Expand All @@ -157,80 +153,80 @@ export default class CrawlDelegatorsTest {
assertIsDeliverTxSuccess(result);
}

@Test('Remove not exist validator delegators success')
public async testRemoveValidatorDelegators() {
const amount = coins(2000000, 'uaura');
const memo = 'test undelegate and remove not exist validator delegators';

const wallet = await DirectSecp256k1HdWallet.fromMnemonic(
'notice oak worry limit wrap speak medal online prefer cluster roof addict wrist behave treat actual wasp year salad speed social layer crew genius',
{
prefix: 'aura',
}
);
const client = await SigningStargateClient.connectWithSigner(
network.find((net) => net.chainId === config.chainId)?.RPC[0] ?? '',
wallet,
defaultSigningClientOptions
);

let result = await client.delegateTokens(
'aura1cyyzpxplxdzkeea7kwsydadg87357qnaysj0zm',
'auravaloper1phaxpevm5wecex2jyaqty2a4v02qj7qmhyhvcg',
amount[0],
defaultSendFee,
memo
);
assertIsDeliverTxSuccess(result);

const validator = await Validator.query().first();

await this.crawlDelegatorsService?.handleJobCrawlValidatorDelegators({
id: validator?.id ?? 1,
address: validator?.operator_address ?? '',
height: validator?.delegators_last_height ?? 0,
});

const delegators = await Delegator.query();

expect(
_.omit(
delegators.find(
(del) =>
del.delegator_address ===
'aura1cyyzpxplxdzkeea7kwsydadg87357qnaysj0zm'
),
['id']
)
).toEqual({
validator_id: validator?.id,
delegator_address: 'aura1cyyzpxplxdzkeea7kwsydadg87357qnaysj0zm',
amount: '2000000',
});

result = await client.undelegateTokens(
'aura1cyyzpxplxdzkeea7kwsydadg87357qnaysj0zm',
'auravaloper1phaxpevm5wecex2jyaqty2a4v02qj7qmhyhvcg',
amount[0],
defaultSendFee,
memo
);
assertIsDeliverTxSuccess(result);

await this.crawlDelegatorsService?.handleJobCrawlValidatorDelegators({
id: validator?.id ?? 1,
address: validator?.operator_address ?? '',
height: validator?.delegators_last_height ?? 0,
});

const updateDelegators = await Delegator.query();

expect(
updateDelegators.find(
(del) =>
del.delegator_address ===
'aura1cyyzpxplxdzkeea7kwsydadg87357qnaysj0zm'
)
).toBeUndefined();
}
// @Test('Remove not exist validator delegators success')
// public async testRemoveValidatorDelegators() {
// const amount = coins(2000000, 'uaura');
// const memo = 'test undelegate and remove not exist validator delegators';
//
// const wallet = await DirectSecp256k1HdWallet.fromMnemonic(
// 'notice oak worry limit wrap speak medal online prefer cluster roof addict wrist behave treat actual wasp year salad speed social layer crew genius',
// {
// prefix: 'aura',
// }
// );
// const client = await SigningStargateClient.connectWithSigner(
// network.find((net) => net.chainId === config.chainId)?.RPC[0] ?? '',
// wallet,
// defaultSigningClientOptions
// );
//
// let result = await client.delegateTokens(
// 'aura1cyyzpxplxdzkeea7kwsydadg87357qnaysj0zm',
// 'auravaloper1phaxpevm5wecex2jyaqty2a4v02qj7qmhyhvcg',
// amount[0],
// defaultSendFee,
// memo
// );
// assertIsDeliverTxSuccess(result);
//
// const validator = await Validator.query().first();
//
// await this.crawlDelegatorsService?.handleJobCrawlValidatorDelegators({
// id: validator?.id ?? 1,
// address: validator?.operator_address ?? '',
// height: validator?.delegators_last_height ?? 0,
// });
//
// const delegators = await Delegator.query();
//
// expect(
// _.omit(
// delegators.find(
// (del) =>
// del.delegator_address ===
// 'aura1cyyzpxplxdzkeea7kwsydadg87357qnaysj0zm'
// ),
// ['id']
// )
// ).toEqual({
// validator_id: validator?.id,
// delegator_address: 'aura1cyyzpxplxdzkeea7kwsydadg87357qnaysj0zm',
// amount: '2000000',
// });
//
// result = await client.undelegateTokens(
// 'aura1cyyzpxplxdzkeea7kwsydadg87357qnaysj0zm',
// 'auravaloper1phaxpevm5wecex2jyaqty2a4v02qj7qmhyhvcg',
// amount[0],
// defaultSendFee,
// memo
// );
// assertIsDeliverTxSuccess(result);
//
// await this.crawlDelegatorsService?.handleJobCrawlValidatorDelegators({
// id: validator?.id ?? 1,
// address: validator?.operator_address ?? '',
// height: validator?.delegators_last_height ?? 0,
// });
//
// const updateDelegators = await Delegator.query();
//
// expect(
// updateDelegators.find(
// (del) =>
// del.delegator_address ===
// 'aura1cyyzpxplxdzkeea7kwsydadg87357qnaysj0zm'
// )
// ).toBeUndefined();
// }
}

0 comments on commit 91b78dc

Please sign in to comment.