Improve initial log fetch, add pruning, fix fetch bug (#11)

* Improve initial log fetch, add pruning, fix fetch bug

* Fix

* Fix deposit function call in utils

* Phantom lint

* Various fixes

* Polishing, add cli util functions

* More polish

* Remove dev state load stub

* Unify OEV log fetch and remove extra functions from cli-utils

* Reset pnpm lock file

* Try to reset pnpm lock file to main again

src/accounts-to-watch.ts

@@ -42,7 +42,7 @@ export const getAccountsToWatch = async (startBlockNumber?: number | null) => {
  // For every borrower we check each of their accounts (from getAccountDetails) and calculate the potential
  // liquidation profitability. Accounts that are potentially profitable to liquidate are added to accountsToWatch
  const accountsToWatchBatch = await checkLiquidationPotentialOfAccounts(accountDetails, accountBatch);
- accountsToWatch.concat(accountsToWatchBatch);
+ accountsToWatch.push(...accountsToWatchBatch);
  }
  console.info(`Fetched details of ${accountsToWatch.length} accounts with borrowed ETH`);
 

src/cli-utils.ts

@@ -61,9 +61,13 @@ const main = async () => {
  ethToSend: parseEther(ethToSend),
  });
 
- const depositTx = await OrbitLiquidator.deposit!({ value: parseEther(ethToSend) });
+ const depositTx = await wallet.connect(blastProvider).sendTransaction({
+ value: parseEther(ethToSend),
+ to: await OrbitLiquidator.getAddress(),
+ });
  await depositTx.wait(1);
  console.info('Deposited', { txHash: depositTx.hash });
+
  return;
  }
  case 'withdraw-all-eth': {

src/commons.ts

@@ -42,7 +42,7 @@ export const BORROWER_LOGS_LOOKBACK_BLOCKS = 300; // The number of blocks to loo
 export const MAX_LOG_RANGE_BLOCKS = 10_000; // The maximum number of blocks to fetch in a single call.
 export const MIN_RPC_DELAY_MS = 100; // The minimum delay between RPC calls in milliseconds.
 export const MAX_BORROWER_DETAILS_MULTICALL = 300; // The maximum number of borrowers to fetch details for in a single multicall.
-export const MIN_USD_BORROW = parseEther('0.1'); // normally 20 | TODO // The minimum amount of USD that a borrower must have borrowed to be considered for liquidation.
+export const MIN_USD_BORROW = parseEther(process.env.MIN_USD_BORROW ?? '0.1'); // The minimum amount of USD that a borrower must have borrowed to be considered for liquidation.
 export const REPORT_FULFILLMENT_DELAY_MS = 300_000;
 export const MAX_COLLATERAL_REPAY_PERCENTAGE = 95; // We leave some buffer to be sure there is enough collateral after the interest accrual.
 

src/constants.ts

@@ -1,15 +1,16 @@
 import { parseEther } from 'ethers';
 
-export const SAFE_COLLATERAL_BUFFER_PERCENT = 3;
+// The percentage to use when simulating a transmutation.
+// For example, 0.1 here would test if liquidations are profitable above 100.1% of the current data feed value
+export const SIMULATION_PERCENTAGE = 0.5;
 
 export const contractAddresses = {
  // Blast network
  api3OevEthUsdProxy: '0xCBE95Ba8fF327a1E3e6Bdade4C598277450145B3',
  api3ServerV1: '0x709944a48cAf83535e43471680fDA4905FB3920a',
  externalMulticallSimulator: '0xb45fe2838F47DCCEe00F635785EAF0c723F742E5',
  multicall3: '0xcA11bde05977b3631167028862bE2a173976CA11',
- OrbitLiquidator: process.env.ETHER_LIQUIDATOR_ADDRESS ?? '0x',
- // OrbitLiquidator: '0x66E9CA29cD757E3c7C063163deCDB04feb1fC2bC',
+ OrbitLiquidator: process.env.ETHER_LIQUIDATOR_ADDRESS!,
  orbitSpaceStation: '0x1E18C3cb491D908241D0db14b081B51be7B6e652',
 
  // OEV network
@@ -27,9 +28,7 @@ export const oTokenAddresses = {
  ofwWETH: '0xB51b76C73fB24f472E0dd63Bb8195bD2170Bc65d',
 };
 
-export const MIN_ETH_BORROW = parseEther('0.01');
-
-export const MIN_LIQUIDATION_PROFIT_USD = parseEther('0.01'); // NOTE: USD has 18 decimals, same as ETH.
+export const MIN_LIQUIDATION_PROFIT_USD = parseEther(process.env.MIN_LIQUIDATION_PROFIT_USD ?? '1.1'); // NOTE: USD has 18 decimals, same as ETH.
 
 export const BID_CONDITION = {
  LTE: 0n,

src/oev-bot.ts

@@ -1,6 +1,6 @@
 import type { AwardedBidEvent } from '@api3/contracts/dist/typechain-types/api3-server-v1/OevAuctionHouse';
 import type { TypedEventLog } from '@api3/contracts/dist/typechain-types/common';
-import { Contract, ethers, formatEther } from 'ethers';
+import { Contract, ethers, formatEther, parseEther } from 'ethers';
 import { chunk, range, uniq } from 'lodash';
 
 import { getAccountsToWatch } from './accounts-to-watch';
@@ -49,6 +49,7 @@ import {
  OEV_BID_VALIDITY,
  oevAuctioneerConfig,
  oTokenAddresses,
+ SIMULATION_PERCENTAGE,
 } from './constants';
 
 /**
@@ -71,19 +72,31 @@ import {
  * - Persist accounts to watch loop: periodically commit the accounts to watch store to disk
  */
 export const runBot = async () => {
+ const sleepTime = process.env.MAIN_LOOP_SLEEP_TIME ? parseInt(process.env.MAIN_LOOP_SLEEP_TIME, 10) : 5_000;
+
+ const { logs, lastFetchedBlock } = await getOevNetworkLogs();
+ storage.oevNetworkData = {
+ lastFetchedBlock,
+ logs,
+ };
+
  while (true) {
- const startBlock = 0;
+ const startBlock = storage.oevNetworkData.lastFetchedBlock + 1;
  const endBlock = await oevNetworkProvider.getBlockNumber();
- const logs = await getOevNetworkLogs(startBlock, endBlock);
+ const { logs } = await getOevNetworkLogs(startBlock, endBlock);
 
  console.info('Fetched OEV network logs', { count: logs.length, startBlock, endBlock });
 
  storage.oevNetworkData = {
  lastFetchedBlock: endBlock,
- logs,
+ logs: [...storage.oevNetworkData.logs, ...logs],
  };
 
- if (storage.targetChainData.lastBlock === targetChainDataInitialBlock) {
+ if (logs.length > 0) {
+ storage.oevNetworkData.logs = pruneLogs(storage.oevNetworkData.logs);
+ }
+
+ if (storage.currentlyActiveBid && storage.targetChainData.lastBlock === targetChainDataInitialBlock) {
  await expediteActiveBids(); // NOTE: We want to expedite the active bids, so that the bot can start fresh.
  }
 
@@ -97,16 +110,35 @@ export const runBot = async () => {
  try {
  const { currentlyActiveBid } = storage;
 
- if (currentlyActiveBid) return attemptLiquidation();
- await findOevLiquidation();
+ if (currentlyActiveBid) {
+ await attemptLiquidation();
+ } else {
+ await findOevLiquidation();
+ }
  } catch (e) {
  console.error(`Encountered an error while attempting a liquidation: `, e);
  }
 
- await sleep(5000);
+ await sleep(sleepTime);
  }
 };
 
+export const pruneLogs = (logs: OevNetworkLog[]) => {
+ const oldestLogEntryTimestamp = Date.now() / 1000 - 25 * 60 * 60;
+ const possibleOldLog = logs
+ .map((log, idx) => ({
+ ...log,
+ idx,
+ }))
+ .find((log) => log.eventName === 'AwardedBid' && log.awardDetails.timestamp < oldestLogEntryTimestamp);
+
+ if (!possibleOldLog) {
+ return logs;
+ }
+
+ return logs.slice(possibleOldLog.idx);
+};
+
 const oevEventTopics = [
  oevAuctionHouse.filters.AwardedBid().fragment.topicHash, // Same as ethers.id('AwardedBid(address,bytes32,bytes32,bytes,uint256)')
  oevAuctionHouse.filters.PlacedBid().fragment.topicHash, // Same as ethers.id('PlacedBid(address,bytes32,bytes32,uint256,uint256,bytes,uint32,uint104,uint104)'),
@@ -151,21 +183,43 @@ const decodeOevNetworkLog = (log: ethers.LogDescription): OevNetworkLog => {
  }
 };
 
-const getOevNetworkLogs = async (startBlock: number, endBlock: number) => {
+/**
+ * Collects and processes OEV network logs, starting at the `latest` block in the chain and working backwards until it
+ * hits the oldest allowable block.
+ */
+export const getOevNetworkLogs = async (startBlock?: number, endBlock?: number) => {
  const allLogs: OevNetworkLog[] = [];
- while (startBlock < endBlock) {
- const actualEndBlock = Math.min(startBlock + 10_000, endBlock);
+ let latestBlock = endBlock ?? (await oevNetworkProvider.getBlockNumber());
+
+ if (startBlock && endBlock && startBlock > endBlock) {
+ return { logs: [], lastFetchedBlock: startBlock };
+ }
+
+ const networkLatestBlock = latestBlock;
+ const oldestAllowedTimestamp = Date.now() - 25 * 60 * 60 * 1000; // Auctioneer only considers the last 24 hours
+ let currentTimestamp = Date.now();
+
+ console.log(`Fetching OEV Network logs`);
+ while (currentTimestamp > oldestAllowedTimestamp && latestBlock > (startBlock ?? 0)) {
+ const fromBlock = startBlock ?? Math.max(latestBlock - 10_000, 0);
  const logs = await oevNetworkProvider.getLogs({
- fromBlock: startBlock,
- toBlock: actualEndBlock,
+ fromBlock,
+ toBlock: latestBlock,
  address: oevAuctionHouse.getAddress(),
  topics: [oevEventTopics, ethers.zeroPadValue(wallet.address, 32), oevAuctioneerConfig.bidTopic],
  });
- allLogs.push(...logs.map((log) => decodeOevNetworkLog(oevAuctionHouse.interface.parseLog(log)!)));
- startBlock += 10_000;
+ const decodedLogs = logs.map((log) => decodeOevNetworkLog(oevAuctionHouse.interface.parseLog(log)!));
+ allLogs.unshift(...decodedLogs);
+
+ const fromBlockContents = (await oevNetworkProvider.getBlock(fromBlock))!;
+ currentTimestamp = fromBlockContents.timestamp * 1000;
+
+ latestBlock = fromBlock;
  }
 
- return allLogs;
+ console.log(`Fetched ${allLogs.length} logs, from block ${latestBlock} to ${networkLatestBlock}`);
+
+ return { logs: allLogs, lastFetchedBlock: networkLatestBlock };
 };
 
 interface Bid {
@@ -402,7 +456,7 @@ const findOevLiquidation = async () => {
  // This means that the OEV bot will be on timer to get its bid awarded and to capture the liquidation opportunity.
  // Higher percentage gives more time the bot, but the downside is the accuracy of profit calculation, because it
  // assumes the collateral price remains the same from the bid time to the liquidation capture.
- const transmutationValue = getPercentageValue(currentEthUsdPrice, 100.2);
+ const transmutationValue = getPercentageValue(currentEthUsdPrice, 100 + SIMULATION_PERCENTAGE);
  const ethUsdDapiName = ethers.encodeBytes32String('ETH/USD');
  const dapiTransmutationCalls = await getDapiTransmutationCalls(
  contractAddresses.api3ServerV1,
@@ -465,7 +519,7 @@ const findOevLiquidation = async () => {
  ...dapiTransmutationCalls,
  {
  target: contractAddresses.OrbitLiquidator,
- data: OrbitLiquidator.interface.encodeFunctionData('getAccountDetails', [borrower, oTokenAddresses.oEtherV2]),
+ data: OrbitLiquidator.interface.encodeFunctionData('getAccountDetails', [borrower]),
  },
  ];
  const returndata = await simulateTransmutationMulticall(externalMulticallSimulator, transmutationCalls);
@@ -487,11 +541,14 @@ const findOevLiquidation = async () => {
  acc.tokenBalance > curr.tokenBalance ? acc : curr
  );
 
- const OrbitLiquidatorBalance = await blastProvider.getBalance(contractAddresses.OrbitLiquidator);
+ const orbitLiquidatorBalance = await blastProvider.getBalance(contractAddresses.OrbitLiquidator);
  const maxBorrowRepay = min(
- (ethBorrowAsset.borrowBalance * (closeFactor as bigint)) / 10n ** 18n,
- OrbitLiquidatorBalance,
- getPercentageValue(maxTokenBalanceAsset.tokenBalance, MAX_COLLATERAL_REPAY_PERCENTAGE) // NOTE: We leave some buffer to be sure there is enough collateral after the interest accrual.
+ (((ethBorrowAsset.borrowBalance * 10n ** 18n) / transmutationValue) * closeFactor) / 10n ** 18n,
+ orbitLiquidatorBalance,
+ getPercentageValue(
+ (maxTokenBalanceAsset.tokenBalance * 10n ** 18n) / transmutationValue,
+ MAX_COLLATERAL_REPAY_PERCENTAGE
+ )
  );
  console.debug('Potential liquidation', {
  borrower,
@@ -517,30 +574,27 @@ const findOevLiquidation = async () => {
  ];
  const liquidateResult = await simulateTransmutationMulticall(externalMulticallSimulator, liquidateBorrowCalls);
 
- const liquidateReturndata = liquidateResult.data.at(-1);
- const [profitEth, profitUsd] = OrbitLiquidator.interface.decodeFunctionResult('liquidate', liquidateReturndata);
+ const liquidateReturndata = liquidateResult.at(-1);
+ const [profitUsd] = OrbitLiquidator.interface.decodeFunctionResult('liquidate', liquidateReturndata);
  if (profitUsd <= MIN_LIQUIDATION_PROFIT_USD) {
  console.info('Liquidation possible, but profit is too low', {
  borrower,
- eth: formatEther(profitEth),
  usd: formatEther(profitUsd),
  });
  continue;
  }
  console.info('Possible liquidation profit', {
  borrower,
  maxBorrowRepay: formatEther(maxBorrowRepay),
- eth: formatEther(profitEth),
  usd: formatEther(profitUsd),
  });
 
- if (!bestLiquidation || profitEth > bestLiquidation.profitEth) {
+ if (!bestLiquidation || profitUsd > bestLiquidation.profitUsd) {
  bestLiquidation = {
  borrowTokenAddress: ethBorrowAsset.oToken,
  borrower,
  collateralTokenAddress: maxTokenBalanceAsset.oToken,
  maxBorrowRepay,
- profitEth,
  profitUsd,
  };
  }
@@ -552,7 +606,9 @@ const findOevLiquidation = async () => {
  }
 
  // Place a bid on the OEV network.
- const bidAmount = getPercentageValue(bestLiquidation.profitEth, 20); // NOTE: This assumes the wallet is going to have enough deposit to cover the bid.
+ const bidAmount =
+ parseEther('0.0000000000001') ??
+ getPercentageValue((bestLiquidation.profitUsd * 10n ** 18n) / transmutationValue, SIMULATION_PERCENTAGE); // NOTE: This assumes the wallet is going to have enough deposit to cover the bid.
  const nonce = ethers.hexlify(ethers.randomBytes(32));
  const bidDetails: BidDetails = {
  oevProxyAddress: contractAddresses.api3OevEthUsdProxy,
@@ -568,7 +624,6 @@ const findOevLiquidation = async () => {
  console.info('Placing bid', {
  ...bestLiquidation,
  maxBorrowRepay: formatEther(bestLiquidation.maxBorrowRepay),
- profitEth: formatEther(bestLiquidation.profitEth),
  profitUsd: formatEther(bestLiquidation.profitUsd),
  bidAmount: formatEther(bidAmount),
  });

src/types.ts

@@ -27,7 +27,6 @@ export interface LiquidationParameters {
  borrower: string;
  collateralTokenAddress: string;
  maxBorrowRepay: bigint;
- profitEth: bigint;
  profitUsd: bigint;
 }