diff --git a/requirements.txt b/requirements.txt index 02d760e..bf63c49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,5 @@ pandas-stubs types-psycopg2 types-requests moralis -dune-client \ No newline at end of file +dune-client +pytest \ No newline at end of file diff --git a/src/compute_fees_single_hash.py b/src/compute_fees_single_hash.py new file mode 100644 index 0000000..b9d2d68 --- /dev/null +++ b/src/compute_fees_single_hash.py @@ -0,0 +1,23 @@ +from hexbytes import HexBytes +from src.fees.compute_fees import compute_all_fees_of_batch +from src.helpers.config import logger + + +def log_token_data(title: str, data: dict, name: str): + logger.info(title) + for token, value in data.items(): + logger.info(f"Token Address: {token}, {name}: {value}") + + +def main(): + protocol_fees, partner_fees, network_fees = compute_all_fees_of_batch( + HexBytes(input("tx hash: ")) + ) + log_token_data("Protocol Fees:", protocol_fees, "Protocol Fee") + log_token_data("Partner Fees:", partner_fees, "Partner Fee") + log_token_data("Network Fees:", network_fees, "Network Fee") + # e.g. input: 0x980fa3f8ff95c504ba61e054e5c3e50ea36b892f865703b8a665564ac0beb1f4 + + +if __name__ == "__main__": + main() diff --git a/src/constants.py b/src/constants.py index 16e9a08..530fddf 100644 --- a/src/constants.py +++ b/src/constants.py @@ -18,6 +18,8 @@ "0x875b6cb035bbd4ac6500fabc6d1e4ca5bdc58a3e2b424ccb5c24cdbebeb009a9" ) +NULL_ADDRESS = Web3.to_checksum_address("0x0000000000000000000000000000000000000000") + REQUEST_TIMEOUT = 5 # Time limit, currently set to 1 full day, after which Coingecko Token List is re-fetched (in seconds) diff --git a/src/fees/compute_fees.py b/src/fees/compute_fees.py index 52c7eba..d4a7e8e 100644 --- a/src/fees/compute_fees.py +++ b/src/fees/compute_fees.py @@ -5,33 +5,59 @@ import math import os from typing import Any +import requests +import json from dotenv import load_dotenv -from eth_typing import Address +from eth_typing import ChecksumAddress from hexbytes import HexBytes +from web3 import Web3 -from src.constants import ( - REQUEST_TIMEOUT, -) -import requests +from src.constants import REQUEST_TIMEOUT, NULL_ADDRESS # types for trades @dataclass class Trade: - """Class for""" + """Class for describing a trade, together with the fees associated with it. + We note that we use the NULL address to indicate that there are no partner fees. + Note that in case an order is placed with the partner fee recipient being the null address, + the partner fee will instead be accounted for as protocol fee and will be withheld by the DAO. + """ - order_uid: HexBytes - sell_amount: int - buy_amount: int - sell_token: HexBytes - buy_token: HexBytes - limit_sell_amount: int - limit_buy_amount: int - kind: str - sell_token_clearing_price: int - buy_token_clearing_price: int - fee_policies: list["FeePolicy"] + def __init__( + self, + order_uid: HexBytes, + sell_amount: int, + buy_amount: int, + sell_token: HexBytes, + buy_token: HexBytes, + limit_sell_amount: int, + limit_buy_amount: int, + kind: str, + sell_token_clearing_price: int, + buy_token_clearing_price: int, + fee_policies: list["FeePolicy"], + partner_fee_recipient: ChecksumAddress, + ): + self.order_uid = order_uid + self.sell_amount = sell_amount + self.buy_amount = buy_amount + self.sell_token = sell_token + self.buy_token = buy_token + self.limit_sell_amount = limit_sell_amount + self.limit_buy_amount = limit_buy_amount + self.kind = kind + self.sell_token_clearing_price = sell_token_clearing_price + self.buy_token_clearing_price = buy_token_clearing_price + self.fee_policies = fee_policies + self.partner_fee_recipient = partner_fee_recipient # if there is no partner, then its value is set to the null address + + total_protocol_fee, partner_fee, network_fee = self.compute_all_fees() + self.total_protocol_fee = total_protocol_fee + self.partner_fee = partner_fee + self.network_fee = network_fee + return def volume(self) -> int: """Compute volume of a trade in the surplus token""" @@ -62,20 +88,28 @@ def surplus(self) -> int: return current_limit_sell_amount - self.sell_amount raise ValueError(f"Order kind {self.kind} is invalid.") - def raw_surplus(self) -> int: - """Compute raw surplus of a trade in the surplus token - First, the application of protocol fees is reversed. Then, surplus of the resulting trade - is computed.""" + def compute_all_fees(self) -> tuple[int, int, int]: raw_trade = deepcopy(self) - for fee_policy in reversed(self.fee_policies): + partner_fee = 0 + for i, fee_policy in enumerate(reversed(self.fee_policies)): raw_trade = fee_policy.reverse_protocol_fee(raw_trade) - return raw_trade.surplus() + ## we assume that partner fee is the last to be applied + if i == 0 and self.partner_fee_recipient != NULL_ADDRESS: + partner_fee = raw_trade.surplus() - self.surplus() + total_protocol_fee = raw_trade.surplus() - self.surplus() - def protocol_fee(self): - """Compute protocol fees of a trade in the surplus token - Protocol fees are computed as the difference of raw surplus and surplus.""" - - return self.raw_surplus() - self.surplus() + surplus_fee = self.compute_surplus_fee() # in the surplus token + network_fee_in_surplus_token = surplus_fee - total_protocol_fee + if self.kind == "sell": + network_fee = int( + network_fee_in_surplus_token + * Fraction( + self.buy_token_clearing_price, self.sell_token_clearing_price + ) + ) + else: + network_fee = network_fee_in_surplus_token + return total_protocol_fee, partner_fee, network_fee def surplus_token(self) -> HexBytes: """Returns the surplus token""" @@ -336,6 +370,14 @@ def get_all_data(self, tx_hash: HexBytes) -> SettlementData: buy_token_clearing_price = clearing_prices[buy_token] fee_policies = self.parse_fee_policies(trade_data["feePolicies"]) + app_data = json.loads(order_data["fullAppData"]) + if "partnerFee" in app_data["metadata"].keys(): + partner_fee_recipient = Web3.to_checksum_address( + HexBytes(app_data["metadata"]["partnerFee"]["recipient"]) + ) + else: + partner_fee_recipient = NULL_ADDRESS + trade = Trade( order_uid=uid, sell_amount=executed_sell_amount, @@ -348,6 +390,7 @@ def get_all_data(self, tx_hash: HexBytes) -> SettlementData: sell_token_clearing_price=sell_token_clearing_price, buy_token_clearing_price=buy_token_clearing_price, fee_policies=fee_policies, + partner_fee_recipient=partner_fee_recipient, ) trades.append(trade) @@ -436,48 +479,35 @@ def parse_fee_policies( return fee_policies -# computing fees -def compute_fee_imbalances( - settlement_data: SettlementData, -) -> tuple[dict[str, tuple[str, int]], dict[str, tuple[str, int]]]: +# function that computes all fees of all orders in a batch +# Note that currently it is NOT working for CoW AMMs as they are not indexed. +def compute_all_fees_of_batch( + tx_hash: HexBytes, +) -> tuple[ + dict[str, tuple[str, int]], + dict[str, tuple[str, int, str]], + dict[str, tuple[str, int]], +]: + orderbook_api = OrderbookFetcher() + settlement_data = orderbook_api.get_all_data(tx_hash) protocol_fees: dict[str, tuple[str, int]] = {} network_fees: dict[str, tuple[str, int]] = {} + partner_fees: dict[str, tuple[str, int, str]] = {} for trade in settlement_data.trades: # protocol fees - protocol_fee_amount = trade.protocol_fee() + protocol_fee_amount = trade.total_protocol_fee - trade.partner_fee protocol_fee_token = trade.surplus_token() protocol_fees[trade.order_uid.to_0x_hex()] = ( protocol_fee_token.to_0x_hex(), protocol_fee_amount, ) - # network fees - surplus_fee = trade.compute_surplus_fee() # in the surplus token - network_fee = surplus_fee - protocol_fee_amount - if trade.kind == "sell": - network_fee_sell = int( - network_fee - * Fraction( - trade.buy_token_clearing_price, trade.sell_token_clearing_price - ) - ) - else: - network_fee_sell = network_fee - + partner_fees[trade.order_uid.to_0x_hex()] = ( + protocol_fee_token.to_0x_hex(), + trade.partner_fee, + trade.partner_fee_recipient, + ) network_fees[trade.order_uid.to_0x_hex()] = ( trade.sell_token.to_0x_hex(), - network_fee_sell, + trade.network_fee, ) - - return protocol_fees, network_fees - - -# combined function - - -def batch_fee_imbalances( - tx_hash: HexBytes, -) -> tuple[dict[str, tuple[str, int]], dict[str, tuple[str, int]]]: - orderbook_api = OrderbookFetcher() - settlement_data = orderbook_api.get_all_data(tx_hash) - protocol_fees, network_fees = compute_fee_imbalances(settlement_data) - return protocol_fees, network_fees + return protocol_fees, partner_fees, network_fees diff --git a/src/helpers/database.py b/src/helpers/database.py index e0abdc8..cb6e896 100644 --- a/src/helpers/database.py +++ b/src/helpers/database.py @@ -89,7 +89,6 @@ def write_prices( def write_fees( self, - chain_name: str, auction_id: int, block_number: int, tx_hash: str, @@ -97,12 +96,18 @@ def write_fees( token_address: str, fee_amount: float, fee_type: str, + recipient: str, ): """Function attempts to write price data to the table.""" tx_hash_bytes = bytes.fromhex(tx_hash[2:]) token_address_bytes = bytes.fromhex(token_address[2:]) order_uid_bytes = bytes.fromhex(order_uid[2:]) + query = read_sql_file("src/sql/insert_fee.sql") + final_recipient = None + if recipient != "": + final_recipient = bytes.fromhex(recipient[2:]) + self.execute_and_commit( query, { @@ -114,5 +119,6 @@ def write_fees( "token_address": token_address_bytes, "fee_amount": fee_amount, "fee_type": fee_type, + "recipient": final_recipient, }, ) diff --git a/src/sql/insert_fee.sql b/src/sql/insert_fee.sql index 67ec10a..7b080b1 100644 --- a/src/sql/insert_fee.sql +++ b/src/sql/insert_fee.sql @@ -1,4 +1,4 @@ -INSERT INTO fees_new ( - chain_name, auction_id, block_number, tx_hash, order_uid, token_address, fee_amount,fee_type -) VALUES ( :chain_name, :auction_id, :block_number, :tx_hash, :order_uid, :token_address, :fee_amount, :fee_type +INSERT INTO fees_per_trade ( + chain_name, auction_id, block_number, tx_hash, order_uid, token_address, fee_amount, fee_type, fee_recipient +) VALUES ( :chain_name, :auction_id, :block_number, :tx_hash, :order_uid, :token_address, :fee_amount, :fee_type, :fee_recipient ); diff --git a/src/test_single_hash.py b/src/test_single_hash.py index 351f97a..3996915 100644 --- a/src/test_single_hash.py +++ b/src/test_single_hash.py @@ -2,7 +2,7 @@ from web3 import Web3 from src.imbalances_script import RawTokenImbalances from src.price_providers.price_feed import PriceFeed -from src.fees.compute_fees import batch_fee_imbalances +from src.fees.compute_fees import compute_all_fees_of_batch from src.transaction_processor import calculate_slippage from src.helpers.config import get_web3_instance, logger from contracts.erc20_abi import erc20_abi @@ -26,7 +26,9 @@ def __init__(self): def compute_data(self, tx_hash: str): token_imbalances = self.imbalances.compute_imbalances(tx_hash) - protocol_fees, network_fees = batch_fee_imbalances(HexBytes(tx_hash)) + protocol_fees, partner_fees, network_fees = compute_all_fees_of_batch( + HexBytes(tx_hash) + ) slippage = calculate_slippage(token_imbalances, protocol_fees, network_fees) eth_slippage = self.calculate_slippage_in_eth(slippage, tx_hash) diff --git a/src/transaction_processor.py b/src/transaction_processor.py index 53dcad2..ff429aa 100644 --- a/src/transaction_processor.py +++ b/src/transaction_processor.py @@ -6,7 +6,7 @@ from src.price_providers.price_feed import PriceFeed from src.helpers.helper_functions import read_sql_file, set_params from src.helpers.config import CHAIN_SLEEP_TIME, logger -from src.fees.compute_fees import batch_fee_imbalances +from src.fees.compute_fees import compute_all_fees_of_batch import time @@ -109,9 +109,11 @@ def process_single_transaction( # Compute Fees if self.process_fees: - protocol_fees, network_fees = self.process_fees_for_transaction( - tx_hash, auction_id, block_number - ) + ( + protocol_fees, + partner_fees, + network_fees, + ) = self.process_fees_for_transaction(tx_hash) # Compute Prices if self.process_prices: @@ -134,7 +136,12 @@ def process_single_transaction( if self.process_fees: self.handle_fees( - protocol_fees, network_fees, auction_id, block_number, tx_hash + protocol_fees, + partner_fees, + network_fees, + auction_id, + block_number, + tx_hash, ) if self.process_prices and prices: @@ -162,15 +169,22 @@ def process_token_imbalances( return {} def process_fees_for_transaction( - self, tx_hash: str, auction_id: int, block_number: int - ) -> tuple[dict[str, tuple[str, int]], dict[str, tuple[str, int]]]: + self, + tx_hash: str, + ) -> tuple[ + dict[str, tuple[str, int]], + dict[str, tuple[str, int, str]], + dict[str, tuple[str, int]], + ]: """Process and return protocol and network fees for a given transaction.""" try: - protocol_fees, network_fees = batch_fee_imbalances(HexBytes(tx_hash)) - return protocol_fees, network_fees + protocol_fees, partner_fees, network_fees = compute_all_fees_of_batch( + HexBytes(tx_hash) + ) + return protocol_fees, partner_fees, network_fees except Exception as e: logger.error(f"Failed to process fees for transaction {tx_hash}: {e}") - return {}, {} + return {}, {}, {} def process_prices_for_tokens( self, @@ -224,6 +238,7 @@ def handle_imbalances( def handle_fees( self, protocol_fees: dict[str, tuple[str, int]], + partner_fees: dict[str, tuple[str, int, str]], network_fees: dict[str, tuple[str, int]], auction_id: int, block_number: int, @@ -234,7 +249,6 @@ def handle_fees( # Write protocol fees for order_uid, (token_address, fee_amount) in protocol_fees.items(): self.db.write_fees( - chain_name=self.chain_name, auction_id=auction_id, block_number=block_number, tx_hash=tx_hash, @@ -242,12 +256,29 @@ def handle_fees( token_address=token_address, fee_amount=float(fee_amount), fee_type="protocol", + recipient="", + ) + + # Write partner fees + for order_uid, ( + token_address, + fee_amount, + recipient, + ) in partner_fees.items(): + self.db.write_fees( + auction_id=auction_id, + block_number=block_number, + tx_hash=tx_hash, + order_uid=order_uid, + token_address=token_address, + fee_amount=float(fee_amount), + fee_type="partner", + recipient=recipient, ) # Write network fees for order_uid, (token_address, fee_amount) in network_fees.items(): self.db.write_fees( - chain_name=self.chain_name, auction_id=auction_id, block_number=block_number, tx_hash=tx_hash, @@ -255,6 +286,7 @@ def handle_fees( token_address=token_address, fee_amount=float(fee_amount), fee_type="network", + recipient="", ) except Exception as err: logger.error( diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/e2e/test_fees.py b/tests/e2e/test_fees.py new file mode 100644 index 0000000..23e08b0 --- /dev/null +++ b/tests/e2e/test_fees.py @@ -0,0 +1,116 @@ +"""End-to-end tests for fee computations""" + +from hexbytes import HexBytes +from src.fees.compute_fees import compute_all_fees_of_batch + +ratio_tolerance = 0.001 +absolute_tolerance = 100 + + +def absolute_tolerance_check(value_1: int, value_2: int, epsilon: float) -> bool: + if abs(value_1 - value_2) < epsilon: + return True + else: + return False + + +def ratio_tolerance_check(ratio: float, epsilon: float) -> bool: + if (ratio > 1 - epsilon) and (ratio < 1 + epsilon): + return True + else: + return False + + +def test_sample_trades(): + tx_hash_list = [ + "0x70d242d7991fbd3f91033e693436df9514cdf615d4a2230c5ed0557af37c073a", + "0xce17b91e8f50a674c317a39cbfb4ca7e417af075a53b1aa0eece5aa957ed0bbe", + "0x87ab7f4ee01388e85a6f1a1ebd6aff885b6a42fac0b9acd5cda9dd66bebfc0b9", + "0x87ab7f4ee01388e85a6f1a1ebd6aff885b6a42fac0b9acd5cda9dd66bebfc0b9", + "0xf9400f66210e3eab46fb66232196cde0e1bbe8cfc694489a13b766eae4a21c66", + ] + orders_list = [ + "0xc4be63dd6e3baf39f4b2ba1709f78c4971ae7d526a40dcea4eea94c5b0133d0831ae23b20c9f5d0e5fb1f864301d13793b63e1dc66d24952", + "0x05babeb0e90f2f3a6ba999f397fbcb5e983eff1c1bade7fe0bb2cb9196919b615c9e070ec97e9cd64bd74b53049ca700ff68111466ce4929", + "0x53e4f7041b532c0952fe3821b4e18a6f6b26fa403fb398efaeca129f2d8e22ce4b41cc5a22e0e2568b1e80756e5784a5b120805066be643a", + "0xb415d0d7e0aeb27df3777de95933fa9b6cf3430e3a332cd73ef87d8e30787cc57bc5ddf54c57fe74bd5b9a14cae952e730bd847666be64f0", + "0xa9fbfbe1f61606162b29c6a5df2eb4c0913929248e5ea9899797851f421f075040a50cf069e992aa4536211b23f286ef88752187ffffffff", + ] + protocol_fees_list = [ + 265066228346000000, + 863347103526919000000000, + 0, + 2439671, + 0, + ] + partner_fees_list = [ + 699996861102526000000, + 0, + 101968968211339000000, + 0, + 0, + ] + network_fees_list = [ + 631761621422024452, + 301172642049802509484032, + 5701811, + 371687632145107059212288, + 247664759315072, + ] + for i, tx_hash in enumerate(tx_hash_list): + protocol_fees, partner_fees, network_fees = compute_all_fees_of_batch( + HexBytes(tx_hash) + ) + uid = orders_list[i] + if protocol_fees_list[i] * protocol_fees[uid][1] == 0: + assert absolute_tolerance_check( + protocol_fees_list[i], protocol_fees[uid][1], absolute_tolerance + ) + else: + protocol_fees_ratio = protocol_fees_list[i] / protocol_fees[uid][1] + assert ratio_tolerance_check(protocol_fees_ratio, ratio_tolerance) + + if partner_fees_list[i] * partner_fees[uid][1] == 0: + assert absolute_tolerance_check( + partner_fees_list[i], partner_fees[uid][1], absolute_tolerance + ) + else: + partner_fees_ratio = partner_fees_list[i] / partner_fees[uid][1] + assert ratio_tolerance_check(partner_fees_ratio, ratio_tolerance) + + if network_fees_list[i] * network_fees[uid][1] == 0: + assert absolute_tolerance_check( + network_fees_list[i], network_fees[uid][1], absolute_tolerance + ) + else: + network_fees_ratio = network_fees_list[i] / network_fees[uid][1] + assert ratio_tolerance_check(network_fees_ratio, ratio_tolerance) + + +# hashes/orders to check + +# Example 1 +# hash = 0x70d242d7991fbd3f91033e693436df9514cdf615d4a2230c5ed0557af37c073a +# order_uid = 0xc4be63dd6e3baf39f4b2ba1709f78c4971ae7d526a40dcea4eea94c5b0133d0831ae23b20c9f5d0e5fb1f864301d13793b63e1dc66d24952 +# Here we have both partner and protocol fees + +# Example 2 +# hash = 0xce17b91e8f50a674c317a39cbfb4ca7e417af075a53b1aa0eece5aa957ed0bbe +# order_uid = 0x05babeb0e90f2f3a6ba999f397fbcb5e983eff1c1bade7fe0bb2cb9196919b615c9e070ec97e9cd64bd74b53049ca700ff68111466ce4929 +# Here we have partner fee but the partner fee recipient is the null address, in which case we redirect it to the DAO as protocol fee + + +# Example 3 +# hash = 0x87ab7f4ee01388e85a6f1a1ebd6aff885b6a42fac0b9acd5cda9dd66bebfc0b9 +# order_uid = 0x53e4f7041b532c0952fe3821b4e18a6f6b26fa403fb398efaeca129f2d8e22ce4b41cc5a22e0e2568b1e80756e5784a5b120805066be643a +# Here we only have partner fee + +# Example 4 +# hash = 0x87ab7f4ee01388e85a6f1a1ebd6aff885b6a42fac0b9acd5cda9dd66bebfc0b9 +# order_uid = 0xb415d0d7e0aeb27df3777de95933fa9b6cf3430e3a332cd73ef87d8e30787cc57bc5ddf54c57fe74bd5b9a14cae952e730bd847666be64f0 +# Here we only have protocol fee + +# Example 5 +# hash = 0xf9400f66210e3eab46fb66232196cde0e1bbe8cfc694489a13b766eae4a21c66 +# order_uid = 0xa9fbfbe1f61606162b29c6a5df2eb4c0913929248e5ea9899797851f421f075040a50cf069e992aa4536211b23f286ef88752187ffffffff +# Here we don't have protocol or partner fee