From eb2389714a167894f2c5573c04709cbe0958875d Mon Sep 17 00:00:00 2001 From: Ali Behjati Date: Thu, 5 Sep 2024 14:37:54 +0200 Subject: [PATCH 01/10] feat: add publisher program to sync cli --- program_admin/__init__.py | 84 ++++++++++ program_admin/cli.py | 14 ++ .../publisher_program_instructions.py | 156 ++++++++++++++++++ tests/test_sync.py | 1 + 4 files changed, 255 insertions(+) create mode 100644 program_admin/publisher_program_instructions.py diff --git a/program_admin/__init__.py b/program_admin/__init__.py index 4561252..72ee458 100644 --- a/program_admin/__init__.py +++ b/program_admin/__init__.py @@ -16,6 +16,15 @@ from program_admin import instructions as pyth_program from program_admin.keys import load_keypair from program_admin.parsing import parse_account +from program_admin.publisher_program_instructions import ( + config_account_pubkey as publisher_program_config_account_pubkey, +) +from program_admin.publisher_program_instructions import ( + create_buffer_account, + initialize_publisher_config, + initialize_publisher_program, + publisher_config_account_pubkey, +) from program_admin.types import ( Network, PythAuthorityPermissionAccount, @@ -56,6 +65,7 @@ class ProgramAdmin: rpc_endpoint: str key_dir: Path program_key: PublicKey + publisher_program_key: Optional[PublicKey] authority_permission_account: Optional[PythAuthorityPermissionAccount] _mapping_accounts: Dict[PublicKey, PythMappingAccount] _product_accounts: Dict[PublicKey, PythProductAccount] @@ -66,6 +76,7 @@ def __init__( network: Network, key_dir: str, program_key: str, + publisher_program_key: Optional[str], commitment: Literal["confirmed", "finalized"], rpc_endpoint: str = "", ): @@ -73,6 +84,9 @@ def __init__( self.rpc_endpoint = rpc_endpoint or RPC_ENDPOINTS[network] self.key_dir = Path(key_dir) self.program_key = PublicKey(program_key) + self.publisher_program_key = ( + PublicKey(publisher_program_key) if publisher_program_key else None + ) self.commitment = Commitment(commitment) self.authority_permission_account = None self._mapping_accounts: Dict[PublicKey, PythMappingAccount] = {} @@ -100,6 +114,12 @@ async def fetch_minimum_balance(self, size: int) -> int: async with AsyncClient(self.rpc_endpoint) as client: return (await client.get_minimum_balance_for_rent_exemption(size)).value + async def account_exists(self, key: PublicKey) -> bool: + async with AsyncClient(self.rpc_endpoint) as client: + response = await client.get_account_info(key) + # The RPC returns null if the account does not exist + return bool(response.value) + async def refresh_program_accounts(self): async with AsyncClient(self.rpc_endpoint) as client: logger.info("Refreshing program accounts") @@ -301,6 +321,19 @@ async def sync( if product_updates: await self.refresh_program_accounts() + # Sync publisher program + ( + publisher_program_instructions, + publisher_program_signers, + ) = await self.sync_publisher_program(ref_publishers) + + if publisher_program_instructions: + instructions.extend(publisher_program_instructions) + if send_transactions: + await self.send_transaction( + publisher_program_instructions, publisher_program_signers + ) + # Sync publishers publisher_transactions = [] @@ -658,3 +691,54 @@ async def resize_price_accounts_v2( if send_transactions: await self.send_transaction(instructions, signers) + + async def sync_publisher_program( + self, ref_publishers: ReferencePublishers + ) -> Tuple[List[TransactionInstruction], List[Keypair]]: + if self.publisher_program_key is None: + return [], [] + + instructions = [] + + authority = load_keypair("funding", key_dir=self.key_dir) + + publisher_program_config = publisher_program_config_account_pubkey( + self.publisher_program_key + ) + + # Initialize the publisher program config if it does not exist + if not (await self.account_exists(publisher_program_config)): + initialize_publisher_program_instruction = initialize_publisher_program( + self.publisher_program_key, authority.public_key + ) + instructions.append(initialize_publisher_program_instruction) + + # Initialize publisher config accounts for new publishers + for publisher in ref_publishers["keys"].values(): + publisher_config_account = publisher_config_account_pubkey( + publisher, self.publisher_program_key + ) + + if not (await self.account_exists(publisher_config_account)): + size = 100048 # This size is for a buffer supporting 5000 price updates + lamports = await self.fetch_minimum_balance(size) + buffer_account, create_buffer_instruction = create_buffer_account( + self.publisher_program_key, + authority.public_key, + publisher, + size, + lamports, + ) + + initialize_publisher_config_instruction = initialize_publisher_config( + self.publisher_program_key, + publisher, + authority.public_key, + buffer_account, + ) + + instructions.extend( + [create_buffer_instruction, initialize_publisher_config_instruction] + ) + + return (instructions, [authority]) diff --git a/program_admin/cli.py b/program_admin/cli.py index e17703d..344a56d 100644 --- a/program_admin/cli.py +++ b/program_admin/cli.py @@ -56,6 +56,7 @@ def delete_price(network, rpc_endpoint, program_key, keys, commitment, product, rpc_endpoint=rpc_endpoint, key_dir=keys, program_key=program_key, + publisher_program_key=None, commitment=commitment, ) funding_keypair = load_keypair("funding", key_dir=keys) @@ -236,6 +237,7 @@ def delete_product( rpc_endpoint=rpc_endpoint, key_dir=keys, program_key=program_key, + publisher_program_key=None, commitment=commitment, ) funding_keypair = load_keypair("funding", key_dir=keys) @@ -275,6 +277,7 @@ def list_accounts(network, rpc_endpoint, program_key, keys, publishers, commitme rpc_endpoint=rpc_endpoint, key_dir=keys, program_key=program_key, + publisher_program_key=None, commitment=commitment, ) @@ -333,6 +336,7 @@ def restore_links(network, rpc_endpoint, program_key, keys, products, commitment rpc_endpoint=rpc_endpoint, key_dir=keys, program_key=program_key, + publisher_program_key=None, commitment=commitment, ) reference_products = parse_products_json(Path(products)) @@ -382,6 +386,12 @@ def restore_links(network, rpc_endpoint, program_key, keys, products, commitment @click.option("--network", help="Solana network", envvar="NETWORK") @click.option("--rpc-endpoint", help="Solana RPC endpoint", envvar="RPC_ENDPOINT") @click.option("--program-key", help="Pyth program key", envvar="PROGRAM_KEY") +@click.option( + "--publisher-program-key", + help="Publisher program key", + envvar="PUBLISHER_PROGRAM_KEY", + default=None, +) @click.option("--keys", help="Path to keys directory", envvar="KEYS") @click.option("--products", help="Path to reference products file", envvar="PRODUCTS") @click.option( @@ -426,6 +436,7 @@ def sync( network, rpc_endpoint, program_key, + publisher_program_key, keys, products, publishers, @@ -442,6 +453,7 @@ def sync( rpc_endpoint=rpc_endpoint, key_dir=keys, program_key=program_key, + publisher_program_key=publisher_program_key, commitment=commitment, ) @@ -495,6 +507,7 @@ def migrate_upgrade_authority( rpc_endpoint=rpc_endpoint, key_dir=keys, program_key=program_key, + publisher_program_key=None, commitment=commitment, ) funding_keypair = load_keypair("funding", key_dir=keys) @@ -544,6 +557,7 @@ def resize_price_accounts_v2( rpc_endpoint=rpc_endpoint, key_dir=keys, program_key=program_key, + publisher_program_key=None, commitment=commitment, ) diff --git a/program_admin/publisher_program_instructions.py b/program_admin/publisher_program_instructions.py new file mode 100644 index 0000000..e13a393 --- /dev/null +++ b/program_admin/publisher_program_instructions.py @@ -0,0 +1,156 @@ +from typing import Tuple + +from construct import Bytes, Int8ul, Struct +from solana import system_program +from solana.publickey import PublicKey +from solana.system_program import SYS_PROGRAM_ID, CreateAccountWithSeedParams +from solana.transaction import AccountMeta, TransactionInstruction + + +def config_account_pubkey(program_key: PublicKey) -> PublicKey: + [config_account, _] = PublicKey.find_program_address( + [b"CONFIG"], + program_key, + ) + return config_account + + +def publisher_config_account_pubkey( + publisher_key: PublicKey, program_key: PublicKey +) -> PublicKey: + [publisher_config_account, _] = PublicKey.find_program_address( + [b"PUBLISHER_CONFIG", bytes(publisher_key)], + program_key, + ) + return publisher_config_account + + +def initialize_publisher_program( + program_key: PublicKey, + authority: PublicKey, +) -> TransactionInstruction: + """ + Pyth publisher program initialize instruction with the given authority + + accounts: + - payer account (signer, writable) - we pass the authority as the payer + - config account (writable) + - system program + """ + + [config_account, bump] = PublicKey.find_program_address( + [b"CONFIG"], + program_key, + ) + + ix_data_layout = Struct( + "bump" / Int8ul, + "authority" / Bytes(32), + ) + + ix_data = ix_data_layout.build( + dict( + bump=bump, + authority=bytes(authority), + ) + ) + + return TransactionInstruction( + data=ix_data, + keys=[ + AccountMeta(pubkey=authority, is_signer=True, is_writable=True), + AccountMeta(pubkey=config_account, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + ], + program_id=program_key, + ) + + +def create_buffer_account( + program_key: PublicKey, + base_pubkey: PublicKey, + publisher_pubkey: PublicKey, + space: int, + lamports: int, +) -> Tuple[PublicKey, TransactionInstruction]: + + seed = str(publisher_pubkey) + new_account_pubkey = PublicKey.create_with_seed( + base_pubkey, + seed, + program_key, + ) + + # space = 100048 # Required space to store 5000 price updates + # lamport = + + return ( + new_account_pubkey, + system_program.create_account_with_seed( + CreateAccountWithSeedParams( + from_pubkey=base_pubkey, + new_account_pubkey=new_account_pubkey, + base_pubkey=base_pubkey, + seed=seed, + program_id=program_key, + lamports=lamports, + space=space, + ) + ), + ) + + +def initialize_publisher_config( + program_key: PublicKey, + publisher_key: PublicKey, + authority: PublicKey, + buffer_account: PublicKey, +) -> TransactionInstruction: + """ + Pyth publisher program initialize publisher config instruction with the given authority + + accounts: + - authority account (signer, writable) + - config account + - publisher config account (writable) + - buffer account (writable) + - system program + """ + + [config_account, config_bump] = PublicKey.find_program_address( + [b"CONFIG"], + program_key, + ) + + [publisher_config_account, publisher_config_bump] = PublicKey.find_program_address( + [b"PUBLISHER_CONFIG", bytes(publisher_key)], + program_key, + ) + + ix_data_layout = Struct( + "config_bump" / Int8ul, + "publisher_config_bump" / Int8ul, + "publisher" / Bytes(32), + ) + + ix_data = ix_data_layout.build( + dict( + config_bump=config_bump, + publisher_config_bump=publisher_config_bump, + publisher=bytes(publisher_key), + ) + ) + + return TransactionInstruction( + data=ix_data, + keys=[ + AccountMeta(pubkey=authority, is_signer=True, is_writable=True), + AccountMeta(pubkey=config_account, is_signer=False, is_writable=True), + AccountMeta( + pubkey=publisher_config_account, is_signer=False, is_writable=True + ), + AccountMeta(pubkey=buffer_account, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + ], + program_id=program_key, + ) diff --git a/tests/test_sync.py b/tests/test_sync.py index 4a0370c..95641f1 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -421,6 +421,7 @@ async def test_sync( network=network, key_dir=key_dir, program_key=pyth_program, + publisher_program_key=None, commitment="confirmed", ) From 61a233d3ea141aaa80c5b39c439444c2a56c1d3b Mon Sep 17 00:00:00 2001 From: Ali Behjati Date: Thu, 5 Sep 2024 21:11:30 +0200 Subject: [PATCH 02/10] fix: use seed properly and update docker image --- Dockerfile | 4 ++-- program_admin/publisher_program_instructions.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2e7ee34..4da2711 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN curl -sSL https://install.python-poetry.org | python ENV PATH="$POETRY_HOME/bin:$PATH" # Install Solana CLI -RUN sh -c "$(curl -sSfL https://release.solana.com/stable/install)" +RUN sh -c "$(curl -sSfL https://release.solana.com/v1.14.17/install)" ENV PATH=$PATH:/root/.local/share/solana/install/active_release/bin @@ -80,7 +80,7 @@ ARG APP_PATH # Install Solana CLI, we redo this step because this Docker target # starts from scratch without the earlier Solana installation -RUN sh -c "$(curl -sSfL https://release.solana.com/stable/install)" +RUN sh -c "$(curl -sSfL https://release.solana.com/v1.14.17/install" ENV PATH=$PATH:/root/.local/share/solana/install/active_release/bin ENV \ diff --git a/program_admin/publisher_program_instructions.py b/program_admin/publisher_program_instructions.py index e13a393..1e8b14d 100644 --- a/program_admin/publisher_program_instructions.py +++ b/program_admin/publisher_program_instructions.py @@ -73,8 +73,10 @@ def create_buffer_account( space: int, lamports: int, ) -> Tuple[PublicKey, TransactionInstruction]: - - seed = str(publisher_pubkey) + # The seed should be str but later is used in rust as + # &str and therefore we use latin1 encoding to map byte + # i to char i + seed = bytes(publisher_pubkey).decode("latin-1") new_account_pubkey = PublicKey.create_with_seed( base_pubkey, seed, From c4c6e46a16503bf66f579f277552ae53ef5096eb Mon Sep 17 00:00:00 2001 From: Ali Behjati Date: Thu, 5 Sep 2024 21:12:53 +0200 Subject: [PATCH 03/10] refactor: remove extra comment --- program_admin/publisher_program_instructions.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/program_admin/publisher_program_instructions.py b/program_admin/publisher_program_instructions.py index 1e8b14d..49f32a2 100644 --- a/program_admin/publisher_program_instructions.py +++ b/program_admin/publisher_program_instructions.py @@ -83,9 +83,6 @@ def create_buffer_account( program_key, ) - # space = 100048 # Required space to store 5000 price updates - # lamport = - return ( new_account_pubkey, system_program.create_account_with_seed( From 2866a35a246517b7eccd720e7506e0e03c76ffa6 Mon Sep 17 00:00:00 2001 From: Ali Behjati Date: Thu, 5 Sep 2024 21:17:59 +0200 Subject: [PATCH 04/10] fix: docker --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4da2711..11836f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -80,7 +80,7 @@ ARG APP_PATH # Install Solana CLI, we redo this step because this Docker target # starts from scratch without the earlier Solana installation -RUN sh -c "$(curl -sSfL https://release.solana.com/v1.14.17/install" +RUN sh -c "$(curl -sSfL https://release.solana.com/v1.14.17/install)" ENV PATH=$PATH:/root/.local/share/solana/install/active_release/bin ENV \ From 52166beeaf6317e4b0fffd90d08c57757b0fd096 Mon Sep 17 00:00:00 2001 From: Ali Behjati Date: Thu, 5 Sep 2024 19:36:07 +0000 Subject: [PATCH 05/10] fix: use str with trimming --- program_admin/publisher_program_instructions.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/program_admin/publisher_program_instructions.py b/program_admin/publisher_program_instructions.py index 49f32a2..fa8ccf6 100644 --- a/program_admin/publisher_program_instructions.py +++ b/program_admin/publisher_program_instructions.py @@ -73,10 +73,15 @@ def create_buffer_account( space: int, lamports: int, ) -> Tuple[PublicKey, TransactionInstruction]: - # The seed should be str but later is used in rust as - # &str and therefore we use latin1 encoding to map byte - # i to char i - seed = bytes(publisher_pubkey).decode("latin-1") + # Since the string representation of the PublicKey is 44 bytes long (base58 encoded) + # and we use 32 bytes of it, the chances of collision are very low. + # + # The seed has a max length of 32 and although the publisher_pubkey is 32 bytes, + # it is impossible to convert it to a string with a length of 32 that the + # underlying library (solders) can handle. We don't know exactly why, but it + # seems to be related to str -> &str conversion in pyo3 that solders uses to + # interact with the Rust implementation of the logic. + seed = str(publisher_pubkey)[:32] new_account_pubkey = PublicKey.create_with_seed( base_pubkey, seed, From a1bd54f0b3693e5949ac01e27f0b93244a3e258e Mon Sep 17 00:00:00 2001 From: Ali Behjati Date: Fri, 6 Sep 2024 14:22:01 +0000 Subject: [PATCH 06/10] fix: add instruction ids --- program_admin/__init__.py | 2 ++ program_admin/publisher_program_instructions.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/program_admin/__init__.py b/program_admin/__init__.py index 72ee458..7bb5152 100644 --- a/program_admin/__init__.py +++ b/program_admin/__init__.py @@ -327,6 +327,8 @@ async def sync( publisher_program_signers, ) = await self.sync_publisher_program(ref_publishers) + logger.debug(f"Syncing publisher program - {len(publisher_program_instructions)} instructions") + if publisher_program_instructions: instructions.extend(publisher_program_instructions) if send_transactions: diff --git a/program_admin/publisher_program_instructions.py b/program_admin/publisher_program_instructions.py index fa8ccf6..e0f0e57 100644 --- a/program_admin/publisher_program_instructions.py +++ b/program_admin/publisher_program_instructions.py @@ -44,12 +44,14 @@ def initialize_publisher_program( ) ix_data_layout = Struct( + "instruction_id" / Int8ul, "bump" / Int8ul, "authority" / Bytes(32), ) ix_data = ix_data_layout.build( dict( + instruction_id=0, bump=bump, authority=bytes(authority), ) @@ -132,6 +134,7 @@ def initialize_publisher_config( ) ix_data_layout = Struct( + "instruction_id" / Int8ul, "config_bump" / Int8ul, "publisher_config_bump" / Int8ul, "publisher" / Bytes(32), @@ -139,6 +142,7 @@ def initialize_publisher_config( ix_data = ix_data_layout.build( dict( + instruction_id=2, config_bump=config_bump, publisher_config_bump=publisher_config_bump, publisher=bytes(publisher_key), From af0ee0b88a1de9fa620b8b16f1ce82189475ff8e Mon Sep 17 00:00:00 2001 From: Ali Behjati Date: Fri, 6 Sep 2024 16:26:17 +0200 Subject: [PATCH 07/10] fix: update formatting --- program_admin/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/program_admin/__init__.py b/program_admin/__init__.py index 7bb5152..219ea55 100644 --- a/program_admin/__init__.py +++ b/program_admin/__init__.py @@ -327,7 +327,9 @@ async def sync( publisher_program_signers, ) = await self.sync_publisher_program(ref_publishers) - logger.debug(f"Syncing publisher program - {len(publisher_program_instructions)} instructions") + logger.debug( + f"Syncing publisher program - {len(publisher_program_instructions)} instructions" + ) if publisher_program_instructions: instructions.extend(publisher_program_instructions) From 347ccb7b6e04a5ff393901edc97a61fcffb36860 Mon Sep 17 00:00:00 2001 From: Ali Behjati Date: Fri, 6 Sep 2024 19:57:50 +0200 Subject: [PATCH 08/10] fix: update init publisher instruction --- program_admin/publisher_program_instructions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program_admin/publisher_program_instructions.py b/program_admin/publisher_program_instructions.py index e0f0e57..b13a23b 100644 --- a/program_admin/publisher_program_instructions.py +++ b/program_admin/publisher_program_instructions.py @@ -155,7 +155,7 @@ def initialize_publisher_config( AccountMeta(pubkey=authority, is_signer=True, is_writable=True), AccountMeta(pubkey=config_account, is_signer=False, is_writable=True), AccountMeta( - pubkey=publisher_config_account, is_signer=False, is_writable=True + pubkey=publisher_config_account, is_signer=False, is_writable=False ), AccountMeta(pubkey=buffer_account, is_signer=False, is_writable=True), AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), From 108ee4e92375a14048ee00a2e3c7c5287479e694 Mon Sep 17 00:00:00 2001 From: Ali Behjati Date: Mon, 9 Sep 2024 14:22:29 +0200 Subject: [PATCH 09/10] fix: remove duplicate account_exists --- program_admin/__init__.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/program_admin/__init__.py b/program_admin/__init__.py index 219ea55..e7aa1fd 100644 --- a/program_admin/__init__.py +++ b/program_admin/__init__.py @@ -114,12 +114,6 @@ async def fetch_minimum_balance(self, size: int) -> int: async with AsyncClient(self.rpc_endpoint) as client: return (await client.get_minimum_balance_for_rent_exemption(size)).value - async def account_exists(self, key: PublicKey) -> bool: - async with AsyncClient(self.rpc_endpoint) as client: - response = await client.get_account_info(key) - # The RPC returns null if the account does not exist - return bool(response.value) - async def refresh_program_accounts(self): async with AsyncClient(self.rpc_endpoint) as client: logger.info("Refreshing program accounts") @@ -711,7 +705,7 @@ async def sync_publisher_program( ) # Initialize the publisher program config if it does not exist - if not (await self.account_exists(publisher_program_config)): + if not (await account_exists(self.rpc_endpoint, publisher_program_config)): initialize_publisher_program_instruction = initialize_publisher_program( self.publisher_program_key, authority.public_key ) @@ -723,7 +717,7 @@ async def sync_publisher_program( publisher, self.publisher_program_key ) - if not (await self.account_exists(publisher_config_account)): + if not (await account_exists(self.rpc_endpoint, publisher_config_account)): size = 100048 # This size is for a buffer supporting 5000 price updates lamports = await self.fetch_minimum_balance(size) buffer_account, create_buffer_instruction = create_buffer_account( From 416f35379c3f11ff3f67dbeb81a7f87948ef427c Mon Sep 17 00:00:00 2001 From: Ali Behjati Date: Mon, 9 Sep 2024 14:50:52 +0200 Subject: [PATCH 10/10] chore: bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index aa50c62..dc43cfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ known_local_folder = ["program_admin"] authors = ["Thomaz "] description = "Syncs products and publishers of the Pyth program" name = "program-admin" -version = "0.1.3" +version = "0.1.4" [tool.poetry.dependencies] click = "^8.1.0"