feat: Support PocketIC server 7.0

CHANGELOG.md

@@ -9,6 +9,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## Unreleased
 
 
+## 3.0.0 - 2024-11-19
+
+### Added 
+- Support for PocketIC server version 7.0.0
+- Load a state directory for any subnet kind with `SubnetConfig.add_subnet_with_state`
+- Verified Application subnet type
+
+### Removed
+- `with_nns_state`. Use `SubnetConfig.add_subnet_with_state` instead
+
+
+
 ## 2.1.0 - 2024-02-08
 
 ### Added

flake.nix

@@ -1,15 +1,15 @@
 {
-  description = "PocketIC Python Libary";
+  description = "PocketIC Python Library";
 
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
     flake-utils.url = "github:numtide/flake-utils";
     pocket-ic-darwin-gz = {
-      url = "https://github.com/dfinity/pocketic/releases/download/3.0.1/pocket-ic-x86_64-darwin.gz";
+      url = "https://github.com/dfinity/pocketic/releases/download/7.0.0/pocket-ic-x86_64-darwin.gz";
       flake = false;
     };
     pocket-ic-linux-gz = {
-      url = "https://github.com/dfinity/pocketic/releases/download/3.0.1/pocket-ic-x86_64-linux.gz";
+      url = "https://github.com/dfinity/pocketic/releases/download/7.0.0/pocket-ic-x86_64-linux.gz";
       flake = false;
     };
   };
@@ -86,7 +86,7 @@
         '';
 
         checks.default = pkgs.runCommand "pocketic-py-tests" {
-          nativeBuildInputs = [ pytest pocketic-py];
+          nativeBuildInputs = [ pytest pocketic-py pkgs.cacert];
           POCKET_IC_BIN = "${pocket-ic}/bin/pocket-ic";
           inherit projectDir;
         } ''

pocket_ic/pocket_ic.py

@@ -3,11 +3,12 @@
 It also contains 'SubnetConfig' and 'SubnetKind', which are used to configure the
 subnets of a PocketIC instance.
 """
+
 import base64
 import ic
 from enum import Enum
 from ic.candid import Types
-from typing import List, Optional, Any
+from typing import Optional, Any
 from pocket_ic.pocket_ic_server import PocketICServer
 
 
@@ -21,6 +22,7 @@ class SubnetKind(Enum):
     NNS = "NNS"
     SNS = "SNS"
     SYSTEM = "System"
+    VERIFIED_APPLICATION = "VerifiedApplication"
 
 
 class SubnetConfig:
@@ -35,24 +37,26 @@ def __init__(
         nns=False,
         sns=False,
         system=0,
+        verified_application=0,
     ) -> None:
-        self.application = application
-        self.bitcoin = bitcoin
-        self.fiduciary = fiduciary
-        self.ii = ii
-        self.nns = nns
-        self.sns = sns
-        self.system = system
+        new = {"state_config": "New", "instruction_config": "Production"}
+        self.application = [new] * application
+        self.bitcoin = new if bitcoin else None
+        self.fiduciary = new if fiduciary else None
+        self.ii = new if ii else None
+        self.nns = new if nns else None
+        self.sns = new if sns else None
+        self.system = [new] * system
+        self.verified_application = [new] * verified_application
 
     def __repr__(self) -> str:
-        return f"SubnetConfigSet(application={self.application}, bitcoin={self.bitcoin}, fiduciary={self.fiduciary}, ii={self.ii}, nns={self.nns}, sns={self.sns}, system={self.system})"
+        return f"SubnetConfigSet(application={self.application}, bitcoin={self.bitcoin}, fiduciary={self.fiduciary}, ii={self.ii}, nns={self.nns}, sns={self.sns}, system={self.system}, verified_application={self.verified_application})"
 
     def validate(self) -> None:
         """Validates the subnet configuration.
 
         Raises:
-            ValueError: if no subnet is configured or if the number of application or system
-                subnets is negative
+            ValueError: if no subnet is configured
         """
         if not (
             self.bitcoin
@@ -62,34 +66,74 @@ def validate(self) -> None:
             or self.sns
             or self.system
             or self.application
+            or self.verified_application
         ):
             raise ValueError("At least one subnet must be configured.")
 
-        if self.application < 0 or self.system < 0:
-            raise ValueError(
-                "The number of application and system subnets must be non-negative."
-            )
+    def add_subnet_with_state(
+        self, subnet_type: SubnetKind, state_dir_path: str, subnet_id: ic.Principal
+    ):
+        """Add a subnet with state loaded form the given state directory.
+        Note that the provided path must be accessible for the PocketIC server process.
+
+        `state_dir` should point to a directory which is expected to have the following structure:
+
+        state_dir/
+         |-- backups
+         |-- checkpoints
+         |-- diverged_checkpoints
+         |-- diverged_state_markers
+         |-- fs_tmp
+         |-- page_deltas
+         |-- states_metadata.pbuf
+         |-- tip
+         `-- tmp
+
+        `subnet_id` should be the subnet ID of the subnet in the state to be loaded"""
+
+        raw_subnet_id = base64.b64encode(subnet_id.bytes).decode()
+
+        new_from_path = {
+            "state_config": {
+                "FromPath": [state_dir_path, {"subnet_id": raw_subnet_id}]
+            },
+            "instruction_config": "Production",
+        }
 
-    def with_nns_state(self, state_dir_path: str, nns_subnet_id: ic.Principal):
-        """Provide an NNS state directory and a subnet id. """
-        self.nns = (state_dir_path, nns_subnet_id)
+        match subnet_type:
+            case SubnetKind.APPLICATION:
+                self.application.append(new_from_path)
+            case SubnetKind.BITCOIN:
+                self.bitcoin = new_from_path
+            case SubnetKind.FIDUCIARY:
+                self.fiduciary = new_from_path
+            case SubnetKind.II:
+                self.ii = new_from_path
+            case SubnetKind.NNS:
+                self.nns = new_from_path
+            case SubnetKind.SNS:
+                self.sns = new_from_path
+            case SubnetKind.SYSTEM:
+                self.system.append(new_from_path)
+            case SubnetKind.VERIFIED_APPLICATION:
+                self.verified_application.append(new_from_path)
 
     def _json(self) -> dict:
-        if isinstance(self.nns, tuple):
-            raw_subnet_id = base64.b64encode(self.nns[1].bytes).decode()
-            nns = {"FromPath": (self.nns[0], {"subnet_id": raw_subnet_id})}
-        elif self.nns:
-            nns = "New"
-        else:
-            nns = None
         return {
-            "application": self.application * ["New"],
-            "bitcoin": "New" if self.bitcoin else None,
-            "fiduciary": "New" if self.fiduciary else None,
-            "ii": "New" if self.ii else None,
-            "nns": nns,
-            "sns": "New" if self.sns else None,
-            "system": self.system * ["New"],
+            "subnet_config_set": {
+                "application": self.application,
+                "bitcoin": self.bitcoin,
+                "fiduciary": self.fiduciary,
+                "ii": self.ii,
+                "nns": self.nns,
+                "sns": self.sns,
+                "system": self.system,
+                "verified_application": self.verified_application,
+            },
+            "state_dir": None,
+            "nonmainnet_features": False,
+            "log_level": None,
+            "bitcoind_addr": None,
         }
 
 
@@ -448,7 +492,7 @@ def create_and_install_canister_with_candid(
             arg = [{"type": canister_arguments[0], "value": init_args}]
         else:
             raise ValueError("The candid file appears to be malformed")
-        
+
         self.add_cycles(canister_id, 2_000_000_000_000)
         self.install_code(canister_id, wasm_module, arg)
         return canister
@@ -501,7 +545,8 @@ def _canister_call(
 
     def _generate_topology(self, topology):
         t = dict()
-        for subnet_id, config in topology.items():
+        subnets = topology["subnet_configs"]
+        for subnet_id, config in subnets.items():
             subnet_id = ic.Principal.from_str(subnet_id)
             subnet_kind = SubnetKind(config["subnet_kind"])
             t.update({subnet_id: subnet_kind})

pocket_ic/pocket_ic_server.py

@@ -9,9 +9,6 @@
 from tempfile import gettempdir
 
 
-HEADERS = {"processing-timeout-ms": "300000"}
-
-
 class PocketICServer:
     """
     An object of this class represents a running PocketIC server. During instantiation,
@@ -46,12 +43,15 @@ def __init__(self) -> None:
 """
             )
 
-        # Attempt to start the PocketIC server if it's not already running.
         mute = (
             "1> /dev/null 2> /dev/null" if "POCKET_IC_MUTE_SERVER" in os.environ else ""
         )
-        os.system(f"{bin_path} --pid {pid} {mute} &")
-        self.url = self._get_url(pid)
+        # Attempt to start the PocketIC server if it's not already running.
+        tmp_dir = gettempdir()
+        port_file_path = f"{tmp_dir}/pocket_ic_{pid}.port"
+
+        os.system(f"{bin_path} --port-file {port_file_path} {mute} &")
+        self.url = self._get_url(port_file_path)
         self.request_client = requests.session()
 
     def new_instance(self, subnet_config: dict) -> Tuple[int, dict]:
@@ -61,7 +61,7 @@ def new_instance(self, subnet_config: dict) -> Tuple[int, dict]:
             str: the new instance ID
         """
         url = f"{self.url}/instances"
-        response = self.request_client.post(url, headers=HEADERS, json=subnet_config)
+        response = self.request_client.post(url, json=subnet_config)
         res = self._check_response(response)["Created"]
         return res["instance_id"], res["topology"]
 
@@ -72,7 +72,7 @@ def list_instances(self) -> List[str]:
             List[str]: a list of instance names
         """
         url = f"{self.url}/instances"
-        response = self.request_client.get(url, headers=HEADERS)
+        response = self.request_client.get(url)
         response = self._check_response(response)
         return response
 
@@ -83,18 +83,18 @@ def delete_instance(self, instance_id: int):
             instance_id (int): the ID of the instance to delete
         """
         url = f"{self.url}/instances/{instance_id}"
-        self.request_client.delete(url, headers=HEADERS)
+        self.request_client.delete(url)
 
     def instance_get(self, endpoint: str, instance_id: int):
         """HTTP get requests for instance endpoints"""
         url = f"{self.url}/instances/{instance_id}/{endpoint}"
-        response = self.request_client.get(url, headers=HEADERS)
+        response = self.request_client.get(url)
         return self._check_response(response)
 
     def instance_post(self, endpoint: str, instance_id: int, body: Optional[dict]):
         """HTTP post requests for instance endpoints"""
         url = f"{self.url}/instances/{instance_id}/{endpoint}"
-        response = self.request_client.post(url, json=body, headers=HEADERS)
+        response = self.request_client.post(url, json=body)
         return self._check_response(response)
 
     def set_blob_store_entry(self, blob: bytes, compression: Optional[str]) -> str:
@@ -109,44 +109,38 @@ def set_blob_store_entry(self, blob: bytes, compression: Optional[str]) -> str:
         """
         url = f"{self.url}/blobstore"
         if compression is None:
-            response = self.request_client.post(url, data=blob, headers=HEADERS)
+            response = self.request_client.post(url, data=blob)
         elif compression == "gzip":
-            headers = HEADERS | {"Content-Encoding": "gzip"}
+            headers = {"Content-Encoding": "gzip"}
             response = self.request_client.post(url, data=blob, headers=headers)
         else:
             raise ValueError('only "gzip" compression is supported')
 
         self._check_status_code(response)
         return response.text
 
-    def _get_url(self, pid: int) -> str:
-        tmp_dir = gettempdir()
-        ready_file_path = f"{tmp_dir}/pocket_ic_{pid}.ready"
-        port_file_path = f"{tmp_dir}/pocket_ic_{pid}.port"
-
+    def _get_url(self, port_file_path: int) -> str:
         stop_at = time.time() + 10  # Wait for the ready file for 10 seconds
 
-        while not os.path.exists(ready_file_path):
-            if time.time() < stop_at:
-                time.sleep(0.1)  # 100ms
-            else:
-                raise TimeoutError("PocketIC failed to start")
+        while True:
+            if os.path.isfile(port_file_path):
+                with open(port_file_path, "r", encoding="utf-8") as port_file:
+                    port = port_file.readline().strip()
+                if port:
+                    return f"http://127.0.0.1:{port}"
 
-        if os.path.isfile(ready_file_path):
-            with open(port_file_path, "r", encoding="utf-8") as port_file:
-                port = port_file.readline().strip()
-        else:
-            raise ValueError(f"{ready_file_path} is not a file!")
+            time.sleep(0.02)  # wait for 20ms
 
-        return f"http://127.0.0.1:{port}"
+            if time.time() > stop_at:
+                raise TimeoutError("PocketIC failed to start")
 
-    def _check_response(self, response):
+    def _check_response(self, response: requests.Response):
         self._check_status_code(response)
         res_json = response.json()
         return res_json
 
-    def _check_status_code(self, response):
+    def _check_status_code(self, response: requests.Response):
         if response.status_code not in [200, 201, 202]:
             raise ConnectionError(
-                f'PocketIC server returned status code {response.status_code}: "{response.reason}"'
+                f'PocketIC server returned status code {response.status_code}: "{response.text}"'
             )

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pocket_ic"
-version = "2.1.0"
+version = "3.0.0"
 description = "PocketIC: A Canister Smart Contract Testing Platform"
 authors = [
     "The Internet Computer Project Developers <dept-testing_&_verification@dfinity.org>",