Testing workflow with basic tests (#39)

* Setup pytest

* Tests for new api

* Tests and updates for utils

* Update testing workflow

* Update testing workflow

* Better env close implementation

* Attempt to isolate error

* Create a lite test suite which uses less memory

.github/workflows/test.yml

@@ -43,4 +43,4 @@ jobs:
       - name: Install evolution gym
         run: pip install -e .
       - name: Run test
-        run: xvfb-run python -m unittest tests/test_render.py
+        run: xvfb-run python -m pytest -s -v -n auto -m lite

evogym/envs/base.py

@@ -119,7 +119,9 @@ def close(self) -> None:
         """
         Close the simulation.
         """
-        self.default_viewer.hide_debug_window() 
+        self.default_viewer.close()
+        del self._default_viewer
+        del self._sim
 
     def get_actuator_indices(self, robot_name: str) -> np.ndarray:
         """

evogym/simulator/SimulatorCPP/Interface.cpp

@@ -582,6 +582,11 @@ void Interface::hide_debug_window() {
 	debug_window_showing = false;
 }
 
+void Interface::close() {
+	glfwDestroyWindow(debug_window);
+	glfwTerminate();
+}
+
 vector<int> Interface::get_debug_window_pos() {
 
 	int xpos, ypos;

evogym/simulator/SimulatorCPP/Interface.h

@@ -68,6 +68,7 @@ class Interface
 
 	void show_debug_window();
 	void hide_debug_window();
+	void close();
 	vector<int> get_debug_window_pos();
 
 	GLFWwindow* get_debug_window_ref();

evogym/simulator/SimulatorCPP/PythonBindings.cpp

@@ -21,6 +21,7 @@ PYBIND11_MODULE(simulator_cpp, m) {
 		.def("render", &Interface::render, py::arg("camera"), py::arg("hide_background") = false, py::arg("hide_grid") = false, py::arg("hide_edges") = false, py::arg("hide_boxels") = false, py::arg("dont_clear") = false)
 		.def("show_debug_window", &Interface::show_debug_window)
 		.def("hide_debug_window", &Interface::hide_debug_window)
+		.def("close", &Interface::close)
 		.def("get_debug_window_pos", &Interface::get_debug_window_pos, py::return_value_policy::copy);
 
 	py::class_<Sim>(m, "Sim")

evogym/utils.py

@@ -57,26 +57,32 @@ def get_uniform(x: int) -> np.ndarray:
     Return a uniform distribution of a given size.
 
     Args:
-        x (int): size of distribution.
+        x (int): size of distribution. Must be positive.
     
     Returns:
         np.ndarray: array representing the probability distribution.
     """
+    assert x > 0, f"Invalid size {x} for uniform distribution. Must be positive."
     return np.ones((x)) / x
 
 def draw(pd: np.ndarray) -> int:
     """
     Sample from a probability distribution.
 
     Args:
-        pd (np.ndarray): array representing the probability of sampling each element.
+        pd (np.ndarray): array representing the relative probability of sampling each element. Entries must be non-negative and sum to a non-zero value. Must contain at least one element.
     
     Returns:
         int: sampled index.
     """
     pd_copy = pd.copy()
-    if (type(pd_copy) != type(np.array([]))):
+    if not isinstance(pd_copy, np.ndarray):
         pd_copy = np.array(pd_copy)
+
+    assert pd_copy.size > 0, f"Invalid size {pd_copy.size} for probability distribution. Must contain at least one element."
+    assert np.all(pd_copy >= 0), f"Invalid probability distribution {pd_copy}. Entries must be non-negative."
+    assert np.sum(pd_copy) > 0, f"Invalid probability distribution {pd_copy}. Entries must sum to a non-zero value."
+
     pd_copy = pd_copy / pd_copy.sum()
 
     rand = random.uniform(0, 1)
@@ -88,17 +94,31 @@ def draw(pd: np.ndarray) -> int:
 
 def sample_robot(
     robot_shape: Tuple[int, int], 
-    pd: np.ndarray = None) -> Tuple[np.ndarray, np.ndarray]:
+    pd: Optional[np.ndarray] = None
+) -> Tuple[np.ndarray, np.ndarray]:
     """
     Return a randomly sampled robot of a particular size.
 
     Args:
         robot_shape (Tuple(int, int)): robot shape to sample `(h, w)`.
-        pd (np.ndarray): `(5,)` array representing the probability of sampling each robot voxel (empty, rigid, soft, h_act, v_act). Defaults to a custom distribution. (default = None)
+        pd (np.ndarray): `(5,)` array representing the relative probability of sampling each robot voxel (empty, rigid, soft, h_act, v_act). Defaults to a custom distribution. (default = None)
     
     Returns:
         Tuple[np.ndarray, np.ndarray]: randomly sampled (valid) robot voxel array and its associated connections array.
+        
+    Throws:
+        If it is not possible to sample a connected robot with at least one actuator.
     """
+
+    h_act, v_act, empty = VOXEL_TYPES['H_ACT'], VOXEL_TYPES['V_ACT'], VOXEL_TYPES['EMPTY']
+
+    if pd is not None:
+        assert pd.shape == (5,), f"Invalid probability distribution {pd}. Must have shape (5,)."
+        if pd[h_act] + pd[v_act] == 0:
+            raise ValueError(f"Invalid probability distribution {pd}. Must have a non-zero probability of sampling an actuator.")
+        if sum(pd) - pd[empty] == 0:
+            raise ValueError(f"Invalid probability distribution {pd}. Must have a non-zero probability of sampling a non-empty voxel.")
+
     done = False
 
     while (not done):
@@ -219,7 +239,7 @@ def has_actuator(robot: np.ndarray) -> bool:
 
 def get_full_connectivity(robot: np.ndarray) -> np.ndarray:
     """
-    Returns a connections array given a connected robot structure. Assumes all adjacent voxels are connected.
+    Returns a connections array given a structure. Assumes all adjacent voxels are connected.
 
     Args:
         robot (np.ndarray): array specifing the voxel structure of the robot.

evogym/viewer.py

@@ -134,6 +134,13 @@ def hide_debug_window(self,) -> None:
         if self._has_init_viewer:
             self._viewer.hide_debug_window()
 
+    def close(self,) -> None:
+        """
+        Close the viewer.
+        """
+        if self._has_init_viewer:
+            self._viewer.close()
+
     def track_objects(self, *objects: Tuple[str]) -> None:
         """
         Set objects for the viewer to automatically track. The viewer tracks objects by adjusting its `pos` and `view_size` automatically every time before rendering.

pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+addopts = --ignore=examples/externals --ignore=evogym/simulator/externals --ignore=evogym/examples/gym_test.py
+markers =
+    lite: mark as part of the lite test suite

requirements.txt

@@ -16,3 +16,5 @@ PyOpenGL-accelerate==3.1.5
 torch==1.10.2
 ttkbootstrap==1.5.1
 typing==3.7.4.3
+pytest
+pytest-xdist

tests/requires_screen/test_screen_render_modes.py

@@ -0,0 +1,78 @@
+import gymnasium as gym
+import pytest
+import warnings
+import numpy as np
+from itertools import product
+
+import evogym.envs
+from evogym import sample_robot
+
+LITE_TEST_ENV_NAMES = [
+    "Pusher-v0",
+    "Walker-v0",
+    "Traverser-v0",
+]
+
+@pytest.mark.lite
+@pytest.mark.parametrize(
+    "render_mode, add_options",
+    list(product(
+        ["human", "screen"],
+        [True, False],
+    ))
+)
+def test_render_modes(render_mode, add_options):
+    """
+    - Env can render to screen
+    """
+
+    body, _ = sample_robot((5, 5))
+    if add_options:
+        env = gym.make("Walker-v0", body=body, render_mode=render_mode, render_options={
+            "verbose": False,
+            "hide_background": False,
+            "hide_grid": False,
+            "hide_edges": False,
+            "hide_voxels": False
+        })
+    else:
+        env = gym.make("Walker-v0", body=body, render_mode=render_mode)
+
+    # Reset
+    obs, info = env.reset(seed=None, options=None)
+
+    for i in range(10):
+
+        # Step -- we don't need to render explicitly
+        action = env.action_space.sample() - 1
+        ob, reward, terminated, truncated, info = env.step(action)
+
+    env.close()
+
+def get_all_env_render_params():
+    return [
+        env_name if env_name not in LITE_TEST_ENV_NAMES 
+        else pytest.param(env_name, marks=pytest.mark.lite) 
+        for env_name in evogym.BASELINE_ENV_NAMES 
+    ]
+
+@pytest.mark.parametrize("env_name", get_all_env_render_params())
+def test_all_env_render(env_name):
+    """
+    - Env can render to screen
+    """
+
+    body, _ = sample_robot((5, 5))
+    env = gym.make(env_name, body=body, render_mode="human")
+
+    # Reset
+    obs, info = env.reset(seed=None, options=None)
+
+    for i in range(10):
+
+        # Step -- we don't need to render explicitly
+        action = env.action_space.sample() - 1
+        ob, reward, terminated, truncated, info = env.step(action)
+
+    env.close()
+

tests/screen_free/test_baseline_envs.py

@@ -0,0 +1,62 @@
+import gymnasium as gym
+import pytest
+import warnings
+
+import evogym.envs
+from evogym import sample_robot
+
+LITE_TEST_ENV_NAMES = [
+    "Pusher-v0",
+    "Walker-v0",
+    "Traverser-v0",
+]
+
+def get_params():
+    return [
+        env_name if env_name not in LITE_TEST_ENV_NAMES 
+        else pytest.param(env_name, marks=pytest.mark.lite) 
+        for env_name in evogym.BASELINE_ENV_NAMES 
+    ]
+
+@pytest.mark.parametrize("env_name", get_params())
+def test_env_creatable_and_has_correct_api(env_name):
+    """
+    - Env is creatable
+    - Env steps for the correct number of steps
+    - Env follows the gym API
+    """
+
+    body, _ = sample_robot((5, 5))
+    env = gym.make(env_name, body=body)
+
+    target_steps = env.spec.max_episode_steps
+    assert isinstance(target_steps, int), f"Env {env_name} does not have a max_episode_steps attribute"
+
+    # Reset
+    obs, info = env.reset(seed=None, options=None)
+
+    # Rollout with random actions
+    n_steps = 0
+    while True:
+        action = env.action_space.sample() - 1
+        ob, reward, terminated, truncated, info = env.step(action)
+
+        n_steps += 1
+
+        if terminated or truncated:
+            env.reset(seed=None, options=None)
+            break
+
+        if n_steps > target_steps:
+            break
+
+    # Make sure we can still step after resetting
+    env.step(env.action_space.sample() - 1)
+
+    # Check that the env terminated after the correct number of steps
+    assert n_steps <= target_steps, f"Env {env_name} terminated after {n_steps} steps, expected at most {target_steps}"
+    if n_steps < target_steps:
+        warnings.warn(f"Env {env_name} terminated early after {n_steps} steps, expected {target_steps}")
+
+    env.close()
+

tests/screen_free/test_img_render_modes.py

@@ -0,0 +1,58 @@
+import gymnasium as gym
+import pytest
+import warnings
+import numpy as np
+from itertools import product
+
+import evogym.envs
+from evogym import sample_robot
+
+LITE_TEST_ENV_NAMES = [
+    "Pusher-v0",
+    "Walker-v0",
+    "Traverser-v0",
+]
+
+def get_params():
+    params = product(
+        evogym.BASELINE_ENV_NAMES,
+        [None, "img", "rgb_array"],
+    )
+    return [
+        param if param[0] not in LITE_TEST_ENV_NAMES 
+        else pytest.param(*param, marks=pytest.mark.lite) 
+        for param in params 
+    ]
+
+
+@pytest.mark.parametrize("env_name, render_mode", get_params())
+def test_render(env_name, render_mode):
+    """
+    - Env can render to none and to image
+    """
+
+    body, _ = sample_robot((5, 5))
+    env = gym.make(env_name, body=body, render_mode=render_mode)
+
+    # Reset
+    obs, info = env.reset(seed=None, options=None)
+
+    for i in range(10):
+
+        # Render
+        result = env.render()
+
+        if render_mode is None:
+            # Result should be None
+            assert result is None, f"Env returned {type(result)} instead of None"
+        else:
+            # Check img
+            assert isinstance(result, np.ndarray), f"Env returned {type(result)} instead of np.ndarray"
+            x, y, c = result.shape
+            assert c == 3, f"Env returned image with {c} channels, expected 3"
+
+        # Step
+        action = env.action_space.sample() - 1
+        ob, reward, terminated, truncated, info = env.step(action)
+
+    env.close()