From bfce5e922f5b0f56c65d65d590e7293d35ab1fb0 Mon Sep 17 00:00:00 2001 From: Maksymilian Wojnar Date: Mon, 17 Jul 2023 21:18:19 +0200 Subject: [PATCH] Move Wi-Fi specific classes to examples --- docs/source/agents.rst | 30 ++-- docs/source/custom_extensions.rst | 18 +-- docs/source/extensions.rst | 21 --- .../ns-3-ccod/ext.py | 4 +- examples/ns-3-ccod/main.py | 3 +- .../ns-3-ra/ext.py | 4 +- examples/ns-3-ra/main.py | 6 +- .../ns-3-ra}/particle_filter.py | 0 examples/ra-sim/ext.py | 128 ++++++++++++++++++ examples/ra-sim/main.py | 5 +- examples/ra-sim/utils.py | 2 +- examples/recommender_system/main.py | 1 + reinforced_lib/agents/wifi/__init__.py | 1 - reinforced_lib/exts/wifi/__init__.py | 2 - requirements/requirements.txt | 4 +- 15 files changed, 161 insertions(+), 68 deletions(-) rename reinforced_lib/exts/wifi/ieee_802_11_ccod.py => examples/ns-3-ccod/ext.py (96%) rename reinforced_lib/exts/wifi/ieee_802_11_ax_ra.py => examples/ns-3-ra/ext.py (97%) rename {reinforced_lib/agents/wifi => examples/ns-3-ra}/particle_filter.py (100%) create mode 100644 examples/ra-sim/ext.py delete mode 100644 reinforced_lib/agents/wifi/__init__.py delete mode 100644 reinforced_lib/exts/wifi/__init__.py diff --git a/docs/source/agents.rst b/docs/source/agents.rst index eeab3a2..527e7d0 100644 --- a/docs/source/agents.rst +++ b/docs/source/agents.rst @@ -103,28 +103,6 @@ Exp3 :members: -Particle filter (Core) ----------------------- - -.. automodule:: reinforced_lib.agents.core.particle_filter - :show-inheritance: - :members: - - -Particle filter (Wi-Fi) ------------------------ - -.. currentmodule:: reinforced_lib.agents.wifi.particle_filter - -.. autoclass:: ParticleFilterState - :show-inheritance: - :members: - -.. autoclass:: ParticleFilter - :show-inheritance: - :members: - - Softmax --------------- @@ -165,3 +143,11 @@ Upper confidence bound (UCB) .. autoclass:: UCB :show-inheritance: :members: + + +Particle filter (Core) +---------------------- + +.. automodule:: reinforced_lib.agents.core.particle_filter + :show-inheritance: + :members: diff --git a/docs/source/custom_extensions.rst b/docs/source/custom_extensions.rst index b77e559..748f11f 100644 --- a/docs/source/custom_extensions.rst +++ b/docs/source/custom_extensions.rst @@ -14,8 +14,8 @@ Key concepts There are three main benefits of using extensions: #. Automatic initialization of agents - an extension can provide default arguments that can be used to - initialize an agent. For example, if we would like to create the ``wifi.ParticleFilter`` - :ref:`agent ` without using any extension, we would probably do it in the + initialize an agent. For example, if we would like to create the Particle Filter (Wi-Fi) + without using any extension, we would probably do it in the following way: .. code-block:: python @@ -27,7 +27,7 @@ There are three main benefits of using extensions: } ) - On the other hand, if we decide to use the ``IEEE_802_11_ax_RA`` :ref:`extension ` extension, + On the other hand, if we decide to use the IEEE 802.11ax RA extension, this parameters can be automatically provided by the extension: .. code-block:: python @@ -72,15 +72,15 @@ There are three main benefits of using extensions: ) The following code is equivalent to the above but makes use of the properly defined - ``IEEE_802_11_ax_RA`` :ref:`extension ` extension: + IEEE 802.11ax RA extension: .. code-block:: python action = rl.sample(**observations) #. Filling missing parameters - some parameters required by the agent can be filled with known values or - calculated based on a set of basic observations. For example, a ``sample`` method of the ``wifi.ParticleFilter`` - :ref:`agent ` requires transmission data rates for each MCS. These values can be found in + calculated based on a set of basic observations. For example, a ``sample`` method of the + Particle Filter (Wi-Fi) requires transmission data rates for each MCS. These values can be found in the IEEE 802.11ax standard documentation. Below is a sample code that could be used to sample the next action from the agent without using any extension: @@ -97,7 +97,7 @@ There are three main benefits of using extensions: } action = rl.sample(**observations) - If we use the ``IEEE_802_11_ax_RA`` :ref:`extension ` extension, part of these parameters can be + If we use the IEEE 802.11ax RA extension, part of these parameters can be provided by the extension: .. code-block:: python @@ -134,7 +134,7 @@ Customizing extensions ---------------------- To create your own extension, you should inherit from the :ref:`abstract class ` ``BaseExt``. We -present adding a custom extension using an example of the ``IEEE_802_11_ax_RA`` :ref:`extension ` extension. +present adding a custom extension using an example of the IEEE 802.11ax RA extension. .. code-block:: python @@ -217,7 +217,7 @@ type in the decorator: else: return 0.0 -The full source code of the IEEE 802.11ax extension can be found `here `_. +The full source code of the IEEE 802.11ax extension can be found `here `_. Rules and limitations diff --git a/docs/source/extensions.rst b/docs/source/extensions.rst index 547909a..8eff19b 100644 --- a/docs/source/extensions.rst +++ b/docs/source/extensions.rst @@ -25,24 +25,3 @@ Gymnasium .. autoclass:: Gymnasium :show-inheritance: :members: - - - -IEEE 802.11ax RA ----------------- - -.. currentmodule:: reinforced_lib.exts.wifi.ieee_802_11_ax_ra - -.. autoclass:: IEEE_802_11_ax_RA - :show-inheritance: - :members: - - -IEEE 802.11 CCOD ----------------- - -.. currentmodule:: reinforced_lib.exts.wifi.ieee_802_11_ccod - -.. autoclass:: IEEE_802_11_CCOD - :show-inheritance: - :members: diff --git a/reinforced_lib/exts/wifi/ieee_802_11_ccod.py b/examples/ns-3-ccod/ext.py similarity index 96% rename from reinforced_lib/exts/wifi/ieee_802_11_ccod.py rename to examples/ns-3-ccod/ext.py index bbab3f1..24cbb62 100644 --- a/reinforced_lib/exts/wifi/ieee_802_11_ccod.py +++ b/examples/ns-3-ccod/ext.py @@ -8,12 +8,12 @@ class IEEE_802_11_CCOD(BaseExt): """ - The IEEE 802.11 extension for the CCOD algorithm [3]_. Provides the preprocessed history + The IEEE 802.11 extension for the CCOD algorithm [1]_. Provides the preprocessed history of the transmission failure probability and default parameters for DRL agents. References ---------- - .. [3] W. Wydmański and S. Szott, "Contention Window Optimization in IEEE 802.11ax + .. [1] W. Wydmański and S. Szott, "Contention Window Optimization in IEEE 802.11ax Networks with Deep Reinforcement Learning," 2021 IEEE Wireless Communications and Networking Conference (WCNC), 2021. https://doi.org/10.1109/WCNC49053.2021.9417575 """ diff --git a/examples/ns-3-ccod/main.py b/examples/ns-3-ccod/main.py index 06aadab..1014d63 100644 --- a/examples/ns-3-ccod/main.py +++ b/examples/ns-3-ccod/main.py @@ -13,10 +13,11 @@ import optax from chex import Array +from ext import IEEE_802_11_CCOD from py_interface import * + from reinforced_lib import RLib from reinforced_lib.agents.deep import DQN, DDPG -from reinforced_lib.exts.wifi import IEEE_802_11_CCOD from reinforced_lib.logs import SourceType, TensorboardLogger diff --git a/reinforced_lib/exts/wifi/ieee_802_11_ax_ra.py b/examples/ns-3-ra/ext.py similarity index 97% rename from reinforced_lib/exts/wifi/ieee_802_11_ax_ra.py rename to examples/ns-3-ra/ext.py index b69765a..1c1b294 100644 --- a/reinforced_lib/exts/wifi/ieee_802_11_ax_ra.py +++ b/examples/ns-3-ra/ext.py @@ -6,14 +6,14 @@ class IEEE_802_11_ax_RA(BaseExt): """ - The IEEE 802.11ax [2]_ extension. Provides the data rates (in Mb/s) for consecutive modulation and coding + The IEEE 802.11ax [1]_ extension. Provides the data rates (in Mb/s) for consecutive modulation and coding schemes (MCS), a reward calculated as the approximate throughput, environment state as a vector, and default parameters for MAB and DRL agents. This extension is adapted to the IEEE 802.11ax network with the following settings: the guard interval is equal to 3200 ns, channel width is 20 MHz, 1 spatial stream is used. References ---------- - .. [2] "IEEE Standard for Information Technology--Telecommunications and Information Exchange between Systems + .. [1] "IEEE Standard for Information Technology--Telecommunications and Information Exchange between Systems Local and Metropolitan Area Networks--Specific Requirements Part 11: Wireless LAN Medium Access Control (MAC) and Physical Layer (PHY) Specifications Amendment 1: Enhancements for High-Efficiency WLAN," in IEEE Std 802.11ax-2021 (Amendment to IEEE Std 802.11-2020) , vol., no., pp.1-767, diff --git a/examples/ns-3-ra/main.py b/examples/ns-3-ra/main.py index aec7e66..53a7c4e 100644 --- a/examples/ns-3-ra/main.py +++ b/examples/ns-3-ra/main.py @@ -1,12 +1,12 @@ from argparse import ArgumentParser from ctypes import * +from ext import IEEE_802_11_ax_RA +from particle_filter import ParticleFilter from py_interface import * from reinforced_lib import RLib from reinforced_lib.agents.mab import * -from reinforced_lib.agents.wifi import * -from reinforced_lib.exts.wifi import IEEE_802_11_ax_RA class Env(Structure): @@ -154,7 +154,7 @@ def run( 'Softmax': {'lr': 2.048, 'alpha': 0.016, 'tau': 4.0, 'multiplier': 0.01}, 'ThompsonSampling': {'decay': 2.0}, 'UCB': {'c': 16.0, 'gamma': 0.996}, - 'ParticleFilter': {'scale': 4.0, 'num_particles': 900} + 'ParticleFilter': {'scale': 4.0, 'particles_num': 900} } run(args, args.pop('ns3Path'), args.pop('mempoolKey'), agent_type[agent], default_params[agent], args.pop('seed')) diff --git a/reinforced_lib/agents/wifi/particle_filter.py b/examples/ns-3-ra/particle_filter.py similarity index 100% rename from reinforced_lib/agents/wifi/particle_filter.py rename to examples/ns-3-ra/particle_filter.py diff --git a/examples/ra-sim/ext.py b/examples/ra-sim/ext.py new file mode 100644 index 0000000..1c1b294 --- /dev/null +++ b/examples/ra-sim/ext.py @@ -0,0 +1,128 @@ +import gymnasium as gym +import numpy as np + +from reinforced_lib.exts import BaseExt, observation, parameter + + +class IEEE_802_11_ax_RA(BaseExt): + """ + The IEEE 802.11ax [1]_ extension. Provides the data rates (in Mb/s) for consecutive modulation and coding + schemes (MCS), a reward calculated as the approximate throughput, environment state as a vector, and + default parameters for MAB and DRL agents. This extension is adapted to the IEEE 802.11ax network with the + following settings: the guard interval is equal to 3200 ns, channel width is 20 MHz, 1 spatial stream is used. + + References + ---------- + .. [1] "IEEE Standard for Information Technology--Telecommunications and Information Exchange between Systems + Local and Metropolitan Area Networks--Specific Requirements Part 11: Wireless LAN Medium Access Control + (MAC) and Physical Layer (PHY) Specifications Amendment 1: Enhancements for High-Efficiency WLAN," + in IEEE Std 802.11ax-2021 (Amendment to IEEE Std 802.11-2020) , vol., no., pp.1-767, + 19 May 2021, doi: 10.1109/IEEESTD.2021.9442429. + """ + + def __init__(self) -> None: + super().__init__() + self.last_time = 0.0 + + observation_space = gym.spaces.Dict({ + 'time': gym.spaces.Box(0.0, np.inf, (1,)), + 'n_successful': gym.spaces.Box(0, np.inf, (1,), np.int32), + 'n_failed': gym.spaces.Box(0, np.inf, (1,), np.int32), + 'n_wifi': gym.spaces.Box(1, np.inf, (1,), np.int32), + 'power': gym.spaces.Box(-np.inf, np.inf, (1,)), + 'cw': gym.spaces.Discrete(32767) + }) + + _wifi_modes_rates = np.array([ + 7.3, + 14.6, + 21.9, + 29.3, + 43.9, + 58.5, + 65.8, + 73.1, + 87.8, + 97.5, + 109.7, + 121.9 + ], dtype=np.float32) + + @observation(observation_type=gym.spaces.Box(0.0, np.inf, (len(_wifi_modes_rates),))) + def rates(self, *args, **kwargs) -> np.ndarray: + return self._wifi_modes_rates + + @observation(observation_type=gym.spaces.Box(-np.inf, np.inf, (len(_wifi_modes_rates),))) + def context(self, *args, **kwargs) -> np.ndarray: + return self.rates() + + @observation(observation_type=gym.spaces.Box(-np.inf, np.inf, (1,))) + def reward(self, action: int, n_successful: int, n_failed: int, *args, **kwargs) -> float: + if n_successful + n_failed > 0: + return self._wifi_modes_rates[action] * n_successful / (n_successful + n_failed) + else: + return 0.0 + + @observation(observation_type=gym.spaces.Box(0.0, np.inf, (1,))) + def delta_time(self, time: float, *args, **kwargs) -> float: + delta_time = time - self.last_time + self.last_time = time + return delta_time + + @observation(observation_type=gym.spaces.Box(-np.inf, np.inf, (6,))) + def env_state( + self, + time: float, + n_successful: int, + n_failed: int, + n_wifi: int, + power: float, + cw: int, + *args, + **kwargs + ) -> np.ndarray: + return np.array([self.delta_time(time), n_successful, n_failed, n_wifi, power, cw], dtype=np.float32) + + @observation(observation_type=gym.spaces.MultiBinary(1)) + def terminal(self, *args, **kwargs) -> bool: + return False + + @parameter(parameter_type=gym.spaces.Box(1, np.inf, (1,), np.int32)) + def n_mcs(self) -> int: + return len(self._wifi_modes_rates) + + @parameter(parameter_type=gym.spaces.Box(1, np.inf, (1,), np.int32)) + def n_arms(self) -> int: + return self.n_mcs() + + @parameter(parameter_type=gym.spaces.Box(-np.inf, np.inf, (1,))) + def default_power(self) -> float: + return 16.0206 + + @parameter(parameter_type=gym.spaces.Box(-np.inf, np.inf, (1,))) + def min_reward(self) -> float: + return 0 + + @parameter(parameter_type=gym.spaces.Box(-np.inf, np.inf, (1,))) + def max_reward(self) -> int: + return self._wifi_modes_rates.max() + + @parameter(parameter_type=gym.spaces.Sequence(gym.spaces.Box(1, np.inf, (1,), np.int32))) + def obs_space_shape(self) -> tuple: + return tuple((6,)) + + @parameter(parameter_type=gym.spaces.Sequence(gym.spaces.Box(1, np.inf, (1,), np.int32))) + def act_space_shape(self) -> tuple: + return tuple((1,)) + + @parameter(parameter_type=gym.spaces.Box(1, np.inf, (1,), np.int32)) + def act_space_size(self) -> int: + return 12 + + @parameter(parameter_type=gym.spaces.Sequence(gym.spaces.Box(-np.inf, np.inf))) + def min_action(self) -> tuple: + return 0 + + @parameter(parameter_type=gym.spaces.Sequence(gym.spaces.Box(-np.inf, np.inf))) + def max_action(self) -> tuple: + return 11 diff --git a/examples/ra-sim/main.py b/examples/ra-sim/main.py index 7a3bdfe..69a24db 100644 --- a/examples/ra-sim/main.py +++ b/examples/ra-sim/main.py @@ -5,10 +5,11 @@ gym.logger.set_level(40) import sim +from ext import IEEE_802_11_ax_RA from utils import ResultsManager, params_str_template, results_str_template + from reinforced_lib import RLib -from reinforced_lib.agents.mab import ThompsonSampling -from reinforced_lib.exts.wifi import IEEE_802_11_ax_RA +from reinforced_lib.agents.mab import * def run( diff --git a/examples/ra-sim/utils.py b/examples/ra-sim/utils.py index afd5463..5d0f5c0 100644 --- a/examples/ra-sim/utils.py +++ b/examples/ra-sim/utils.py @@ -1,4 +1,4 @@ -from reinforced_lib.exts.wifi import IEEE_802_11_ax_RA +from ext import IEEE_802_11_ax_RA class ResultsManager: diff --git a/examples/recommender_system/main.py b/examples/recommender_system/main.py index 7a402ce..88e4853 100644 --- a/examples/recommender_system/main.py +++ b/examples/recommender_system/main.py @@ -3,6 +3,7 @@ import env from ext import RecommenderSystemExt + from reinforced_lib import RLib from reinforced_lib.agents.mab import EGreedy from reinforced_lib.logs import PlotsLogger, SourceType diff --git a/reinforced_lib/agents/wifi/__init__.py b/reinforced_lib/agents/wifi/__init__.py deleted file mode 100644 index 942cebb..0000000 --- a/reinforced_lib/agents/wifi/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from reinforced_lib.agents.wifi.particle_filter import ParticleFilter diff --git a/reinforced_lib/exts/wifi/__init__.py b/reinforced_lib/exts/wifi/__init__.py deleted file mode 100644 index dad5377..0000000 --- a/reinforced_lib/exts/wifi/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from reinforced_lib.exts.wifi.ieee_802_11_ax_ra import IEEE_802_11_ax_RA -from reinforced_lib.exts.wifi.ieee_802_11_ccod import IEEE_802_11_CCOD diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 7dcd2b9..a30e0b9 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,7 +1,7 @@ chex~=0.1.81 cloudpickle~=2.2.1 -dm-haiku~=0.0.9 -gymnasium~=0.28.1 +dm-haiku~=0.0.10 +gymnasium~=0.29.0 jax~=0.4.13 jaxlib~=0.4.13 lz4~=4.3.2