diff --git a/doc/_notebooks/cartpole/a2c.ipynb b/doc/_notebooks/cartpole/a2c.ipynb
index c8a1159..c9f7f3a 100644
--- a/doc/_notebooks/cartpole/a2c.ipynb
+++ b/doc/_notebooks/cartpole/a2c.ipynb
@@ -19,6 +19,17 @@
     "%tensorboard --logdir ./data/tensorboard"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run this cell to fix rendering errors.\n",
+    "import os\n",
+    "os.environ['SDL_VIDEODRIVER'] = 'dummy'"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
diff --git a/doc/_notebooks/cartpole/dqn.ipynb b/doc/_notebooks/cartpole/dqn.ipynb
index 38649ad..aef21a0 100644
--- a/doc/_notebooks/cartpole/dqn.ipynb
+++ b/doc/_notebooks/cartpole/dqn.ipynb
@@ -19,6 +19,17 @@
     "%tensorboard --logdir ./data/tensorboard"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run this cell to fix rendering errors.\n",
+    "import os\n",
+    "os.environ['SDL_VIDEODRIVER'] = 'dummy'"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
diff --git a/doc/_notebooks/cartpole/iqn.ipynb b/doc/_notebooks/cartpole/iqn.ipynb
index 18c1b7c..4292f95 100644
--- a/doc/_notebooks/cartpole/iqn.ipynb
+++ b/doc/_notebooks/cartpole/iqn.ipynb
@@ -19,6 +19,17 @@
     "%tensorboard --logdir ./data/tensorboard"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run this cell to fix rendering errors.\n",
+    "import os\n",
+    "os.environ['SDL_VIDEODRIVER'] = 'dummy'"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
diff --git a/doc/_notebooks/cartpole/model_based.ipynb b/doc/_notebooks/cartpole/model_based.ipynb
index 3a7f770..059b5fd 100644
--- a/doc/_notebooks/cartpole/model_based.ipynb
+++ b/doc/_notebooks/cartpole/model_based.ipynb
@@ -19,6 +19,17 @@
     "%tensorboard --logdir ./data/tensorboard"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run this cell to fix rendering errors.\n",
+    "import os\n",
+    "os.environ['SDL_VIDEODRIVER'] = 'dummy'"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
diff --git a/doc/_notebooks/pendulum/ddpg.ipynb b/doc/_notebooks/pendulum/ddpg.ipynb
index 9ddb11e..40777b3 100644
--- a/doc/_notebooks/pendulum/ddpg.ipynb
+++ b/doc/_notebooks/pendulum/ddpg.ipynb
@@ -19,6 +19,17 @@
     "%tensorboard --logdir ./data/tensorboard"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run this cell to fix rendering errors.\n",
+    "import os\n",
+    "os.environ['SDL_VIDEODRIVER'] = 'dummy'"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
diff --git a/doc/_notebooks/pendulum/dsac.ipynb b/doc/_notebooks/pendulum/dsac.ipynb
index 94431aa..a71d272 100644
--- a/doc/_notebooks/pendulum/dsac.ipynb
+++ b/doc/_notebooks/pendulum/dsac.ipynb
@@ -19,6 +19,17 @@
     "%tensorboard --logdir ./data/tensorboard"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run this cell to fix rendering errors.\n",
+    "import os\n",
+    "os.environ['SDL_VIDEODRIVER'] = 'dummy'"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -189,6 +200,10 @@
    "version": "3.8.2"
   }
  },
+ "colab": {
+  "name": "dsac.ipynb",
+  "provenance": []
+ },
  "nbformat": 4,
  "nbformat_minor": 2
 }
\ No newline at end of file
diff --git a/doc/_notebooks/pendulum/ppo.ipynb b/doc/_notebooks/pendulum/ppo.ipynb
index 1f9534e..e7ff713 100644
--- a/doc/_notebooks/pendulum/ppo.ipynb
+++ b/doc/_notebooks/pendulum/ppo.ipynb
@@ -19,6 +19,17 @@
     "%tensorboard --logdir ./data/tensorboard"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run this cell to fix rendering errors.\n",
+    "import os\n",
+    "os.environ['SDL_VIDEODRIVER'] = 'dummy'"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
diff --git a/doc/_notebooks/pendulum/sac.ipynb b/doc/_notebooks/pendulum/sac.ipynb
index 5cef601..83a1800 100644
--- a/doc/_notebooks/pendulum/sac.ipynb
+++ b/doc/_notebooks/pendulum/sac.ipynb
@@ -19,6 +19,17 @@
     "%tensorboard --logdir ./data/tensorboard"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run this cell to fix rendering errors.\n",
+    "import os\n",
+    "os.environ['SDL_VIDEODRIVER'] = 'dummy'"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
diff --git a/doc/_notebooks/pendulum/td3.ipynb b/doc/_notebooks/pendulum/td3.ipynb
index da15792..d6f5ff5 100644
--- a/doc/_notebooks/pendulum/td3.ipynb
+++ b/doc/_notebooks/pendulum/td3.ipynb
@@ -19,6 +19,17 @@
     "%tensorboard --logdir ./data/tensorboard"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run this cell to fix rendering errors.\n",
+    "import os\n",
+    "os.environ['SDL_VIDEODRIVER'] = 'dummy'"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
diff --git a/doc/_notebooks/pendulum/td4.ipynb b/doc/_notebooks/pendulum/td4.ipynb
index c1b803d..b1331c9 100644
--- a/doc/_notebooks/pendulum/td4.ipynb
+++ b/doc/_notebooks/pendulum/td4.ipynb
@@ -19,6 +19,17 @@
     "%tensorboard --logdir ./data/tensorboard"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run this cell to fix rendering errors.\n",
+    "import os\n",
+    "os.environ['SDL_VIDEODRIVER'] = 'dummy'"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -181,6 +192,10 @@
    "version": "3.8.2"
   }
  },
+ "colab": {
+  "name": "td4.ipynb",
+  "provenance": []
+ },
  "nbformat": 4,
  "nbformat_minor": 2
 }
\ No newline at end of file
diff --git a/doc/create_notebooks.py b/doc/create_notebooks.py
index be2c9f2..f3b5681 100755
--- a/doc/create_notebooks.py
+++ b/doc/create_notebooks.py
@@ -55,6 +55,18 @@
     "nbformat_minor": 2
 }
 
+sdl_videodriver_cell = {
+    "cell_type": "code",
+    "execution_count": None,
+    "metadata": {},
+    "outputs": [],
+    "source": [
+        "# Run this cell to fix rendering errors.\n",
+        "import os\n",
+        "os.environ['SDL_VIDEODRIVER'] = 'dummy'",
+    ]
+}
+
 tensorboard_cell = {
     "cell_type": "code",
     "execution_count": None,
@@ -83,7 +95,9 @@
         with open(f_in) as r, open(f'{f_out}', 'w') as w:
             lines = list(r)
             nb['colab']['name'] = os.path.split(f_out)[1]
-            nb['cells'][-1]['source'] = lines  # the actual code
+            nb['cells'][-1]['source'] = lines
+            if any(("CartPole-v" in line or "Pendulum-v" in line) for line in lines):
+                nb['cells'].insert(1, sdl_videodriver_cell)
             if any("tensorboard_dir=" in line for line in lines):
                 nb['cells'].insert(1, tensorboard_cell)
             if 'atari' in f_in: