diff --git a/pywayland/server/client.py b/pywayland/server/client.py
index 4386e1b..f7cd182 100644
--- a/pywayland/server/client.py
+++ b/pywayland/server/client.py
@@ -103,7 +103,7 @@ def add_destroy_listener(self, listener: Listener) -> None:
         :param listener: The listener object
         :type listener: :class:`~pywayland.server.Listener`
         """
-        assert self._ptr is not None
+        assert self._ptr is not None and listener._ptr is not None
         lib.wl_client_add_destroy_listener(self._ptr, listener._ptr)
 
     @ensure_valid
diff --git a/pywayland/server/listener.py b/pywayland/server/listener.py
index b97c67b..a9fac38 100644
--- a/pywayland/server/listener.py
+++ b/pywayland/server/listener.py
@@ -62,16 +62,18 @@ def __init__(self, function: Callable) -> None:
         self.container = ffi.new("struct wl_listener_container *")  # type: ignore[assignment]
         self.container.handle = self._handle
 
-        self._ptr = ffi.addressof(self.container.destroy_listener)
+        self._ptr: ffi.ListenerCData | None = ffi.addressof(
+            self.container.destroy_listener
+        )
         self._ptr.notify = lib.notify_func
         self._notify = function
         self._signal = None
 
     def remove(self) -> None:
         """Remove the listener"""
-        if self._ptr.link != ffi.NULL:
+        if self._ptr and self._ptr.link != ffi.NULL:
             lib.wl_list_remove(ffi.addressof(self._ptr.link))
-            self.link = None
+            self._ptr = None
 
 
 class Signal: