diff --git a/python/xoscar/backends/pool.py b/python/xoscar/backends/pool.py
index 6e3859d8..52cb664e 100644
--- a/python/xoscar/backends/pool.py
+++ b/python/xoscar/backends/pool.py
@@ -41,7 +41,7 @@
     ServerClosed,
 )
 from ..metrics import init_metrics
-from ..utils import implements, register_asyncio_task_timeout_detector
+from ..utils import implements, is_zero_ip, register_asyncio_task_timeout_detector
 from .allocate_strategy import AddressSpecified, allocated_type
 from .communication import (
     Channel,
@@ -164,7 +164,10 @@ def __init__(
     ):
         # register local pool for local actor lookup.
         # The pool is weakrefed, so we don't need to unregister it.
-        register_local_pool(external_address, self)
+        if not is_zero_ip(external_address):
+            # Only register_local_pool when we listen on non-zero ip (because all-zero ip is wildcard address),
+            # avoid mistaken with another remote service listen on non-zero ip with the same port.
+            register_local_pool(external_address, self)
         self.process_index = process_index
         self.label = label
         self.external_address = external_address
diff --git a/python/xoscar/utils.py b/python/xoscar/utils.py
index 643c065d..6c60f973 100644
--- a/python/xoscar/utils.py
+++ b/python/xoscar/utils.py
@@ -480,6 +480,10 @@ def is_v6_zero_ip(ip_port_addr: str) -> bool:
     return True
 
 
+def is_zero_ip(ip_port_addr: str) -> bool:
+    return is_v4_zero_ip(ip_port_addr) or is_v6_zero_ip(ip_port_addr)
+
+
 def is_v6_ip(ip_port_addr: str) -> bool:
     arr = ip_port_addr.split("://", 1)[-1].split(":")
     return len(arr) > 1