diff --git a/jdaviz/core/freezable_state.py b/jdaviz/core/freezable_state.py
index 0d94eb2cdf..8bb8b8b723 100644
--- a/jdaviz/core/freezable_state.py
+++ b/jdaviz/core/freezable_state.py
@@ -63,7 +63,6 @@ def _convert_units_y_limits(self, old_unit, new_unit):
 
         if old_unit != new_unit:
             if old_unit is None or new_unit is None:
-                self._reset_y_limits()
                 return
 
             x_corners = np.array([self.x_min, self.x_min, self.x_max, self.x_max])