diff --git a/CHANGES.rst b/CHANGES.rst
index a9564160cc..b0c1701727 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -338,6 +338,9 @@ New Features
 
 - Support for Scatter plots/markers in plot options. [#2193]
 
+- Allow user-loaded data to be removed from the app (in specviz and imviz)
+  so long as it is not the only user-loaded data [#2211]
+
 Cubeviz
 ^^^^^^^
 
diff --git a/jdaviz/app.py b/jdaviz/app.py
index 27cd360670..324befb3a4 100644
--- a/jdaviz/app.py
+++ b/jdaviz/app.py
@@ -1968,6 +1968,10 @@ def set_data_visibility(self, viewer_reference, data_label, visible=True, replac
                 viewer.on_limits_change()  # Trigger compass redraw
 
     def vue_data_item_remove(self, event):
+        # Hack for disappearing subsets. Temporarily add the data back to the viewer to
+        # transfer ownership of subsets
+        self.add_data_to_viewer(event['viewer_id'], event['item_name'])
+
         self.data_collection.remove(self.data_collection[event['item_name']])
 
     def vue_close_snackbar_message(self, event):
diff --git a/jdaviz/components/tooltip.vue b/jdaviz/components/tooltip.vue
index e31cb349d4..0a71d9802e 100644
--- a/jdaviz/components/tooltip.vue
+++ b/jdaviz/components/tooltip.vue
@@ -66,7 +66,7 @@ const tooltips = {
   'viewer-data-radio': 'Switch visibility to layers associated with this data entry',
   'viewer-data-enable': 'Load data entry into this viewer',
   'viewer-data-disable': 'Disable data within this viewer (will be hidden and unavailable from plugins until re-enabled)',
-  'viewer-data-delete': 'Remove data entry across entire app',
+  'viewer-data-delete': 'Remove data entry across entire app (might affect existing subsets)',
 
   'table-prev': 'Select previous row in table',
   'table-next': 'Select next row in table',
diff --git a/jdaviz/components/viewer_data_select.vue b/jdaviz/components/viewer_data_select.vue
index 7a93b0c41a..654f590584 100644
--- a/jdaviz/components/viewer_data_select.vue
+++ b/jdaviz/components/viewer_data_select.vue
@@ -50,6 +50,7 @@
             :icon="layer_icons[item.name]"
             :viewer="viewer"
             :multi_select="multi_select"
+            :n_data_entries="nDataEntries"
             @data-item-visibility="$emit('data-item-visibility', $event)"
             @data-item-unload="$emit('data-item-unload', $event)"
             @data-item-remove="$emit('data-item-remove', $event)"
@@ -79,6 +80,7 @@
               :icon="layer_icons[item.name]"
               :viewer="viewer"
               :multi_select="multi_select"
+              :n_data_entries="nDataEntries"
               @data-item-visibility="$emit('data-item-visibility', $event)"
               @data-item-remove="$emit('data-item-remove', $event)"
             ></j-viewer-data-select-item>
@@ -216,6 +218,10 @@ module.exports = {
     extraDataItems() {
       return this.$props.data_items.filter((item) => this.itemIsVisible(item, true))
     },
+    nDataEntries() {
+      // return number of data entries in the entire plugin that were NOT created by a plugin
+      return this.$props.data_items.filter((item) => item.meta.Plugin === undefined).length
+    },
   }
 };
 </script>
diff --git a/jdaviz/components/viewer_data_select_item.vue b/jdaviz/components/viewer_data_select_item.vue
index f0f74e170d..ac4e2e85f8 100644
--- a/jdaviz/components/viewer_data_select_item.vue
+++ b/jdaviz/components/viewer_data_select_item.vue
@@ -50,7 +50,7 @@
       <j-tooltip tipid='viewer-data-delete'>
         <v-btn
           icon
-          @click="$emit('data-item-remove', {item_name: item.name})"
+          @click="$emit('data-item-remove', {item_name: item.name, viewer_id: viewer.id})"
         ><v-icon>mdi-delete</v-icon></v-btn>
       </j-tooltip>
     </div>
@@ -60,7 +60,7 @@
 <script>
 
 module.exports = {
-  props: ['item', 'icon', 'multi_select', 'viewer'],
+  props: ['item', 'icon', 'multi_select', 'viewer', 'n_data_entries'],
   methods: {
     selectClicked() {
       prevVisibleState = this.visibleState
@@ -114,14 +114,15 @@ module.exports = {
         // forbid unloading the original reference cube
         // this logic might need to be generalized if supporting custom data labels
         // per-cube or renaming data labels
+        const extension = this.itemNameExtension
         if (this.$props.viewer.reference === 'flux-viewer') {
-          return this.$props.item.name.indexOf('[FLUX]') === -1
+          return ['SCI', 'FLUX'].indexOf(extension) !== -1
         } else if (this.$props.viewer.reference === 'uncert-viewer') {
-          return this.$props.item.name.indexOf('[IVAR]') === -1
+          return ['IVAR', 'ERR'].indexOf(extension) !== -1
         } else if (this.$props.viewer.reference === 'mask-viewer') {
-          return this.$props.item.name.indexOf('[MASK]') === -1
+          return ['MASK', 'DQ'].indexOf(extension) !== -1
         } else if (this.$props.viewer.reference === 'spectrum-viewer') {
-          return this.$props.item.name.indexOf('[FLUX]') === -1          
+          return ['SCI', 'FLUX'].indexOf(extension) !== -1
         }
       } else if (this.$props.viewer.config === 'specviz2d') {
         if (this.$props.viewer.reference === 'spectrum-2d-viewer') {
@@ -133,14 +134,12 @@ module.exports = {
       return true
     },
     isDeletable() {
-      // only allow deleting products from plugins.  We might want to allow some non-plugin
-      // data to also be deleted in the future, but would probably need more advanced logic
-      // to ensure essential data isn't removed that would break the app.
-      if (this.$props.item.meta.Plugin === undefined) {
-        return false
-      }
-      // for any exceptions not above, enable deleting
-      return !this.isSelected
+      isLastDataset = (this.$props.n_data_entries <= 1)
+      notSelected = !this.isSelected
+      isMosviz = this.$props.viewer.config === 'mosviz'
+      isCubeviz = this.$props.viewer.config === 'cubeviz'
+      isPluginData = !(this.$props.item.meta.Plugin === undefined)
+      return notSelected && (isPluginData || (!isLastDataset && !isMosviz && !isCubeviz))
     },
     selectTipId() {
       if (this.multi_select) {
diff --git a/jdaviz/configs/cubeviz/plugins/tests/test_data_selection.py b/jdaviz/configs/cubeviz/plugins/tests/test_data_selection.py
index e3f024c047..c49f970ddb 100644
--- a/jdaviz/configs/cubeviz/plugins/tests/test_data_selection.py
+++ b/jdaviz/configs/cubeviz/plugins/tests/test_data_selection.py
@@ -31,6 +31,7 @@ def test_data_selection(cubeviz_helper, spectrum1d_cube, tmpdir):
     assert len(fv.layers) == 2
     assert len([layer for layer in fv.layers if layer.visible]) == 2
 
-    app.vue_data_item_remove({'item_name': app.state.data_items[1]['name']})
+    app.vue_data_item_remove({'item_name': app.state.data_items[1]['name'],
+                              'viewer_id': 'cubeviz-0'})
 
     assert len(fv.layers) == 1