Extended operation no_op by parameter memory_alloc for container

CHANGES.md

@@ -9,6 +9,8 @@
   the time dimension larger than one were not handled properly. This affected 
   OZONE in a monthly resolution, SEALEVEL in a monthly resolution, and several
   ICESHEETS datasets. (via updated version of xcube-cci)
+* Extended operation `no_op` by parameter `memory_alloc` for container 
+  stress testing. (#980)
 
 ## Version 3.1.3
 

cate/ops/utility.py

@@ -28,6 +28,7 @@
 All operations in this module are tagged with the ``"utility"`` tag.
 
 """
+import numpy as np
 import pandas as pd
 import xarray as xr
 from datetime import timezone
@@ -232,27 +233,46 @@ def dummy_ds(lon_dim: int = 360,
 
 @op(tags=['utility'])
 @op_input('step_duration', units='seconds')
-@op_input('error_type', value_set=['Value', 'OS', 'Memory', 'Network', 'Data Access', 'Validation'])
+@op_input('error_type', value_set=list(_ERROR_TYPES.keys()))
 def no_op(num_steps: int = 20,
           step_duration: float = 0.5,
           fail_before: bool = False,
           fail_after: bool = False,
           error_type: str = 'Value',
+          memory_alloc: str = '0',
           monitor: Monitor = Monitor.NONE) -> bool:
     """
     An operation that basically does nothing but spending configurable time.
     It may be useful for testing purposes.
 
     :param num_steps: Number of steps to iterate.
     :param step_duration: How much time to spend in each step in seconds.
-    :param fail_before: If the operation should fail before spending time doing nothing (raise a ValidationError).
-    :param fail_after: If the operation should fail after spending time doing nothing (raise a ValueError).
+    :param fail_before: If the operation should fail before spending
+        time doing nothing (raise a ValidationError).
+    :param fail_after: If the operation should fail after spending
+        time doing nothing (raise a ValueError).
     :param error_type: The type of error to raise.
+    :param memory_alloc: Memory allocation in each step, e.g., "250M", "1G".
     :param monitor: A progress monitor.
     :return: Always True
     """
     import time
-    with monitor.starting('Computing nothing', num_steps):
+
+    memory_size = 0
+    if memory_alloc:
+        unit = memory_alloc[-1].lower()
+        factors = dict(b=1, k=1000, m=1000**2, g=1000**3, t=1000**4)
+        factor = 1
+        if unit in factors:
+            factor = factors[unit]
+            memory_alloc = memory_alloc[:-1]
+        memory_size = round(factor * float(memory_alloc))
+
+    memory = []
+    message = 'Allocating memory' if memory_size else 'Computing nothing'
+    with monitor.starting(message, num_steps):
+        if memory_size:
+            memory.append(np.zeros(memory_size, dtype=np.uint8))
         if fail_before:
             error_class = _ERROR_TYPES[error_type]
             raise error_class(f'This is a test: intentionally failed with a {error_type} error'
@@ -264,6 +284,7 @@ def no_op(num_steps: int = 20,
             error_class = _ERROR_TYPES[error_type]
             raise error_class(f'Intentionally failed failed with a {error_type} error'
                               f' after {num_steps} times doing nothing.')
+
     return True
 
 

tests/ops/test_utility.py

@@ -11,7 +11,8 @@
 from cate.core.ds import NetworkError
 from cate.core.op import OP_REGISTRY
 from cate.core.types import ValidationError
-from cate.ops.utility import merge, sel, from_data_frame, identity, literal, pandas_fillna, no_op
+from cate.ops.utility import merge, sel, from_data_frame, identity, literal, \
+    pandas_fillna, no_op
 from cate.util.misc import object_to_qualified_name
 
 
@@ -75,13 +76,15 @@ def test_nominal(self):
         ds = new_ds()
 
         sel_ds = sel(ds=ds, time='2014-09-06')
-        self.assertEqual(set(sel_ds.coords.keys()), {'lon', 'lat', 'time', 'reference_time'})
+        self.assertEqual(set(sel_ds.coords.keys()),
+                         {'lon', 'lat', 'time', 'reference_time'})
         self.assertEqual(sel_ds.dims['lon'], 4)
         self.assertEqual(sel_ds.dims['lat'], 2)
         self.assertNotIn('time', sel_ds.dims)
 
         sel_ds = sel(ds=ds, point=(34.51, 10.25))
-        self.assertEqual(set(sel_ds.coords.keys()), {'lon', 'lat', 'time', 'reference_time'})
+        self.assertEqual(set(sel_ds.coords.keys()),
+                         {'lon', 'lat', 'time', 'reference_time'})
         self.assertNotIn('lon', sel_ds.dims)
         self.assertNotIn('lat', sel_ds.dims)
         self.assertEqual(sel_ds.dims['time'], 10)
@@ -95,13 +98,15 @@ def test_registered(self):
         ds = new_ds()
 
         sel_ds = reg_op(ds=ds, time='2014-09-06')
-        self.assertEqual(set(sel_ds.coords.keys()), {'lon', 'lat', 'time', 'reference_time'})
+        self.assertEqual(set(sel_ds.coords.keys()),
+                         {'lon', 'lat', 'time', 'reference_time'})
         self.assertEqual(sel_ds.dims['lon'], 4)
         self.assertEqual(sel_ds.dims['lat'], 2)
         self.assertNotIn('time', sel_ds.dims)
 
         sel_ds = reg_op(ds=ds, point=(34.51, 10.25))
-        self.assertEqual(set(sel_ds.coords.keys()), {'lon', 'lat', 'time', 'reference_time'})
+        self.assertEqual(set(sel_ds.coords.keys()),
+                         {'lon', 'lat', 'time', 'reference_time'})
         self.assertNotIn('lon', sel_ds.dims)
         self.assertNotIn('lat', sel_ds.dims)
         self.assertEqual(sel_ds.dims['time'], 10)
@@ -209,7 +214,8 @@ def test_nominal(self):
                 'B': [5, 6, 8, 7, 5, np.nan, np.nan, np.nan, 1, 2, 7, 6]}
         expected = {'A': [1, 2, 3, 3, 4, 9, 9, 9, 1, 0, 4, 6],
                     'B': [5, 6, 8, 7, 5, 5, 5, 5, 1, 2, 7, 6]}
-        time = pd.date_range('2000-01-01', freq='MS', periods=12, tz=timezone.utc)
+        time = pd.date_range('2000-01-01', freq='MS', periods=12,
+                             tz=timezone.utc)
 
         expected = pd.DataFrame(data=expected, index=time, dtype=float)
         df = pd.DataFrame(data=data, index=time, dtype=float)
@@ -234,7 +240,8 @@ def test_registered(self):
                 'B': [5, 6, 8, 7, 5, np.nan, np.nan, np.nan, 1, 2, 7, 6]}
         expected = {'A': [1, 2, 3, 3, 4, 9, 9, 9, 1, 0, 4, 6],
                     'B': [5, 6, 8, 7, 5, 5, 5, 5, 1, 2, 7, 6]}
-        time = pd.date_range('2000-01-01', freq='MS', periods=12, tz=timezone.utc)
+        time = pd.date_range('2000-01-01', freq='MS', periods=12,
+                             tz=timezone.utc)
 
         expected = pd.DataFrame(data=expected, index=time, dtype=float)
         df = pd.DataFrame(data=data, index=time, dtype=float)
@@ -255,6 +262,16 @@ def test_nominal(self):
         with self.assertRaises(ValidationError):
             no_op(step_duration=0.001, fail_after=True, error_type='????')
 
+    def test_alloc_memory(self):
+        # We actually cannot verify that it works, so here are
+        # a few smoke tests
+        no_op(step_duration=0, num_steps=3, memory_alloc='120M')
+        no_op(step_duration=0, num_steps=200, memory_alloc='124B')
+        no_op(step_duration=0, num_steps=2, memory_alloc='0.1G')
+        no_op(step_duration=0, num_steps=5, memory_alloc='0.01T')
+        with self.assertRaises(ValueError):
+            no_op(step_duration=0, num_steps=5, memory_alloc='x')
+
 
 def new_ds():
     lon = [10.1, 10.2, 10.3, 10.4]
@@ -266,8 +283,10 @@ def new_ds():
     lon_res = len(lon)
     lat_res = len(lat)
 
-    temperature = (15 + 8 * np.random.randn(lon_res, lat_res, time_res)).round(decimals=1)
-    precipitation = (10 * np.random.rand(lon_res, lat_res, time_res)).round(decimals=1)
+    temperature = (15 + 8 * np.random.randn(lon_res, lat_res, time_res)).round(
+        decimals=1)
+    precipitation = (10 * np.random.rand(lon_res, lat_res, time_res)).round(
+        decimals=1)
 
     ds = xr.Dataset({'temperature': (['lon', 'lat', 'time'], temperature),
                      'precipitation': (['lon', 'lat', 'time'], precipitation)