IGNITE-20470 [ducktests] Test for the cache dump (#10998)

modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dump/DumpUtility.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.ducktest.tests.dump;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+
+/**
+ * Application to control the cache dump operations (just create for now).
+ */
+public class DumpUtility extends IgniteAwareApplication {
+    /** {@inheritDoc} */
+    @Override public void run(JsonNode jsonNode) {
+        String cmd = jsonNode.get("cmd").asText();
+        String dumpName = jsonNode.get("dumpName").asText();
+
+        markInitialized();
+
+        switch (cmd) {
+            case "create":
+                long startTime = System.nanoTime();
+
+                ignite.snapshot().createDump(dumpName, null).get();
+
+                long resultTime = System.nanoTime() - startTime;
+
+                recordResult("DUMP_CREATE_TIME_MS", resultTime / 1_000_000);
+
+                break;
+            default:
+                throw new RuntimeException("Wrong cmd parameter for the dump control utility: '" + cmd + "'");
+        }
+
+        markFinished();
+    }
+}

modules/ducktests/tests/ignitetest/services/utils/control_utility.py

@@ -220,6 +220,17 @@ def snapshot_create(self, snapshot_name: str, timeout_sec: int = 60):
         raise TimeoutError(f'Failed to wait for the snapshot operation to complete: '
                            f'snapshot_name={snapshot_name} in {timeout_sec} seconds.')
 
+    def snapshot_check(self, snapshot_name: str):
+        """
+        Check snapshot.
+        :param snapshot_name: Name of snapshot.
+        """
+        res = self.__run(f"--snapshot check {snapshot_name}")
+
+        assert "The check procedure has finished, no conflicts have been found." in res
+
+        return res
+
     def start_performance_statistics(self):
         """
         Start performance statistics collecting in the cluster.
@@ -425,10 +436,10 @@ def __form_cmd(self, node_ip, cmd):
             auth = f" --user {self.username} --password {self.password} "
 
         return "%s %s" % \
-               (envs_to_exports(self.__envs()),
+               (envs_to_exports(self.envs()),
                 self._cluster.script(f"{self.BASE_COMMAND} --host {node_ip} {cmd} {ssl} {auth}"))
 
-    def __envs(self):
+    def envs(self):
         """
         :return: environment set.
         """

modules/ducktests/tests/ignitetest/services/utils/dump_utility.py

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ignitetest.services.ignite_app import IgniteApplicationService
+
+
+class DumpUtility:
+    """
+    Control the cache dump operations.
+    """
+    def __init__(self, test_context, cluster):
+        self.cluster = cluster
+
+        self.app = IgniteApplicationService(
+            test_context,
+            cluster.config._replace(client_mode=True),
+            java_class_name="org.apache.ignite.internal.ducktest.tests.dump.DumpUtility"
+        )
+
+    def create(self, dump_name):
+        """
+        Create cache dump.
+        :param dump_name: Name of the dump.
+        """
+        self.app.params = {
+            "cmd": "create",
+            "dumpName": dump_name
+        }
+
+        self.app.start(clean=False)
+
+        self.app.wait()
+
+        dump_create_time_ms = self.app.extract_result("DUMP_CREATE_TIME_MS")
+
+        return int(dump_create_time_ms) if dump_create_time_ms != "" else None

modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py

@@ -581,6 +581,14 @@ def alive_nodes(self) -> list:
         """
         return [node for node in self.nodes if self.alive(node)]
 
+    @staticmethod
+    def get_file_size(node, file):
+        out = IgniteAwareService.exec_command(node, f'du -s --block-size=1 {file}')
+
+        data = out.split("\t")
+
+        return int(data[0])
+
 
 def node_failed_event_pattern(failed_node_id=None):
     """Failed node pattern in log."""

modules/ducktests/tests/ignitetest/tests/dump_test.py

@@ -0,0 +1,123 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Module contains cache dump tests.
+"""
+import os
+import re
+from ducktape.mark import defaults
+
+from ignitetest.services.ignite import IgniteService
+from ignitetest.services.utils.control_utility import ControlUtility
+from ignitetest.services.utils.dump_utility import DumpUtility
+from ignitetest.services.utils.ignite_aware import IgniteAwareService
+from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, DataStorageConfiguration
+from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration
+from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster
+from ignitetest.tests.util import DataGenerationParams, preload_data
+from ignitetest.utils import cluster, ignite_versions
+from ignitetest.utils.ignite_test import IgniteTest
+from ignitetest.utils.version import IgniteVersion, DEV_BRANCH
+
+DUMP_NAME = "test_dump"
+CACHE_NAME = "test-cache"
+
+
+class DumpTest(IgniteTest):
+    """
+    Test cache dump.
+    """
+
+    @cluster(num_nodes=5)
+    @ignite_versions(str(DEV_BRANCH))
+    @defaults(nodes=[1, 3], backups=[1], cache_count=[1], entry_count=[50_000], entry_size=[1024], preloaders=[1])
+    def dump_test(self, ignite_version, nodes, backups, cache_count, entry_count, entry_size, preloaders):
+        """
+        Basic dump test.
+        """
+        data_gen_params = DataGenerationParams(backups=backups, cache_count=cache_count, entry_count=entry_count,
+                                               entry_size=entry_size, preloaders=preloaders)
+
+        ignite_config = IgniteConfiguration(
+            version=IgniteVersion(ignite_version),
+            data_storage=DataStorageConfiguration(
+                checkpoint_frequency=5000,
+                default=DataRegionConfiguration(
+                    persistence_enabled=True,
+                    max_size=data_gen_params.data_region_max_size
+                )
+            ),
+            metric_exporters={'org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi'}
+        )
+
+        ignite = IgniteService(self.test_context, ignite_config, num_nodes=nodes)
+        ignite.start()
+
+        control_utility = ControlUtility(ignite)
+        control_utility.activate()
+
+        preload_data(
+            self.test_context,
+            ignite.config._replace(client_mode=True, discovery_spi=from_ignite_cluster(ignite)),
+            data_gen_params=data_gen_params)
+
+        result = self.get_data_region_size(ignite)
+
+        result.update(self.create_dump(ignite))
+
+        result.update(self.check_dump(ignite))
+
+        return result
+
+    def create_dump(self, ignite):
+        dump_utility = DumpUtility(self.test_context, ignite)
+
+        dump_create_time_ms = dump_utility.create(DUMP_NAME)
+
+        dump_size = {}
+        for node in ignite.nodes:
+            dump_size[node.consistent_id] = IgniteAwareService.get_file_size(
+                node, os.path.join(ignite.snapshots_dir, DUMP_NAME))
+
+        return {
+            "dump_create_time_ms": dump_create_time_ms,
+            "dump_size": dump_size
+        }
+
+    @staticmethod
+    def check_dump(ignite):
+        control_utility = ControlUtility(ignite)
+
+        output = control_utility.snapshot_check(DUMP_NAME)
+
+        pattern = re.compile("Execution time: (?P<time_ms>\\d+) ms")
+        match = pattern.search(output)
+
+        return {
+            "dump_check_time_ms": int(match.group("time_ms")) if match else None
+        }
+
+    @staticmethod
+    def get_data_region_size(ignite):
+        data_region_size = {}
+
+        for node in ignite.nodes:
+            mbean = node.jmx_client().find_mbean('.*group=io.*name="dataregion.default"')
+            data_region_size[node.consistent_id] = int(next(mbean.TotalUsedSize))
+
+        return {
+            "data_region_size": data_region_size
+        }