IGNITE-19950 Creation of cache dump implemented

modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java

@@ -419,6 +419,7 @@ public void testGetAllView() throws Exception {
             new TreeSet<>(Arrays.asList(
                 "SYS.METRICS",
                 "SYS.SERVICES",
+                "SYS.SNAPSHOT",
                 "SYS.CACHE_GROUPS",
                 "SYS.CACHES",
                 "SYS.TASKS",
@@ -896,6 +897,13 @@ public void testGetAllColumns() throws Exception {
                 "SYS.SERVICES.STATICALLY_CONFIGURED.null",
                 "SYS.SERVICES.SERVICE_ID.null",
                 "SYS.SERVICES.TOPOLOGY_SNAPSHOT.null",
+                "SYS.SNAPSHOT.BASELINE_NODES.null",
+                "SYS.SNAPSHOT.CACHE_GROUPS.null",
+                "SYS.SNAPSHOT.CONSISTENT_ID.null",
+                "SYS.SNAPSHOT.INCREMENT_INDEX.null",
+                "SYS.SNAPSHOT.NAME.null",
+                "SYS.SNAPSHOT.SNAPSHOT_RECORD_SEGMENT.null",
+                "SYS.SNAPSHOT.TYPE.null",
                 "SYS.TASKS.AFFINITY_CACHE_NAME.null",
                 "SYS.TASKS.INTERNAL.null",
                 "SYS.TASKS.END_TIME.null",

modules/core/src/main/java/org/apache/ignite/IgniteSnapshot.java

@@ -34,6 +34,8 @@
  * <li>Concurrent updates from {@link IgniteDataStreamer} with default {@link IgniteDataStreamer#allowOverwrite()}
  * setting (false) into a persistent cache can cause that cache data stored inconsistent.</li>
  * </ul>
+ * You can create cache dump, also. Cache dump is consistent files consist of entry by entry cache content.
+ * This type of snapshots supports in-memory caches.
  */
 public interface IgniteSnapshot {
     /**
@@ -61,6 +63,16 @@ public interface IgniteSnapshot {
      */
     public IgniteFuture<Void> cancelSnapshot(String name);
 
+    /**
+     * Creates cache groups dump.
+     * Dump is consistent entry by entry backup of cache group content.
+     * In-memory cache groups also supported.
+     *
+     * @param name Dump name.
+     * @return Future which will be completed when dump ends.
+     */
+    public IgniteFuture<Void> createDump(String name);
+
     /**
      * Restore cache group(s) from the snapshot.
      * <p>

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java

@@ -79,6 +79,7 @@
 import org.apache.ignite.internal.processors.cache.dr.GridCacheDrManager;
 import org.apache.ignite.internal.processors.cache.jta.CacheJtaManagerAdapter;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.DumpEntryChangeListener;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryManager;
 import org.apache.ignite.internal.processors.cache.store.CacheStoreManager;
@@ -114,6 +115,7 @@
 import org.apache.ignite.plugin.security.SecurityException;
 import org.apache.ignite.plugin.security.SecurityPermission;
 import org.jetbrains.annotations.Nullable;
+
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISABLE_TRIGGERING_CACHE_INTERCEPTOR_ON_CONFLICT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_READ_LOAD_BALANCING;
 import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
@@ -283,6 +285,9 @@ public class GridCacheContext<K, V> implements Externalizable {
     /** Recovery mode flag. */
     private volatile boolean recoveryMode;
 
+    /** Dump callback. */
+    private volatile DumpEntryChangeListener dumpLsnr;
+
     /** */
     private final boolean disableTriggeringCacheInterceptorOnConflict =
         Boolean.parseBoolean(System.getProperty(IGNITE_DISABLE_TRIGGERING_CACHE_INTERCEPTOR_ON_CONFLICT, "false"));
@@ -2348,6 +2353,19 @@ public AtomicReference<IgniteInternalFuture<Boolean>> lastRemoveAllJobFut() {
         return lastRmvAllJobFut;
     }
 
+    /** */
+    public DumpEntryChangeListener dumpListener() {
+        return dumpLsnr;
+    }
+
+    /** */
+    public void dumpListener(DumpEntryChangeListener dumpEntryChangeLsnr) {
+        assert this.dumpLsnr == null || dumpEntryChangeLsnr == null;
+        assert cacheType == CacheType.USER;
+
+        this.dumpLsnr = dumpEntryChangeLsnr;
+    }
+
     /** {@inheritDoc} */
     @Override public void writeExternal(ObjectOutput out) throws IOException {
         U.writeString(out, igniteInstanceName());

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java

@@ -66,6 +66,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
 import org.apache.ignite.internal.processors.cache.persistence.StorageException;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.DumpEntryChangeListener;
 import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryListener;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
@@ -1542,6 +1543,11 @@ else if (interceptorVal != val0)
             assert ttl >= 0 : ttl;
             assert expireTime >= 0 : expireTime;
 
+            DumpEntryChangeListener dumpLsnr = cctx.dumpListener();
+
+            if (dumpLsnr != null)
+                dumpLsnr.beforeChange(cctx, key, old, extras == null ? CU.EXPIRE_TIME_ETERNAL : extras.expireTime(), ver);
+
             // Detach value before index update.
             val = cctx.kernalContext().cacheObjects().prepareForCache(val, cctx);
 
@@ -1735,6 +1741,11 @@ protected Object keyValue(boolean cpy) {
                 }
             }
 
+            DumpEntryChangeListener dumpLsnr = cctx.dumpListener();
+
+            if (dumpLsnr != null)
+                dumpLsnr.beforeChange(cctx, key, old, extras == null ? CU.EXPIRE_TIME_ETERNAL : extras.expireTime(), ver);
+
             removeValue();
 
             update(null, 0, 0, newVer, true);
@@ -3734,6 +3745,11 @@ private boolean onExpired(CacheObject expiredVal, GridCacheVersion obsoleteVer)
         if (mvccExtras() != null)
             return false;
 
+        DumpEntryChangeListener dumpLsnr = cctx.dumpListener();
+
+        if (dumpLsnr != null)
+            dumpLsnr.beforeChange(cctx, key, expiredVal, extras == null ? CU.TTL_MINIMUM : extras.expireTime(), ver);
+
         if (cctx.deferredDelete() && !detached() && !isInternal()) {
             if (!deletedUnlocked() && !isStartVersion()) {
                 update(null, 0L, 0L, ver, true);
@@ -6143,6 +6159,18 @@ else if (interceptorVal != updated0) {
                 }
             }
 
+            DumpEntryChangeListener dumpLsnr = cctx.dumpListener();
+
+            if (dumpLsnr != null) {
+                dumpLsnr.beforeChange(
+                    cctx,
+                    entry.key,
+                    oldVal,
+                    entry.extras == null ? CU.EXPIRE_TIME_ETERNAL : entry.extras.expireTime(),
+                    entry.ver
+                );
+            }
+
             updated = cctx.kernalContext().cacheObjects().prepareForCache(updated, cctx);
 
             if (writeThrough)
@@ -6246,6 +6274,18 @@ private void remove(@Nullable GridCacheVersionConflictContext<?, ?> conflictCtx,
                 }
             }
 
+            DumpEntryChangeListener dumpLsnr = cctx.dumpListener();
+
+            if (dumpLsnr != null) {
+                dumpLsnr.beforeChange(
+                    cctx,
+                    entry.key,
+                    oldVal,
+                    entry.extras == null ? CU.EXPIRE_TIME_ETERNAL : entry.extras.expireTime(),
+                    entry.ver
+                );
+            }
+
             if (writeThrough)
                 // Must persist inside synchronization in non-tx mode.
                 cctx.store().remove(null, entry.key);

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridLocalConfigManager.java

@@ -541,8 +541,12 @@ private boolean inMemoryCdcCache(CacheConfiguration<?, ?> cfg) {
     public File cacheConfigurationFile(CacheConfiguration<?, ?> ccfg) {
         File cacheWorkDir = cacheWorkDir(ccfg);
 
-        return ccfg.getGroupName() == null ? new File(cacheWorkDir, CACHE_DATA_FILENAME) :
-            new File(cacheWorkDir, ccfg.getName() + CACHE_DATA_FILENAME);
+        return new File(cacheWorkDir, cacheDataFilename(ccfg));
+    }
+
+    /** @return Name of cache data filename. */
+    public static String cacheDataFilename(CacheConfiguration<?, ?> ccfg) {
+        return ccfg.getGroupName() == null ? CACHE_DATA_FILENAME : (ccfg.getName() + CACHE_DATA_FILENAME);
     }
 
     /**

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java

@@ -82,35 +82,40 @@ class BinaryMetadataFileStore {
      * @param log Logger.
      * @param binaryMetadataFileStoreDir Path to binary metadata store configured by user, should include binary_meta
      * and consistentId.
+     * @param forceEnabled If {@code true} then will write files even if persistence and CDC disabled.
      */
     BinaryMetadataFileStore(
         final ConcurrentMap<Integer, BinaryMetadataHolder> metadataLocCache,
         final GridKernalContext ctx,
         final IgniteLogger log,
-        final File binaryMetadataFileStoreDir
+        final File binaryMetadataFileStoreDir,
+        final boolean forceEnabled
     ) throws IgniteCheckedException {
         this.metadataLocCache = metadataLocCache;
         this.ctx = ctx;
 
-        enabled = CU.isPersistenceEnabled(ctx.config()) || CU.isCdcEnabled(ctx.config());
+        enabled = forceEnabled || CU.isPersistenceEnabled(ctx.config()) || CU.isCdcEnabled(ctx.config());
 
         this.log = log;
 
         if (!enabled)
             return;
 
-        fileIOFactory = ctx.config().getDataStorageConfiguration().getFileIOFactory();
+        DataStorageConfiguration dsCfg = ctx.config().getDataStorageConfiguration();
+
+        fileIOFactory = dsCfg == null ? new DataStorageConfiguration().getFileIOFactory() : dsCfg.getFileIOFactory();
 
         final String nodeFolderName = ctx.pdsFolderResolver().resolveFolders().folderName();
 
         if (binaryMetadataFileStoreDir != null)
             metadataDir = binaryMetadataFileStoreDir;
-        else
+        else {
             metadataDir = new File(U.resolveWorkDirectory(
                 ctx.config().getWorkDirectory(),
                 DataStorageConfiguration.DFLT_BINARY_METADATA_PATH,
                 false
             ), nodeFolderName);
+        }
 
         fixLegacyFolder(nodeFolderName);
     }

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java

@@ -261,7 +261,8 @@ public static File binaryWorkDir(String igniteWorkDir, String consId) {
                     CU.isPersistenceEnabled(ctx.config()) && binaryMetadataFileStoreDir == null ?
                         resolveBinaryWorkDir(ctx.config().getWorkDirectory(),
                             ctx.pdsFolderResolver().resolveFolders().folderName()) :
-                        binaryMetadataFileStoreDir);
+                        binaryMetadataFileStoreDir,
+                    false);
 
                 metadataFileStore.start();
             }
@@ -1019,7 +1020,9 @@ public BinaryMetadata binaryMetadata(int typeId) throws BinaryObjectException {
                 ctx,
                 log,
                 resolveBinaryWorkDir(dir.getAbsolutePath(),
-                    ctx.pdsFolderResolver().resolveFolders().folderName()));
+                    ctx.pdsFolderResolver().resolveFolders().folderName()),
+                true
+            );
 
             for (BinaryType type : types)
                 writer.mergeAndWriteMetadata(((BinaryTypeImpl)type).metadata());
@@ -1037,7 +1040,7 @@ public BinaryMetadata binaryMetadata(int typeId) throws BinaryObjectException {
         try {
             ConcurrentMap<Integer, BinaryMetadataHolder> metaCache = new ConcurrentHashMap<>();
 
-            new BinaryMetadataFileStore(metaCache, ctx, log, metadataDir)
+            new BinaryMetadataFileStore(metaCache, ctx, log, metadataDir, false)
                 .restoreMetadata();
 
             Collection<BinaryMetadata> metadata = F.viewReadOnly(metaCache.values(), BinaryMetadataHolder::metadata);
@@ -1073,7 +1076,7 @@ public BinaryMetadata binaryMetadata(int typeId) throws BinaryObjectException {
 
         ConcurrentMap<Integer, BinaryMetadataHolder> metaCache = new ConcurrentHashMap<>();
 
-        new BinaryMetadataFileStore(metaCache, ctx, log, metadataDir).restoreMetadata(typeId);
+        new BinaryMetadataFileStore(metaCache, ctx, log, metadataDir, false).restoreMetadata(typeId);
 
         addMetaLocally(typeId, metaCache.get(typeId).metadata().wrap(binaryContext()), false);
     }

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java

@@ -916,14 +916,23 @@ public static int partId(String partFileName) {
      * @return List of cache partitions in given directory.
      */
     public static List<File> cachePartitionFiles(File cacheDir) {
+        return cachePartitionFiles(cacheDir, FILE_SUFFIX);
+    }
+
+    /**
+     * @param cacheDir Cache directory to check.
+     * @param ext File extension.
+     * @return List of cache partitions in given directory.
+     */
+    public static List<File> cachePartitionFiles(File cacheDir, String ext) {
         File[] files = cacheDir.listFiles();
 
         if (files == null)
             return Collections.emptyList();
 
         return Arrays.stream(files)
             .filter(File::isFile)
-            .filter(f -> f.getName().endsWith(FILE_SUFFIX))
+            .filter(f -> f.getName().endsWith(ext))
             .collect(Collectors.toList());
     }