From 1fb6ad0a8f8a8db622c956bee727417f0f5d70ec Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Fri, 12 Apr 2024 14:24:57 +0300 Subject: [PATCH] IGNITE-22020 Add 'keepRaw' parameter to DumpReaderConfiguration (#11309) --- .../org/apache/ignite/dump/DumpReader.java | 2 +- .../ignite/dump/DumpReaderConfiguration.java | 36 +++++++- .../snapshot/dump/AbstractCacheDumpTest.java | 1 + .../dump/IgniteCacheDumpSelf2Test.java | 88 +++++++++++++++++++ 4 files changed, 122 insertions(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java b/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java index 6dcfbb7cce7e5..125d9c959fe7c 100644 --- a/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java +++ b/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java @@ -77,7 +77,7 @@ public DumpReader(DumpReaderConfiguration cfg, IgniteLogger log) { @Override public void run() { ackAsciiLogo(); - try (Dump dump = new Dump(cfg.dumpRoot(), null, cfg.keepBinary(), false, encryptionSpi(), log)) { + try (Dump dump = new Dump(cfg.dumpRoot(), null, cfg.keepBinary(), cfg.keepRaw(), encryptionSpi(), log)) { DumpConsumer cnsmr = cfg.consumer(); cnsmr.start(); diff --git a/modules/core/src/main/java/org/apache/ignite/dump/DumpReaderConfiguration.java b/modules/core/src/main/java/org/apache/ignite/dump/DumpReaderConfiguration.java index aeb62fba05b6b..14c202991b6cd 100644 --- a/modules/core/src/main/java/org/apache/ignite/dump/DumpReaderConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/dump/DumpReaderConfiguration.java @@ -19,6 +19,7 @@ import java.io.File; import java.time.Duration; +import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.internal.processors.cache.CacheObject; import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.lang.IgniteExperimental; @@ -53,9 +54,18 @@ public class DumpReaderConfiguration { /** Stop processing partitions if consumer fail to process one. */ private final boolean failFast; - /** If {@code true} then don't deserialize {@link KeyCacheObject} and {@link CacheObject}. */ + /** + * If {@code true} and if {@link #keepRaw} is {@code false} then keeps {@link DumpEntry#key()} and + * {@link DumpEntry#value()} as {@link BinaryObject}. + */ private final boolean keepBinary; + /** + * If {@code true}, doesn't deserialize cache data and keeps {@link DumpEntry#key()} as {@link KeyCacheObject} and + * {@link DumpEntry#value()} as {@link CacheObject}. If {@code true}, disables {@link #keepBinary}. + */ + private final boolean keepRaw; + /** Cache group names. */ private final String[] cacheGrpNames; @@ -70,7 +80,7 @@ public class DumpReaderConfiguration { * @param cnsmr Dump consumer. */ public DumpReaderConfiguration(File dir, DumpConsumer cnsmr) { - this(dir, cnsmr, DFLT_THREAD_CNT, DFLT_TIMEOUT, true, true, null, false, null); + this(dir, cnsmr, DFLT_THREAD_CNT, DFLT_TIMEOUT, true, true, false, null, false, null); } /** @@ -79,7 +89,11 @@ public DumpReaderConfiguration(File dir, DumpConsumer cnsmr) { * @param thCnt Count of threads to consume dumped partitions. * @param timeout Timeout of dump reader invocation. * @param failFast Stop processing partitions if consumer fail to process one. - * @param keepBinary If {@code true} then don't deserialize {@link KeyCacheObject} and {@link CacheObject}. + * @param keepBinary If {@code true} and if {@link #keepRaw} is {@code false} then keeps {@link DumpEntry#key()} and + * {@link DumpEntry#value()} as {@link BinaryObject}. + * @param keepRaw If {@code true}, doesn't deserialize cache data and keeps {@link DumpEntry#key()} as + * {@link KeyCacheObject} and {@link DumpEntry#value()} as {@link CacheObject}. If {@code true}, + * disables {@link #keepBinary}. * @param cacheGrpNames Cache group names. * @param skipCopies Skip copies. * @param encSpi Encryption SPI. @@ -91,6 +105,7 @@ public DumpReaderConfiguration( Duration timeout, boolean failFast, boolean keepBinary, + boolean keepRaw, String[] cacheGrpNames, boolean skipCopies, EncryptionSpi encSpi @@ -101,6 +116,7 @@ public DumpReaderConfiguration( this.timeout = timeout; this.failFast = failFast; this.keepBinary = keepBinary; + this.keepRaw = keepRaw; this.cacheGrpNames = cacheGrpNames; this.skipCopies = skipCopies; this.encSpi = encSpi; @@ -131,11 +147,23 @@ public boolean failFast() { return failFast; } - /** @return If {@code true} then don't deserialize {@link KeyCacheObject} and {@link CacheObject}. */ + /** + * Actual only if {@link #keepRaw} is {@code false}. + * + * @return {@code True} if {@link DumpEntry#key()} and {@link DumpEntry#value()} are kept as {@link BinaryObject}. + */ public boolean keepBinary() { return keepBinary; } + /** + * @return {@code True} if {@link DumpEntry#key()} and {@link DumpEntry#value()} are kept as {@link KeyCacheObject} + * and {@link CacheObject} respectively. + */ + public boolean keepRaw() { + return keepRaw; + } + /** @return Cache group names. */ public String[] cacheGroupNames() { return cacheGrpNames; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java index 678b2d9ab938f..38bf7dfb2c0d1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java @@ -428,6 +428,7 @@ else if (!cacheName.equals(DEFAULT_CACHE_NAME)) DFLT_THREAD_CNT, DFLT_TIMEOUT, true, false, + false, cacheGrpNames, skipCopies, null diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelf2Test.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelf2Test.java index ea65e472dc0e6..67f6a928d7186 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelf2Test.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelf2Test.java @@ -34,6 +34,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiConsumer; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -41,9 +42,11 @@ import java.util.stream.IntStream; import java.util.zip.ZipInputStream; import org.apache.commons.io.IOUtils; +import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.IgniteException; +import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.binary.BinaryType; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheEntryVersion; @@ -83,6 +86,7 @@ import org.apache.ignite.internal.processors.cacheobject.UserCacheObjectImpl; import org.apache.ignite.internal.processors.dr.GridDrType; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteFuture; @@ -170,6 +174,86 @@ public class IgniteCacheDumpSelf2Test extends GridCommonAbstractTest { cleanPersistenceDir(); } + /** */ + @Test + public void testDumpRawData() throws Exception { + IgniteEx ign = startGrids(3); + + Ignite cli = startClientGrid(G.allGrids().size()); + + cli.createCache(defaultCacheConfiguration()); + + for (int i = 0; i < KEYS_CNT; ++i) + cli.cache(DEFAULT_CACHE_NAME).put(i, USER_FACTORY.apply(i)); + + cli.snapshot().createDump(DMP_NAME, null).get(); + + AtomicBoolean keepRaw = new AtomicBoolean(); + AtomicBoolean keepBinary = new AtomicBoolean(); + + DumpConsumer cnsmr = new DumpConsumer() { + @Override public void start() { + // No-op. + } + + @Override public void onMappings(Iterator mappings) { + // No-op. + } + + @Override public void onTypes(Iterator types) { + // No-op. + } + + @Override public void onCacheConfigs(Iterator caches) { + // No-op. + } + + @Override public void onPartition(int grp, int part, Iterator data) { + data.forEachRemaining(de -> { + if (keepRaw.get()) { + assertTrue(de.key() instanceof KeyCacheObject); + assertTrue(de.value() instanceof CacheObject); + } + else { + assertTrue(de.key() instanceof Integer); + + if (keepBinary.get()) + assertTrue(de.value() instanceof BinaryObject); + else + assertTrue(de.value() instanceof User); + } + }); + } + + @Override public void stop() { + // No-op. + } + }; + + for (boolean raw : Arrays.asList(true, false)) { + for (boolean binary : Arrays.asList(true, false)) { + keepRaw.set(raw); + keepBinary.set(binary); + + new DumpReader( + new DumpReaderConfiguration( + dumpDirectory(ign, DMP_NAME), + cnsmr, + DFLT_THREAD_CNT, + DFLT_TIMEOUT, + true, + keepBinary.get(), + keepRaw.get(), + null, + false, + null + ), + log + ).run(); + } + } + } + /** Checks a dump when it is created with the data streamer just after a restart. */ @Test public void testDumpAfterRestartWithStreamer() throws Exception { @@ -748,6 +832,7 @@ public void testDumpEntryConflictVersion() throws Exception { DFLT_TIMEOUT, true, false, + false, null, false, null @@ -812,6 +897,7 @@ public void testReadEncrypted() throws Exception { DFLT_TIMEOUT, true, false, + false, null, false, null @@ -840,6 +926,7 @@ public void testReadEncrypted() throws Exception { DFLT_TIMEOUT, true, false, + false, null, false, encSpi @@ -871,6 +958,7 @@ public void testReadEncrypted() throws Exception { DFLT_TIMEOUT, true, false, + false, null, false, encryptionSpi()