From b6f49f80d5778afacb1fc55bf556dcb1718553c3 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Wed, 30 Aug 2023 19:18:07 +0300 Subject: [PATCH 1/7] IGNITE-20309 Snapshot creation must fail if check throws (#10913) --- .../SnapshotPartitionsQuickVerifyHandler.java | 5 ++++ .../SnapshotPartitionsVerifyHandler.java | 14 +++++++-- .../IgniteClusterSnapshotHandlerTest.java | 30 +++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsQuickVerifyHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsQuickVerifyHandler.java index 78efc2af1099e..8bc5166e8d396 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsQuickVerifyHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsQuickVerifyHandler.java @@ -71,6 +71,11 @@ public SnapshotPartitionsQuickVerifyHandler(GridCacheSharedContext cctx) { String name, Collection>> results ) throws IgniteCheckedException { + for (SnapshotHandlerResult> result : results) { + if (result.error() != null) + throw new IgniteCheckedException(result.error()); + } + if (results.stream().anyMatch(r -> r.data() == null)) return; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java index e0e654d49ae81..3547d5edf30b2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java @@ -106,8 +106,18 @@ public SnapshotPartitionsVerifyHandler(GridCacheSharedContext cctx) { SnapshotMetadata meta = opCtx.metadata(); - Set grps = F.isEmpty(opCtx.groups()) ? new HashSet<>(meta.partitions().keySet()) : - opCtx.groups().stream().map(CU::cacheId).collect(Collectors.toSet()); + Set grps = F.isEmpty(opCtx.groups()) + ? new HashSet<>(meta.partitions().keySet()) + : opCtx.groups().stream().map(CU::cacheId).collect(Collectors.toSet()); + + if (type() == SnapshotHandlerType.CREATE) { + grps = grps.stream().filter(grp -> grp == MetaStorage.METASTORAGE_CACHE_ID || + CU.affinityNode( + cctx.localNode(), + cctx.kernalContext().cache().cacheGroupDescriptor(grp).config().getNodeFilter() + ) + ).collect(Collectors.toSet()); + } Set partFiles = new HashSet<>(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java index 1c3ae2ef5bc18..d68095da4b2a4 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java @@ -31,6 +31,7 @@ import java.util.function.Function; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; @@ -48,6 +49,7 @@ import org.junit.Test; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.SNAPSHOT_METAFILE_EXT; +import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause; /** * Snapshot custom handlers test. @@ -405,4 +407,32 @@ public void testHandlerSnapshotLocation() throws Exception { U.delete(snpDir); } } + + /** + * Test ensures that snapshot fails if some files are absent during the check. + * @see SnapshotPartitionsQuickVerifyHandler + */ + @Test + public void testHandlerExceptionFailSnapshot() throws Exception { + handlers.add(new SnapshotHandler() { + @Override public SnapshotHandlerType type() { + return SnapshotHandlerType.CREATE; + } + + @Override public Void invoke(SnapshotHandlerContext ctx) { + // Someone removes snapshot files during creation. + // In this case snapshot must fail. + U.delete(ctx.snapshotDirectory()); + + return null; + } + }); + + IgniteEx ignite = startGridsWithCache(1, CACHE_KEYS_RANGE, valueBuilder(), dfltCacheCfg); + + assertThrowsWithCause( + () -> snp(ignite).createSnapshot("must_fail", null, false, onlyPrimary).get(getTestTimeout()), + IgniteException.class + ); + } } From 44dabc06329e45238942c2a4528eae83f20111b4 Mon Sep 17 00:00:00 2001 From: liyujue <18624049226@163.com> Date: Thu, 31 Aug 2023 18:35:40 +0800 Subject: [PATCH 2/7] IGNITE-19568 Ignite Docs: change {nodeId} to {consistentId} in the snapshot docs (#10738) --- docs/_docs/snapshots/snapshots.adoc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/_docs/snapshots/snapshots.adoc b/docs/_docs/snapshots/snapshots.adoc index b40b16ce83d1c..00a0eac64cb75 100644 --- a/docs/_docs/snapshots/snapshots.adoc +++ b/docs/_docs/snapshots/snapshots.adoc @@ -224,10 +224,10 @@ The detailed procedure looks as follows: . Stop the cluster you intend to restore . Remove all files from the checkpoint `$IGNITE_HOME/work/cp` directory . Do the following on each node: - - Remove the files related to the `{nodeId}` from the `$IGNITE_HOME/work/db/binary_meta` directory. - - Remove the files related to the `{nodeId}` from the `$IGNITE_HOME/work/db/marshaller` directory. - - Remove the files and sub-directories related to the `{nodeId}` under your `$IGNITE_HOME/work/db` directory. Clean the link:persistence/native-persistence#configuring-persistent-storage-directory[`db/{node_id}`] directory separately if it's not located under the Ignite `work` dir. - - Copy the files belonging to a node with the `{node_id}` from the snapshot into the `$IGNITE_HOME/work/` directory. If the `db/{node_id}` directory is not located under the Ignite `work` dir then you need to copy data files there. + - Remove the files related to the `{consistentId}` from the `$IGNITE_HOME/work/db/binary_meta` directory. + - Remove the files related to the `{consistentId}` from the `$IGNITE_HOME/work/db/marshaller` directory. + - Remove the files and sub-directories related to the `{consistentId}` under your `$IGNITE_HOME/work/db` directory. Clean the link:persistence/native-persistence#configuring-persistent-storage-directory[`db/{consistentId}`] directory separately if it's not located under the Ignite `work` dir. + - Copy the files belonging to a node with the `{consistentId}` from the snapshot into the `$IGNITE_HOME/work/` directory. If the `db/{consistentId}` directory is not located under the Ignite `work` dir then you need to copy data files there. . Restart the cluster === Automatic Snapshot Restore Procedure From d43deea374b84cb698adc4025b644492328f8650 Mon Sep 17 00:00:00 2001 From: liyujue <18624049226@163.com> Date: Thu, 31 Aug 2023 18:36:09 +0800 Subject: [PATCH 3/7] IGNITE-20070 Ignite Docs: Remove docs related to DataRegionMetrics and so on (#10865) --- docs/_data/toc.yaml | 10 +- .../configuring-metrics.adoc | 27 - docs/_docs/monitoring-metrics/metrics.adoc | 503 ------------------ .../new-metrics-system.adoc | 281 +++++++++- 4 files changed, 279 insertions(+), 542 deletions(-) delete mode 100644 docs/_docs/monitoring-metrics/metrics.adoc diff --git a/docs/_data/toc.yaml b/docs/_data/toc.yaml index d5b0faf497490..065de1549e0ff 100644 --- a/docs/_data/toc.yaml +++ b/docs/_data/toc.yaml @@ -430,16 +430,12 @@ url: monitoring-metrics/cluster-id - title: Cluster States url: monitoring-metrics/cluster-states - - title: Metrics - items: - - title: Configuring Metrics - url: monitoring-metrics/configuring-metrics - - title: JMX Metrics - url: monitoring-metrics/metrics - - title: New Metrics System + - title: Metrics System items: - title: Introduction url: monitoring-metrics/new-metrics-system + - title: Configuring Metrics + url: monitoring-metrics/configuring-metrics - title: Metrics url: monitoring-metrics/new-metrics - title: System Views diff --git a/docs/_docs/monitoring-metrics/configuring-metrics.adoc b/docs/_docs/monitoring-metrics/configuring-metrics.adoc index 929e4ec8d3a52..8ccad23104d78 100644 --- a/docs/_docs/monitoring-metrics/configuring-metrics.adoc +++ b/docs/_docs/monitoring-metrics/configuring-metrics.adoc @@ -80,14 +80,6 @@ include::{dotnetFile}[tags=data-region-metrics,indent=0] tab:C++[unsupported] -- -Data region metrics can be enabled/disabled at runtime via the following JMX Bean: - -*Data Region MBean*:: -+ ----- -org.apache:group=DataRegionMetrics,name= ----- - == Enabling Persistence-related Metrics Persistence-related metrics can be enabled/disabled in the data storage configuration: @@ -110,22 +102,3 @@ include::{dotnetFile}[tags=data-storage-metrics,indent=0] ---- tab:C++[unsupported] -- - - -You can enable "Persistent Store" metrics at runtime via the following MXBean: - -*Persistent Store MBean*:: -+ --- ----- -org.apache:group="Persistent Store",name=DataStorageMetrics ----- -[cols="1,4",opts="header"] -|=== -| Operation | Description -| EnableMetrics | Enable persistent data storage metrics. -|=== --- - - - diff --git a/docs/_docs/monitoring-metrics/metrics.adoc b/docs/_docs/monitoring-metrics/metrics.adoc deleted file mode 100644 index 5511e3f593c49..0000000000000 --- a/docs/_docs/monitoring-metrics/metrics.adoc +++ /dev/null @@ -1,503 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -= JMX Metrics - -:table_opts: cols="3,2,8,2", opts="stretch,header" - -== Overview - -The Apache Ignite 2.8 release introduced a new mechanism for collecting metrics, which is intended to replace all -the legacy metrics below. Please, check the link:monitoring-metrics/new-metrics-system[New Metrics System]. - -Ignite exposes a large number of metrics useful for monitoring your cluster or application. -You can use JMX and a monitoring tool, such as JConsole to access these metrics via JMX. -You can also access them programmatically. - -On this page, we've collected the most useful metrics and grouped them into various common categories based on the monitoring task. - -== Enabling JMX for Ignite - -By default, the JMX automatic configuration is disabled. -To enable it, configure the following environment variables: - -* For `control.sh`, configure the `CONTROL_JVM_OPTS` variable -* For `ignite.sh`, configure the `JVM_OPTS` variable - -For example: - -[source,shell] ----- -JVM_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=${JMX_PORT} \ --Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" ----- - - -// link:monitoring-metrics/configuring-metrics[Configuring Metrics] - -== Understanding MBean's ObjectName - -Every JMX Mbean has an https://docs.oracle.com/javase/8/docs/api/javax/management/ObjectName.html[ObjectName,window=_blank]. -The ObjectName is used to identify the bean. -The ObjectName consists of a domain and a list of key properties, and can be represented as a string as follows: - - domain: key1 = value1 , key2 = value2 - -All Ignite metrics have the same domain: `org.apache.` where the classloader ID is optional (omitted if you set `IGNITE_MBEAN_APPEND_CLASS_LOADER_ID=false`). In addition, each metric has two properties: `group` and `name`. -For example: - - org.apache:group=SPIs,name=TcpDiscoverySpi - -This MBean provides various metrics related to node discovery. - -The MBean ObjectName can be used to identify the bean in UI tools like JConsole. -For example, JConsole displays MBeans in a tree-like structure where all beans are first grouped by domain and then by the 'group' property: - -image::images/jconsole.png[] - -{sp}+ - -== Monitoring the Amount of Data - -If you do not use link:persistence/native-persistence[Native persistence] (i.e., all your data is kept in memory), you would want to monitor RAM usage. -If you use Native persistence, in addition to RAM, you should monitor the size of the data storage on disk. - -The size of the data loaded into a node is available at different levels of aggregation. You can monitor for: - -* The total size of the data the node keeps on disk or in RAM. This amount is the sum of the size of each configured data region (in the simplest case, only the default data region) plus the sizes of the system data regions. -* The size of a specific link:memory-configuration/data-regions[data region] on that node. The data region size is the sum of the sizes of all cache groups. -* The size of a specific cache/cache group on that node, including the backup partitions. - -These metrics can be enabled/disabled for each level separately and are exposed via different JMX beans listed below. - - -=== Allocated Space vs. Actual Size of Data - -There is no way to get the exact size of the data (neither in RAM nor on disk). Instead, there are two ways to estimate it. - -You can get the size of the space _allocated_ for storing the data. -(The "space" here refers either to the space in RAM or on disk depending on whether you use Native persistence or not.) -Space is allocated when the size of the storage gets full and more entries need to be added. -However, when you remove entries from caches, the space is not deallocated. -It is reused when new entries need to be added to the storage on subsequent write operations. Therefore, the allocated size does not decrease when you remove entries from the caches. -The allocated size is available at the level of data storage, data region, and cache group metrics. -The metric is called `TotalAllocatedSize`. - -You can also get an estimate of the actual size of data by multiplying the number of link:memory-centric-storage#data-pages[data pages] in use by the fill factor. The fill factor is the ratio of the size of data in a page to the page size, averaged over all pages. The number of pages in use and the fill factor are available at the level of data <>. - -Add up the estimated size of all data regions to get the estimated total amount of data on the node. - - -:allocsize_note: Note that when Native persistence is disabled, this metric shows the total size of the allocated space in RAM. - -=== Monitoring RAM Memory Usage -The amount of data in RAM can be monitored for each data region through the following MBeans: - -Mbean's Object Name: :: -+ --- ----- -group=DataRegionMetrics,name= ----- -[{table_opts}] -|=== -| Attribute | Type | Description | Scope - -| PagesFillFactor| float | The average size of data in pages as a ratio of the page size. When Native persistence is enabled, this metric is applicable only to the persistent storage (i.e. pages on disk). | Node -| TotalUsedPages | long | The number of data pages that are currently in use. When Native persistence is enabled, this metric is applicable only to the persistent storage (i.e. pages on disk).| Node -| PhysicalMemoryPages |long | The number of the allocated pages in RAM. | Node -| PhysicalMemorySize |long |The size of the allocated space in RAM in bytes. | Node -|=== --- - -If you have multiple data regions, add up the sizes of all data regions to get the total size of the data on the node. - -=== Monitoring Storage Size - -Persistent storage, when enabled, saves all application data on disk. -The total amount of data each node keeps on disk consists of the persistent storage (application data), the link:persistence/native-persistence#write-ahead-log[WAL files], and link:persistence/native-persistence#wal-archive[WAL Archive] files. - -==== Persistent Storage Size -To monitor the size of the persistent storage on disk, use the following metrics: - -Mbean's Object Name: :: -+ --- ----- -group="Persistent Store",name=DataStorageMetrics ----- -[{table_opts}] -|=== -| Attribute | Type | Description | Scope -| TotalAllocatedSize | long | The size of the space allocated on disk for the entire data storage (in bytes). {allocsize_note} | Node -| WalTotalSize | long | Total size of the WAL files in bytes, including the WAL archive files. | Node -| WalArchiveSegments | int | The number of WAL segments in the archive. | Node -|=== - -[cols="1,4",opts="header"] -|=== -|Operation | Description -| enableMetrics | Enable collection of metrics related to the persistent storage at runtime. -| disableMetrics | Disable metrics collection. -|=== --- - -==== Data Region Size - -For each configured data region, Ignite creates a separate JMX Bean that exposes specific information about the region. Metrics collection for data regions are disabled by default. You can link:monitoring-metrics/configuring-metrics#enabling-data-region-metrics[enable it in the data region configuration, or via JMX at runtime] (see the Bean's operations below). - -The size of the data region on a node comprises the size of all partitions (including backup partitions) that this node owns for all caches in that data region. - -Data region metrics are available in the following MBean: - -Mbean's Object Name: :: -+ --- ----- -group=DataRegionMetrics,name= ----- - -[{table_opts}] -|=== -| Attribute | Type | Description | Scope - -| TotalAllocatedSize | long | The size of the space allocated for this data region (in bytes). {allocsize_note} | Node -| PagesFillFactor| float | The average amount of data in pages as a ratio of the page size. | Node -| TotalUsedPages | long | The number of data pages that are currently in use. | Node -| PhysicalMemoryPages |long |The number of data pages in this data region held in RAM. | Node -| PhysicalMemorySize | long |The size of the allocated space in RAM in bytes.| Node -|=== - -[cols="1,4",opts="header"] -|=== -|Operation | Description -| enableMetrics | Enable metrics collection for this data region. -| disableMetrics | Disable metrics collection for this data region. -|=== --- - -==== Cache Group Size - -If you don't use link:configuring-caches/cache-groups[cache groups], each cache will be its own group. -There is a separate JMX bean for each cache group. -The name of the bean corresponds to the name of the group. - -Mbean's Object Name: :: -+ --- ----- -group="Cache groups",name= ----- -[{table_opts}] -|=== -| Attribute | Type | Description | Scope -|TotalAllocatedSize |long | The amount of space allocated for the cache group on this node. | Node -|=== --- - -== Monitoring Checkpointing Operations -Checkpointing may slow down cluster operations. -You may want to monitor how much time each checkpoint operation takes, so that you can tune the properties that affect checkpointing. -You may also want to monitor the disk performance to see if the slow-down is caused by external reasons. - -See link:persistence/persistence-tuning#pages-writes-throttling[Pages Writes Throttling] and link:persistence/persistence-tuning#adjusting-checkpointing-buffer-size[Checkpointing Buffer Size] for performance tips. - -Mbean's Object Name: :: -+ --- - group="Persistent Store",name=DataStorageMetrics -[{table_opts}] -|=== -| Attribute | Type | Description | Scope -| DirtyPages | long | The number of pages in memory that have been changed but not yet synchronized to disk. Those will be written to disk during next checkpoint. | Node -|LastCheckpointDuration | long | The time in milliseconds it took to create the last checkpoint. | Node -|CheckpointBufferSize | long | The size of the checkpointing buffer. | Global -|=== --- - - -== Monitoring Rebalancing -link:data-rebalancing[Rebalancing] is the process of moving partitions between the cluster nodes so that the data is always distributed in a balanced manner. Rebalancing is triggered when a new node joins, or an existing node leaves the cluster. - -If you have multiple caches, they will be rebalanced sequentially. -There are several metrics that you can use to monitor the progress of the rebalancing process for a specific cache. - -In the new metric system, link:monitoring-metrics/new-metrics#caches[Cache metrics]: -[{table_opts}] -|=== -| Attribute | Type | Description | Scope -|RebalancingStartTime | long | This metric shows the time when rebalancing of local partitions started for the cache. This metric will return 0 if the local partitions do not participate in the rebalancing. The time is returned in milliseconds. | Node -| EstimatedRebalancingFinishTime | long | Expected time of completion of the rebalancing process. | Node -| KeysToRebalanceLeft | long | The number of keys on the node that remain to be rebalanced. You can monitor this metric to learn when the rebalancing process finishes.| Node -|=== --- - - -== Monitoring Topology -Topology refers to the set of nodes in a cluster. There are a number of metrics that expose the information about the topology of the cluster. If the topology changes too frequently or has a size that is different from what you expect, you may want to look into whether there are network problems. - - -Mbean's Object Name: :: -+ --- ----- -group=Kernal,name=ClusterMetricsMXBeanImpl ----- -[{table_opts}] -|=== -| Attribute | Type | Description | Scope -| TotalServerNodes| long |The number of server nodes in the cluster.| Global -| TotalClientNodes| long |The number of client nodes in the cluster. | Global -| TotalBaselineNodes | long | The number of nodes that are registered in the link:clustering/baseline-topology[baseline topology]. When a node goes down, it remains registered in the baseline topology and you need to remote it manually. | Global -| ActiveBaselineNodes | long | The number of nodes that are currently active in the baseline topology. | Global -|=== --- - -Mbean's Object Name: :: -+ --- ----- -group=SPIs,name=TcpDiscoverySpi ----- -[{table_opts}] -|=== -| Attribute | Type | Description | Scope -| Coordinator | String | The node ID of the current coordinator node.| Global -| CoordinatorNodeFormatted|String a| -Detailed information about the coordinator node. -.... -TcpDiscoveryNode [id=e07ad289-ff5b-4a73-b3d4-d323a661b6d4, -consistentId=fa65ff2b-e7e2-4367-96d9-fd0915529c25, -addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.25.4.200], -sockAddrs=[mymachine.local/172.25.4.200:47500, -/0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, -order=2, intOrder=2, lastExchangeTime=1568187777249, loc=false, -ver=8.7.5#20190520-sha1:d159cd7a, isClient=false] -.... - -| Global -|=== --- - -== Monitoring Caches - -See the new metric system, link:monitoring-metrics/new-metrics#caches[Cache metrics]. - -=== Monitoring Build and Rebuild Indexes - -To get an estimate on how long it takes to rebuild cache indexes, you can use one of the metrics listed below: - -. `IsIndexRebuildInProgress` - tells whether indexes are being built or rebuilt at the moment; -. `IndexBuildCountPartitionsLeft` - gives the remaining number of partitions (by cache group) for indexes to rebuild. - -Note that the `IndexBuildCountPartitionsLeft` metric allows to estimate only an approximate number of indexes left to rebuild. -For a more accurate estimate, use the `IndexRebuildKeyProcessed` cache metric: - -* Use `isIndexRebuildInProgress` to know whether the indexes are being rebuilt for the cache. - -* Use `IndexRebuildKeysProcessed` to know the number of keys with rebuilt indexes. If the rebuilding is in progress, it gives a number of keys with indexes being rebuilt at the current moment. Otherwise, it gives a total number of the of keys with rebuilt indexes. The values are reset before the start of each rebuilding. - -== Monitoring Transactions -Note that if a transaction spans multiple nodes (i.e., if the keys that are changed as a result of the transaction execution are located on multiple nodes), the counters will increase on each node. For example, the 'TransactionsCommittedNumber' counter will increase on each node where the keys affected by the transaction are stored. - -Mbean's Object Name: :: -+ --- ----- -group=TransactionMetrics,name=TransactionMetricsMxBeanImpl ----- - -[{table_opts}] -|=== -| Attribute | Type | Description | Scope -| LockedKeysNumber | long | The number of keys locked on the node. | Node -| TransactionsCommittedNumber |long | The number of transactions that have been committed on the node | Node -| TransactionsRolledBackNumber | long | The number of transactions that were rolled back. | Node -| OwnerTransactionsNumber | long | The number of transactions initiated on the node. | Node -| TransactionsHoldingLockNumber | long | The number of open transactions that hold a lock on at least one key on the node.| Node -|=== --- - -//// -this isn't in 8.7.6 yet -{sp}+ - -Mbean's Object Name: :: -`group=Transactions,name=TransactionsMXBeanImpl` -*Attributes:*:: -{sp} -+ --- -[{table_opts}] -|=== -| Attribute | Type | Description | Scope -| TotalNodeSystemTime | long | system time | Node -| TotalNodeUserTime | | | Node -| NodeSystemTimeHistogram | | | Node -| NodeUserTimeHistogram | | | Node -|=== --- - -//// - - -//// -{sp}+ - - -== Monitoring Compute Jobs - -Mbean's Object Name: :: -`group= ,name=` -*Attributes:*:: -{sp} -+ --- -[{table_opts}] -|=== -| Attribute | Type | Description | Scope -| | | | -|=== --- - -//// - - -//// -== Monitoring Snapshots - -Mbean's Object Name: :: -+ --- ----- -group=TODO ,name= TODO ----- -[{table_opts}] -|=== -| Attribute | Type | Description | Scope -| LastSnapshotOperation | | | -| LastSnapshotStartTime || | -| SnapshotInProgress | | | -|=== --- -//// - - -//// -== Monitoring Memory Consumption - -JVM memory - -Mbean's Object Name: :: -+ ----- -group=Kernal,name=ClusterMetricsMXBeanImpl ----- -*Attributes:*:: -+ -[{table_opts}] -|=== -| Attribute | Type | Description | Scope -| HeapMemoryUsed | long | The Java heap size on the node. | Node -|=== - -//// - - -== Monitoring Client Connections -Metrics related to JDBC/ODBC or thin client connections. - -Mbean's Object Name: :: -+ --- ----- -group=Clients,name=ClientListenerProcessor ----- -[{table_opts}] -|=== -| Attribute | Type | Description | Scope -| Connections | java.util.List a| A list of strings, each string containing information about a connection: - -.... -JdbcClient [id=4294967297, user=, -rmtAddr=127.0.0.1:39264, locAddr=127.0.0.1:10800] -.... -| Node -|=== - -[cols="1,4",opts="header"] -|=== -|Operation | Description -| dropConnection (id)| Disconnect a specific client. -| dropAllConnections | Disconnect all clients. -|=== --- - - -== Monitoring Message Queues -When thread pools queues' are growing, it means that the node cannot keep up with the load, or there was an error while processing messages in the queue. -Continuous growth of the queue size can lead to OOM errors. - - -=== Communication Message Queue -The queue of outgoing communication messages contains communication messages that are waiting to be sent to other nodes. -If the size is growing, it means there is a problem. - -Mbean's Object Name: :: -+ --- ----- -group=SPIs,name=TcpCommunicationSpi ----- -[{table_opts}] -|=== -| Attribute | Type | Description | Scope -| OutboundMessagesQueueSize | int | The size of the queue of outgoing communication messages. | Node -|=== --- - -=== Discovery Messages Queue - -The queue of discovery messages. - -Mbean's Object Name: :: -+ --- ----- -group=SPIs,name=TcpDiscoverySpi ----- -[{table_opts}] -|=== -| Attribute | Type | Description | Scope -| MessageWorkerQueueSize | int | The size of the queue of discovery messages that are waiting to be sent to other nodes. | Node -|AvgMessageProcessingTime|long| Average message processing time. | Node -|=== --- - -//// - -== Monitoring Executor Queue Size - -There is a number of executor thread pools running within each node that are dedicated to specific tasks. -You may want to monitor the size of the executor's queues. -You can read more about the thread pools on the link:perf-troubleshooting-guide/thread-pools-tuning[Thread Tuning Page] - -There is a JMX Bean for each thread pool. - -//// - - - - - diff --git a/docs/_docs/monitoring-metrics/new-metrics-system.adoc b/docs/_docs/monitoring-metrics/new-metrics-system.adoc index e300ebbe3e1b7..62a6cba315800 100644 --- a/docs/_docs/monitoring-metrics/new-metrics-system.adoc +++ b/docs/_docs/monitoring-metrics/new-metrics-system.adoc @@ -12,17 +12,16 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -= New Metrics System += Metrics System :javaFile: {javaCodeDir}/ConfiguringMetrics.java == Overview -Ignite 2.8 introduced a new mechanism for collecting metrics, which is intended to replace the link:monitoring-metrics/metrics[legacy metrics system]. -This section explains the new system and how you can use it to monitor your cluster. -//the types of metrics and how to export them, but first let's explore the basic concepts of the new metrics mechanism in Ignite. +This section explains the metrics system and how you can use it to monitor your cluster. +//the types of metrics and how to export them, but first let's explore the basic concepts of the metrics mechanism in Ignite. -Let's explore the basic concepts of the new metrics system in Ignite. +Let's explore the basic concepts of the metrics system in Ignite. First, there are different metrics. Each metric has a name and a return value. The return value can be a simple value like `String`, `long`, or `double`, or can represent a Java object. @@ -115,6 +114,47 @@ tab:C#/.NET[] tab:C++[unsupported] -- +==== Enabling JMX for Ignite + +By default, the JMX automatic configuration is disabled. +To enable it, configure the following environment variables: + +* For `control.sh`, configure the `CONTROL_JVM_OPTS` variable +* For `ignite.sh`, configure the `JVM_OPTS` variable + +For example: + +[source,shell] +---- +JVM_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=${JMX_PORT} \ +-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" +---- + + +// link:monitoring-metrics/configuring-metrics[Configuring Metrics] + +==== Understanding MBean's ObjectName + +Every JMX Mbean has an https://docs.oracle.com/javase/8/docs/api/javax/management/ObjectName.html[ObjectName,window=_blank]. +The ObjectName is used to identify the bean. +The ObjectName consists of a domain and a list of key properties, and can be represented as a string as follows: + + domain: key1 = value1 , key2 = value2 + +All Ignite metrics have the same domain: `org.apache.` where the classloader ID is optional (omitted if you set `IGNITE_MBEAN_APPEND_CLASS_LOADER_ID=false`). In addition, each metric has two properties: `group` and `name`. +For example: + + org.apache:group=SPIs,name=TcpDiscoverySpi + +This MBean provides various metrics related to node discovery. + +The MBean ObjectName can be used to identify the bean in UI tools like JConsole. +For example, JConsole displays MBeans in a tree-like structure where all beans are first grouped by domain and then by the 'group' property: + +image::images/jconsole.png[] + +{sp}+ + === SQL View `SqlViewMetricExporterSpi` is enabled by default, `SqlViewMetricExporterSpi` exposes metrics via the `SYS.METRICS` view. @@ -209,5 +249,236 @@ Example of the metric names if the bounds are [10,100]: * `histogram_10_100` - between 10 and 100. * `histogram_100_inf` - more than 100. +== Common monitoring tasks +=== Monitoring the Amount of Data + +If you do not use link:persistence/native-persistence[Native persistence] (i.e., all your data is kept in memory), you would want to monitor RAM usage. +If you use Native persistence, in addition to RAM, you should monitor the size of the data storage on disk. + +The size of the data loaded into a node is available at different levels of aggregation. You can monitor for: + +* The total size of the data the node keeps on disk or in RAM. This amount is the sum of the size of each configured data region (in the simplest case, only the default data region) plus the sizes of the system data regions. +* The size of a specific link:memory-configuration/data-regions[data region] on that node. The data region size is the sum of the sizes of all cache groups. +* The size of a specific cache/cache group on that node, including the backup partitions. + + +==== Allocated Space vs. Actual Size of Data + +There is no way to get the exact size of the data (neither in RAM nor on disk). Instead, there are two ways to estimate it. + +You can get the size of the space _allocated_ for storing the data. +(The "space" here refers either to the space in RAM or on disk depending on whether you use Native persistence or not.) +Space is allocated when the size of the storage gets full and more entries need to be added. +However, when you remove entries from caches, the space is not deallocated. +It is reused when new entries need to be added to the storage on subsequent write operations. Therefore, the allocated size does not decrease when you remove entries from the caches. +The allocated size is available at the level of data storage, data region, and cache group metrics. +The metric is called `TotalAllocatedSize`. + +You can also get an estimate of the actual size of data by multiplying the number of link:memory-centric-storage#data-pages[data pages] in use by the fill factor. The fill factor is the ratio of the size of data in a page to the page size, averaged over all pages. The number of pages in use and the fill factor are available at the level of data <>. + +Add up the estimated size of all data regions to get the estimated total amount of data on the node. + + +:allocsize_note: Note that when Native persistence is disabled, this metric shows the total size of the allocated space in RAM. + +==== Monitoring RAM Memory Usage +The amount of data in RAM can be monitored for each data region through the following metrics: + +[{table_opts}] +|=== +| Attribute | Type | Description | Scope + +| PagesFillFactor| float | The average size of data in pages as a ratio of the page size. When Native persistence is enabled, this metric is applicable only to the persistent storage (i.e. pages on disk). | Node +| TotalUsedPages | long | The number of data pages that are currently in use. When Native persistence is enabled, this metric is applicable only to the persistent storage (i.e. pages on disk).| Node +| PhysicalMemoryPages |long | The number of the allocated pages in RAM. | Node +| PhysicalMemorySize |long |The size of the allocated space in RAM in bytes. | Node +|=== + +If you have multiple data regions, add up the sizes of all data regions to get the total size of the data on the node. + +==== Monitoring Storage Size + +Persistent storage, when enabled, saves all application data on disk. +The total amount of data each node keeps on disk consists of the persistent storage (application data), the link:persistence/native-persistence#write-ahead-log[WAL files], and link:persistence/native-persistence#wal-archive[WAL Archive] files. + +===== Persistent Storage Size +To monitor the size of the persistent storage on disk, use the following metrics: + +[{table_opts}] +|=== +| Attribute | Type | Description | Scope +| TotalAllocatedSize | long | The size of the space allocated on disk for the entire data storage (in bytes). {allocsize_note} | Node +| WalTotalSize | long | Total size of the WAL files in bytes, including the WAL archive files. | Node +| WalArchiveSegments | int | The number of WAL segments in the archive. | Node +|=== + +===== Data Region Size + +For each configured data region, Metrics collection for data regions are disabled by default. You can link:monitoring-metrics/configuring-metrics#enabling-data-region-metrics[enable it in the data region configuration. + +The size of the data region on a node comprises the size of all partitions (including backup partitions) that this node owns for all caches in that data region. +[{table_opts}] +|=== +| Attribute | Type | Description | Scope +| TotalAllocatedSize | long | The size of the space allocated for this data region (in bytes). {allocsize_note} | Node +| PagesFillFactor| float | The average amount of data in pages as a ratio of the page size. | Node +| TotalUsedPages | long | The number of data pages that are currently in use. | Node +| PhysicalMemoryPages |long |The number of data pages in this data region held in RAM. | Node +| PhysicalMemorySize | long |The size of the allocated space in RAM in bytes.| Node +|=== + +===== Cache Group Size + +If you don't use link:configuring-caches/cache-groups[cache groups], each cache will be its own group. + +[{table_opts}] +|=== +| Attribute | Type | Description | Scope +|TotalAllocatedSize |long | The amount of space allocated for the cache group on this node. | Node +|=== + +=== Monitoring Checkpointing Operations +Checkpointing may slow down cluster operations. +You may want to monitor how much time each checkpoint operation takes, so that you can tune the properties that affect checkpointing. +You may also want to monitor the disk performance to see if the slow-down is caused by external reasons. + +See link:persistence/persistence-tuning#pages-writes-throttling[Pages Writes Throttling] and link:persistence/persistence-tuning#adjusting-checkpointing-buffer-size[Checkpointing Buffer Size] for performance tips. + +[{table_opts}] +|=== +| Attribute | Type | Description | Scope +| DirtyPages | long | The number of pages in memory that have been changed but not yet synchronized to disk. Those will be written to disk during next checkpoint. | Node +|LastCheckpointDuration | long | The time in milliseconds it took to create the last checkpoint. | Node +|CheckpointBufferSize | long | The size of the checkpointing buffer. | Global +|=== + +=== Monitoring Rebalancing +link:data-rebalancing[Rebalancing] is the process of moving partitions between the cluster nodes so that the data is always distributed in a balanced manner. Rebalancing is triggered when a new node joins, or an existing node leaves the cluster. + +If you have multiple caches, they will be rebalanced sequentially. +There are several metrics that you can use to monitor the progress of the rebalancing process for a specific cache. + +In the metric system, link:monitoring-metrics/new-metrics#caches[Cache metrics]: +[{table_opts}] +|=== +| Attribute | Type | Description | Scope +|RebalancingStartTime | long | This metric shows the time when rebalancing of local partitions started for the cache. This metric will return 0 if the local partitions do not participate in the rebalancing. The time is returned in milliseconds. | Node +| EstimatedRebalancingFinishTime | long | Expected time of completion of the rebalancing process. | Node +| KeysToRebalanceLeft | long | The number of keys on the node that remain to be rebalanced. You can monitor this metric to learn when the rebalancing process finishes.| Node +|=== + +=== Monitoring Topology +Topology refers to the set of nodes in a cluster. There are a number of metrics that expose the information about the topology of the cluster. If the topology changes too frequently or has a size that is different from what you expect, you may want to look into whether there are network problems. + +[{table_opts}] +|=== +| Attribute | Type | Description | Scope +| TotalServerNodes| long |The number of server nodes in the cluster.| Global +| TotalClientNodes| long |The number of client nodes in the cluster. | Global +| TotalBaselineNodes | long | The number of nodes that are registered in the link:clustering/baseline-topology[baseline topology]. When a node goes down, it remains registered in the baseline topology and you need to remote it manually. | Global +| ActiveBaselineNodes | long | The number of nodes that are currently active in the baseline topology. | Global +|=== + +[{table_opts}] +|=== +| Attribute | Type | Description | Scope +| Coordinator | String | The node ID of the current coordinator node.| Global +| CoordinatorNodeFormatted|String a| +Detailed information about the coordinator node. +.... +TcpDiscoveryNode [id=e07ad289-ff5b-4a73-b3d4-d323a661b6d4, +consistentId=fa65ff2b-e7e2-4367-96d9-fd0915529c25, +addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.25.4.200], +sockAddrs=[mymachine.local/172.25.4.200:47500, +/0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, +order=2, intOrder=2, lastExchangeTime=1568187777249, loc=false, +ver=8.7.5#20190520-sha1:d159cd7a, isClient=false] +.... + +| Global +|=== + +=== Monitoring Caches + +See the new metric system, link:monitoring-metrics/new-metrics#caches[Cache metrics]. + +==== Monitoring Build and Rebuild Indexes + +To get an estimate on how long it takes to rebuild cache indexes, you can use one of the metrics listed below: + +. `IsIndexRebuildInProgress` - tells whether indexes are being built or rebuilt at the moment; +. `IndexBuildCountPartitionsLeft` - gives the remaining number of partitions (by cache group) for indexes to rebuild. + +Note that the `IndexBuildCountPartitionsLeft` metric allows to estimate only an approximate number of indexes left to rebuild. +For a more accurate estimate, use the `IndexRebuildKeyProcessed` cache metric: + +* Use `isIndexRebuildInProgress` to know whether the indexes are being rebuilt for the cache. + +* Use `IndexRebuildKeysProcessed` to know the number of keys with rebuilt indexes. If the rebuilding is in progress, it gives a number of keys with indexes being rebuilt at the current moment. Otherwise, it gives a total number of the of keys with rebuilt indexes. The values are reset before the start of each rebuilding. + +=== Monitoring Transactions +Note that if a transaction spans multiple nodes (i.e., if the keys that are changed as a result of the transaction execution are located on multiple nodes), the counters will increase on each node. For example, the 'TransactionsCommittedNumber' counter will increase on each node where the keys affected by the transaction are stored. + +[{table_opts}] +|=== +| Attribute | Type | Description | Scope +| LockedKeysNumber | long | The number of keys locked on the node. | Node +| TransactionsCommittedNumber |long | The number of transactions that have been committed on the node | Node +| TransactionsRolledBackNumber | long | The number of transactions that were rolled back. | Node +| OwnerTransactionsNumber | long | The number of transactions initiated on the node. | Node +| TransactionsHoldingLockNumber | long | The number of open transactions that hold a lock on at least one key on the node.| Node +|=== + +=== Monitoring Snapshots + +[{table_opts}] +|=== +| Attribute | Type | Description | Scope +| LastSnapshotOperation | | | +| LastSnapshotStartTime || | +| SnapshotInProgress | | | +|=== + +=== Monitoring Client Connections +Metrics related to JDBC/ODBC or thin client connections. + +[{table_opts}] +|=== +| Attribute | Type | Description | Scope +| Connections | java.util.List a| A list of strings, each string containing information about a connection: + +.... +JdbcClient [id=4294967297, user=, +rmtAddr=127.0.0.1:39264, locAddr=127.0.0.1:10800] +.... +| Node +|=== + + +=== Monitoring Message Queues +When thread pools queues' are growing, it means that the node cannot keep up with the load, or there was an error while processing messages in the queue. +Continuous growth of the queue size can lead to OOM errors. + +==== Communication Message Queue +The queue of outgoing communication messages contains communication messages that are waiting to be sent to other nodes. +If the size is growing, it means there is a problem. + +[{table_opts}] +|=== +| Attribute | Type | Description | Scope +| OutboundMessagesQueueSize | int | The size of the queue of outgoing communication messages. | Node +|=== + +==== Discovery Messages Queue + +The queue of discovery messages. + +[{table_opts}] +|=== +| Attribute | Type | Description | Scope +| MessageWorkerQueueSize | int | The size of the queue of discovery messages that are waiting to be sent to other nodes. | Node +|AvgMessageProcessingTime|long| Average message processing time. | Node +|=== +-- From dbf1c7825d74809cd6859c85a8ac9ed9ac071e39 Mon Sep 17 00:00:00 2001 From: Nikita Amelchev Date: Thu, 31 Aug 2023 22:43:09 +0300 Subject: [PATCH 4/7] IGNITE-20097 Fixed WAL logging to an archived segment after node restart (#10887) --- .../wal/FileWriteAheadLogManager.java | 53 ++++- ...RestartWithWalForceArchiveTimeoutTest.java | 46 +++- .../ignite/cdc/WalRolloverOnStopTest.java | 202 ++++++++++++++++++ .../testsuites/IgnitePdsTestSuite2.java | 2 + 4 files changed, 292 insertions(+), 11 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/cdc/WalRolloverOnStopTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index f68b366fbb6ff..c1f04bd3baab7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -1455,16 +1455,25 @@ private FileWriteHandle rollOver(FileWriteHandle cur, @Nullable WALRecord rec) t * @throws StorageException If failed to initialize WAL write handle. */ private FileWriteHandle restoreWriteHandle(@Nullable WALPointer lastReadPtr) throws StorageException { - long absIdx = lastReadPtr == null ? 0 : lastReadPtr.index(); - @Nullable FileArchiver archiver0 = archiver; - long segNo = archiver0 == null ? absIdx : absIdx % dsCfg.getWalSegments(); + long absIdx; + int off; - File curFile = new File(walWorkDir, fileName(segNo)); + if (lastReadPtr == null) { + absIdx = 0; + off = 0; + } + else if (nextSegmentInited(lastReadPtr)) { + absIdx = lastReadPtr.index() + 1; + off = HEADER_RECORD_SIZE; + } + else { + absIdx = lastReadPtr.index(); + off = lastReadPtr.fileOffset() + lastReadPtr.length(); + } - int off = lastReadPtr == null ? 0 : lastReadPtr.fileOffset(); - int len = lastReadPtr == null ? 0 : lastReadPtr.length(); + File curFile = segmentFile(absIdx); try { SegmentIO fileIO = new SegmentIO(absIdx, ioFactory.create(curFile)); @@ -1494,7 +1503,7 @@ private FileWriteHandle restoreWriteHandle(@Nullable WALPointer lastReadPtr) thr ", offset=" + off + ", ver=" + serVer + ']'); } - FileWriteHandle hnd = fileHandleManager.initHandle(fileIO, off + len, ser); + FileWriteHandle hnd = fileHandleManager.initHandle(fileIO, off, ser); segmentAware.curAbsWalIdx(absIdx); @@ -1545,6 +1554,36 @@ private FileWriteHandle restoreWriteHandle(@Nullable WALPointer lastReadPtr) thr } } + /** */ + private File segmentFile(long absIdx) { + long segNo = archiver == null ? absIdx : absIdx % dsCfg.getWalSegments(); + + return new File(walWorkDir, fileName(segNo)); + } + + /** @return {@code True} if the given pointer is the last in a segment and a next segment has been initialized. */ + private boolean nextSegmentInited(WALPointer ptr) { + try { + try (WALIterator iter = replay(new WALPointer(ptr.index(), ptr.fileOffset() + ptr.length(), 0))) { + if (iter.hasNext()) + return false; + } + + long nextIdx = ptr.index() + 1; + + try (SegmentIO fileIO = new SegmentIO(nextIdx, ioFactory.create(segmentFile(nextIdx), READ))) { + readSegmentHeader(fileIO, segmentFileInputFactory); + } + + return true; + } + catch (Exception ignored) { + // No-op. + } + + return false; + } + /** * Fills the file header for a new segment. Calling this method signals we are done with the segment and it can be * archived. If we don't have prepared file yet and achiever is busy this method blocks. diff --git a/modules/core/src/test/java/org/apache/ignite/cdc/RestartWithWalForceArchiveTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/cdc/RestartWithWalForceArchiveTimeoutTest.java index d28abb3e5b1f1..57cdee034dbe7 100644 --- a/modules/core/src/test/java/org/apache/ignite/cdc/RestartWithWalForceArchiveTimeoutTest.java +++ b/modules/core/src/test/java/org/apache/ignite/cdc/RestartWithWalForceArchiveTimeoutTest.java @@ -41,6 +41,9 @@ public class RestartWithWalForceArchiveTimeoutTest extends GridCommonAbstractTes @Parameterized.Parameter public WALMode walMode; + /** */ + private long walForceArchiveTimeout; + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); @@ -49,7 +52,7 @@ public class RestartWithWalForceArchiveTimeoutTest extends GridCommonAbstractTes cfg.setDataStorageConfiguration(new DataStorageConfiguration() .setWalMode(walMode) - .setWalForceArchiveTimeout(60 * 60 * 1000) // 1 hour to make sure auto archive will not work. + .setWalForceArchiveTimeout(walForceArchiveTimeout) .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true))); return cfg; @@ -61,12 +64,17 @@ public static Collection parameters() { return EnumSet.of(WALMode.FSYNC, WALMode.LOG_ONLY, WALMode.BACKGROUND); } - /** */ - @Test - public void testRestart() throws Exception { + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { stopAllGrids(true); cleanPersistenceDir(); + } + + /** */ + @Test + public void testRestart() throws Exception { + walForceArchiveTimeout = 60 * 60 * 1000; // 1 hour to make sure auto archive will not work. Supplier restart = () -> { stopAllGrids(true); @@ -92,4 +100,34 @@ public void testRestart() throws Exception { for (int i = 0; i < 5; i++) restart.get(); } + + /** */ + @Test + public void testRestartAfterArchive() throws Exception { + walForceArchiveTimeout = 1000; + + IgniteEx srv = startGrid(0); + + srv.cluster().state(ACTIVE); + + IgniteCache cache = srv.getOrCreateCache(DEFAULT_CACHE_NAME); + + cache.put(1, 1); + + forceCheckpoint(); + + Thread.sleep(2 * walForceArchiveTimeout); + + stopGrid(0); + srv = startGrid(0); + cache = srv.cache(DEFAULT_CACHE_NAME); + + cache.put(2, 2); + + stopGrid(0); + srv = startGrid(0); + cache = srv.cache(DEFAULT_CACHE_NAME); + + assertEquals(2, cache.size()); + } } diff --git a/modules/core/src/test/java/org/apache/ignite/cdc/WalRolloverOnStopTest.java b/modules/core/src/test/java/org/apache/ignite/cdc/WalRolloverOnStopTest.java new file mode 100644 index 0000000000000..2e21eb86f0aea --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/cdc/WalRolloverOnStopTest.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.cdc; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.pagemem.wal.WALIterator; +import org.apache.ignite.internal.pagemem.wal.record.DataEntry; +import org.apache.ignite.internal.pagemem.wal.record.DataRecord; +import org.apache.ignite.internal.pagemem.wal.record.WALRecord; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener; +import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer; +import org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAware; +import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiTuple; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH; +import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.DATA_RECORD_V2; +import static org.apache.ignite.testframework.GridTestUtils.runAsync; + +/** + * This tests check that the following scenario will works correctly. + */ +@RunWith(Parameterized.class) +public class WalRolloverOnStopTest extends GridCommonAbstractTest { + /** WAL mode. */ + @Parameterized.Parameter + public WALMode walMode; + + /** @return Test parameters. */ + @Parameterized.Parameters(name = "walMode={0}") + public static Collection parameters() { + return Arrays.asList(new Object[][] {{WALMode.BACKGROUND}, {WALMode.LOG_ONLY}, {WALMode.FSYNC}}); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName) + .setDataStorageConfiguration(new DataStorageConfiguration() + .setWalAutoArchiveAfterInactivity(1500L) + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setPersistenceEnabled(true))); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + + cleanPersistenceDir(); + } + + /** + * Test scenario: + * + * 0. {@link DataStorageConfiguration#getWalAutoArchiveAfterInactivity()} > 0. + * 1. Node is gracefully stopping using {@link G#stop(String, boolean)}. + * 2. T0: {@code Checkpointer#doCheckpoint()} execute last checkpoint on stop and freeze. + * 3. T1: Rollover segment after inactivity timeout. + * 4. T2: Archive segment. + * + * After restart WAL should log in the next segment. + * */ + @Test + public void testWallRollover() throws Exception { + AtomicLong curIdx = new AtomicLong(); + + for (int i = 0; i < 2; i++) { + IgniteEx ign = startGrid(0); + + GridCacheDatabaseSharedManager db = + (GridCacheDatabaseSharedManager)ign.context().cache().context().database(); + + SegmentAware aware = GridTestUtils.getFieldValue(ign.context().cache().context().wal(), "segmentAware"); + + ign.cluster().state(ClusterState.ACTIVE); + + IgniteCache cache = ign.getOrCreateCache("my-cache"); + + CountDownLatch waitAfterCp = new CountDownLatch(1); + AtomicLong cntr = new AtomicLong(0); + + db.addCheckpointListener(new CheckpointListener() { + @Override public void afterCheckpointEnd(Context ctx) { + if (!ign.context().isStopping()) + return; + + try { + waitAfterCp.await(getTestTimeout(), TimeUnit.MILLISECONDS); + + cntr.incrementAndGet(); + } + catch (InterruptedException e) { + throw new IgniteException(e); + } + } + + @Override public void onMarkCheckpointBegin(Context ctx) { + // No-op. + } + + @Override public void onCheckpointBegin(Context ctx) { + // No-op. + } + + @Override public void beforeCheckpointBegin(Context ctx) { + // No-op. + } + }); + + int maxKey = (i + 1) * 3; + + for (int j = i * 3; j < maxKey; j++) + cache.put(j, j); + + curIdx.set(aware.curAbsWalIdx()); + + IgniteInternalFuture fut = runAsync(() -> { + try { + aware.awaitSegmentArchived(curIdx.get()); + + cntr.incrementAndGet(); + } + catch (IgniteInterruptedCheckedException e) { + throw new IgniteException(e); + } + finally { + waitAfterCp.countDown(); + } + }); + + G.stop(ign.name(), false); + + fut.get(getTestTimeout()); + + // Checkpoint will happens two time because of segment archivation. + assertEquals("Should successfully wait for current segment archivation", 3, cntr.get()); + + IgniteWalIteratorFactory.IteratorParametersBuilder builder = + new IgniteWalIteratorFactory.IteratorParametersBuilder() + .log(ign.log()) + .filesOrDirs( + U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_ARCHIVE_PATH, false)) + .filter((type, ptr) -> type == DATA_RECORD_V2); + + Set keys = new HashSet<>(); + + try (WALIterator it = new IgniteWalIteratorFactory().iterator(builder)) { + while (it.hasNext()) { + IgniteBiTuple tup = it.next(); + + DataRecord rec = (DataRecord)tup.get2(); + + for (DataEntry entry : rec.writeEntries()) + keys.add(entry.key().value(null, false)); + } + } + + for (int j = 0; j < maxKey; j++) + assertTrue(keys.contains(j)); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java index 8b39f3e54ed90..2d3e06fffd475 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java @@ -26,6 +26,7 @@ import org.apache.ignite.cdc.CdcSelfTest; import org.apache.ignite.cdc.RestartWithWalForceArchiveTimeoutTest; import org.apache.ignite.cdc.WalForCdcTest; +import org.apache.ignite.cdc.WalRolloverOnStopTest; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.HistoricalRebalanceCheckpointTest; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.HistoricalRebalanceHeuristicsTest; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.HistoricalRebalanceTwoPartsInDifferentCheckpointsTest; @@ -155,6 +156,7 @@ public static void addRealPageStoreTests(List> suite, Collection GridTestUtils.addTestIfNeeded(suite, CdcSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CdcCacheVersionTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, RestartWithWalForceArchiveTimeoutTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, WalRolloverOnStopTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, WalForCdcTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CdcCacheConfigOnRestartTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CdcNonDefaultWorkDirTest.class, ignoredTests); From 9b5d30c8556323a41ea18a3e3f326aa745322ba2 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Tue, 5 Sep 2023 05:15:06 +0800 Subject: [PATCH 5/7] IGNITE-18830 Fixed erroneous logging of failed index rebuild (#10831) --- .../internal/processors/query/GridQueryProcessor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java index b3415101a40f6..d5b53e0af6cc2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java @@ -2637,8 +2637,10 @@ private IgniteInternalFuture rebuildIndexesFromHash0( idxFut.listen(() -> { Throwable err = idxFut.error(); - if (isNull(err) && log.isInfoEnabled()) - log.info("Finished indexes rebuilding for cache " + cacheInfo); + if (isNull(err)) { + if (log.isInfoEnabled()) + log.info("Finished indexes rebuilding for cache " + cacheInfo); + } else if (!(err instanceof NodeStoppingException)) log.error("Failed to rebuild indexes for cache " + cacheInfo, err); From 8b21334f9cbe6960d73cc63dfe093d84ba7d12c8 Mon Sep 17 00:00:00 2001 From: IgGusev Date: Tue, 5 Sep 2023 16:44:10 +0400 Subject: [PATCH 6/7] IGNITE-20314 Updated opens for Java 17 (#10923) --- docs/_docs/includes/java9.adoc | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/_docs/includes/java9.adoc b/docs/_docs/includes/java9.adoc index f636c6c3d5783..0e2316ab57fb3 100644 --- a/docs/_docs/includes/java9.adoc +++ b/docs/_docs/includes/java9.adoc @@ -41,17 +41,22 @@ tab:Java 11[] tab:Java 17[] [source,shell] ---- ---add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED \ ---add-opens=java.base/jdk.internal.misc=ALL-UNNAMED \ ---add-opens=java.base/sun.nio.ch=ALL-UNNAMED \ ---add-opens=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED \ ---add-opens=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED \ ---add-opens=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED \ ---add-opens=java.base/java.io=ALL-UNNAMED \ ---add-opens=java.base/java.nio=ALL-UNNAMED \ ---add-opens=java.base/java.util=ALL-UNNAMED \ ---add-opens=java.base/java.util.concurrent=ALL-UNNAMED \ ---add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED \ +--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED +--add-opens=java.base/sun.nio.ch=ALL-UNNAMED +--add-opens=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED +--add-opens=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED +--add-opens=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED +--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED +--add-opens=java.base/java.io=ALL-UNNAMED +--add-opens=java.base/java.nio=ALL-UNNAMED +--add-opens=java.base/java.util=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED +--add-opens=java.base/java.lang.invoke=ALL-UNNAMED +--add-opens=java.base/java.math=ALL-UNNAMED +--add-opens=java.sql/java.sql=ALL-UNNAMED +--add-opens=java.base/java.net=ALL-UNNAMED ---- -- From 24754c215803b9b5d0bd97905609dbda05043d3d Mon Sep 17 00:00:00 2001 From: "bin.yin" <616468159@qq.com> Date: Fri, 8 Sep 2023 10:17:59 +0300 Subject: [PATCH 7/7] IGNITE-18330 Fix javadoc for tx.suspend()/tx.resume() - Fixes #10604. Signed-off-by: Aleksey Plekhanov --- .../ignite/cache/store/jdbc/CacheJdbcPojoStoreFactory.java | 2 +- .../internal/processors/cache/GridCacheSharedContext.java | 2 +- .../processors/cache/distributed/near/GridNearTxLocal.java | 2 +- .../main/java/org/apache/ignite/transactions/Transaction.java | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreFactory.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreFactory.java index 0bf6de6e3e601..4fecc335345fc 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreFactory.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreFactory.java @@ -205,7 +205,7 @@ public CacheJdbcPojoStoreFactory setDataSource(DataSource dataSrc) { } /** - * Get maximum batch size for delete and delete operations. + * Get maximum batch size for write and delete operations. * * @return Maximum batch size. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java index e6f9f70e12455..4b7ccb7fcd2be 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java @@ -1135,7 +1135,7 @@ public IgniteInternalFuture rollbackTxAsync(GridNearTxLocal tx) { } /** - * Suspends transaction. It could be resume later. Supported only for optimistic transactions. + * Suspends transaction. It could be resume later. * * @param tx Transaction to suspend. * @throws IgniteCheckedException If suspension failed. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java index 7e218c68581a4..0d76760d1ede9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java @@ -3460,7 +3460,7 @@ public void addKeyMapping(IgniteTxKey key, ClusterNode node) { } /** - * Suspends transaction. It could be resumed later. Supported only for optimistic transactions. + * Suspends transaction. It could be resumed later. * * @throws IgniteCheckedException If the transaction is in an incorrect state, or timed out. */ diff --git a/modules/core/src/main/java/org/apache/ignite/transactions/Transaction.java b/modules/core/src/main/java/org/apache/ignite/transactions/Transaction.java index e81867b9cacf5..5e6d9f61f5b1a 100644 --- a/modules/core/src/main/java/org/apache/ignite/transactions/Transaction.java +++ b/modules/core/src/main/java/org/apache/ignite/transactions/Transaction.java @@ -277,14 +277,14 @@ public interface Transaction extends AutoCloseable, IgniteAsyncSupport { public IgniteFuture rollbackAsync() throws IgniteException; /** - * Resume a transaction if it was previously suspended. Supported only for optimistic transactions. + * Resume a transaction if it was previously suspended. * * @throws IgniteException If resume failed. */ public void resume() throws IgniteException; /** - * Suspends a transaction. It could be resumed later. Supported only for optimistic transactions. + * Suspends a transaction. It could be resumed later. * * @throws IgniteException If suspension failed. */