From a7b57f3e918c28af57e904907140c4d07c02bf24 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Fri, 6 Dec 2024 10:44:24 -0500 Subject: [PATCH 1/4] HDFS-17685: Option to explicitly choose DFS client lease renewal interval --- .../hdfs/client/HdfsClientConfigKeys.java | 3 + .../hdfs/client/impl/DfsClientConf.java | 8 +++ .../hadoop/hdfs/client/impl/LeaseRenewer.java | 46 ++++++++++----- .../hdfs/client/impl/TestLeaseRenewer.java | 59 +++++++++++++++++++ .../src/main/resources/hdfs-default.xml | 10 ++++ 5 files changed, 110 insertions(+), 16 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index 2044530506757..2a991b212c70e 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -202,6 +202,9 @@ public interface HdfsClientConfigKeys { long DFS_CLIENT_DEAD_NODE_DETECTION_PROBE_SUSPECT_NODE_INTERVAL_MS_DEFAULT = 300; // 300ms + String DFS_CLIENT_LEASE_RENEWAL_INTERVAL_KEY = "dfs.client.lease.renewal.interval.ms"; + int DFS_CLIENT_LEASE_RENEWAL_INTERVAL_DEFAULT = 0; + // refreshing LocatedBlocks period. A value of 0 disables the feature. String DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_MS_KEY = "dfs.client.refresh.read-block-locations.ms"; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java index 445612f2f8397..da3dac33161fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java @@ -108,6 +108,7 @@ public class DfsClientConf { .class); private final int hdfsTimeout; // timeout value for a DFS operation. + private final int leaseRenewalIntervalMs; private final int maxFailoverAttempts; private final int maxRetryAttempts; @@ -170,6 +171,9 @@ public class DfsClientConf { public DfsClientConf(Configuration conf) { // The hdfsTimeout is currently the same as the ipc timeout hdfsTimeout = Client.getRpcTimeout(conf); + leaseRenewalIntervalMs = conf.getInt( + HdfsClientConfigKeys.DFS_CLIENT_LEASE_RENEWAL_INTERVAL_KEY, + HdfsClientConfigKeys.DFS_CLIENT_LEASE_RENEWAL_INTERVAL_DEFAULT); maxRetryAttempts = conf.getInt( Retry.MAX_ATTEMPTS_KEY, @@ -431,6 +435,10 @@ public int getHdfsTimeout() { return hdfsTimeout; } + public int getLeaseRenewalIntervalMs() { + return leaseRenewalIntervalMs; + } + /** * @return the maxFailoverAttempts */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java index 33e40764a5365..6a3fccf931e2b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.net.SocketTimeoutException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -248,14 +249,7 @@ private synchronized void addClient(final DFSClient dfsc) { //client not found, add it dfsclients.add(dfsc); - //update renewal time - final int hdfsTimeout = dfsc.getConf().getHdfsTimeout(); - if (hdfsTimeout > 0) { - final long half = hdfsTimeout/2; - if (half < renewal) { - this.renewal = half; - } - } + renewal = getNewRenewalIntervalMs(dfsclients); } private synchronized void clearClients() { @@ -378,17 +372,37 @@ public synchronized void closeClient(final DFSClient dfsc) { } } - //update renewal time - if (renewal == dfsc.getConf().getHdfsTimeout()/2) { - long min = HdfsConstants.LEASE_SOFTLIMIT_PERIOD; - for(DFSClient c : dfsclients) { - final int timeout = c.getConf().getHdfsTimeout(); - if (timeout > 0 && timeout < min) { - min = timeout; + renewal = getNewRenewalIntervalMs(dfsclients); + } + + @VisibleForTesting + static long getNewRenewalIntervalMs(Collection dfsClients) { + // Update renewal time to the first applicable of: + // 1. Requested renewal time amongst all DFSClients, if requested and smaller than the default renewal interval + // 2. Half the HDFS timeout amongst all DFSClients, if requested and smaller than the default renewal interval + // 3. Default renewal time of HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2 + // #2 exists because users with small timeouts want to find out quickly when a NameNode dies, and a small lease renewal interval will help to inform them quickly. See HDFS-278. + long min = HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2; + for (DFSClient c : dfsClients) { + final int newLeaseRenewalInterval = c.getConf().getLeaseRenewalIntervalMs(); + if (newLeaseRenewalInterval > 0 && newLeaseRenewalInterval < min) { + min = newLeaseRenewalInterval; + } + } + if (min <= HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2) { + return min; + } + + for (DFSClient c : dfsClients) { + final int hdfsTimeout = c.getConf().getHdfsTimeout(); + if (hdfsTimeout > 0) { + final long half = hdfsTimeout/2; + if (half < min) { + min = half; } } - renewal = min/2; } + return min; } public void interruptAndJoin() throws InterruptedException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/client/impl/TestLeaseRenewer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/client/impl/TestLeaseRenewer.java index f1a11edeefcd1..62b41d7886430 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/client/impl/TestLeaseRenewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/client/impl/TestLeaseRenewer.java @@ -20,6 +20,7 @@ import java.util.function.Supplier; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSOutputStream; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; @@ -37,7 +38,10 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; + import static org.junit.Assert.assertSame; +import static org.mockito.Mockito.doReturn; public class TestLeaseRenewer { private final String FAKE_AUTHORITY="hdfs://nn1/"; @@ -284,4 +288,59 @@ private static int countThreadMatching(Pattern pattern) { } return count; } + + @Test + public void testDefaultRenewalInterval() { + final DfsClientConf mockConf = Mockito.mock(DfsClientConf.class); + doReturn(0).when(mockConf).getLeaseRenewalIntervalMs(); + doReturn(0).when(mockConf).getHdfsTimeout(); + DFSClient dfsClient = Mockito.mock(DFSClient.class); + doReturn(mockConf).when(dfsClient).getConf(); + // Use default renewal interval if both explicit renewal interval and HDFS timeout are not set + Assert.assertEquals(HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2, LeaseRenewer.getNewRenewalIntervalMs(ImmutableList.of(dfsClient))); + } + + @Test + public void testShortExplicitRenewalInterval() { + final DfsClientConf mockConf = Mockito.mock(DfsClientConf.class); + doReturn(37).when(mockConf).getLeaseRenewalIntervalMs(); + doReturn(100).when(mockConf).getHdfsTimeout(); + DFSClient dfsClient = Mockito.mock(DFSClient.class); + doReturn(mockConf).when(dfsClient).getConf(); + // Explicit renewal interval is shorter than half the HDFS timeout, renewer should use it + Assert.assertEquals(37, LeaseRenewer.getNewRenewalIntervalMs(ImmutableList.of(dfsClient))); + } + + @Test + public void testMediumExplicitRenewalInterval() { + final DfsClientConf mockConf = Mockito.mock(DfsClientConf.class); + doReturn(370).when(mockConf).getLeaseRenewalIntervalMs(); + doReturn(100).when(mockConf).getHdfsTimeout(); + DFSClient dfsClient = Mockito.mock(DFSClient.class); + doReturn(mockConf).when(dfsClient).getConf(); + // Explicit renewal interval is longer than half the HDFS timeout, renewer should use it regardless + Assert.assertEquals(370, LeaseRenewer.getNewRenewalIntervalMs(ImmutableList.of(dfsClient))); + } + + @Test + public void testLongExplicitRenewalIntervalWithShortHdfsTimeout() { + final DfsClientConf mockConf = Mockito.mock(DfsClientConf.class); + doReturn(37000).when(mockConf).getLeaseRenewalIntervalMs(); + doReturn(100).when(mockConf).getHdfsTimeout(); + DFSClient dfsClient = Mockito.mock(DFSClient.class); + doReturn(mockConf).when(dfsClient).getConf(); + // Explicit renewal interval is longer than HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2, but half the HDFS timeout is shorter, so renewer should use the HDFS timeout + Assert.assertEquals(50, LeaseRenewer.getNewRenewalIntervalMs(ImmutableList.of(dfsClient))); + } + + @Test + public void testLongExplicitRenewalIntervalWithLongHdfsTimeout() { + final DfsClientConf mockConf = Mockito.mock(DfsClientConf.class); + doReturn(37000).when(mockConf).getLeaseRenewalIntervalMs(); + doReturn(120000).when(mockConf).getHdfsTimeout(); + DFSClient dfsClient = Mockito.mock(DFSClient.class); + doReturn(mockConf).when(dfsClient).getConf(); + // Explicit renewal interval and HDFS timeout is longer than HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2, so renewer should use default renewal interval + Assert.assertEquals(HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2, LeaseRenewer.getNewRenewalIntervalMs(ImmutableList.of(dfsClient))); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 2ab25f8329ce6..969f89219484f 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -3444,6 +3444,16 @@ + + dfs.client.lease.renewal.interval.ms + 0 + + If set between 0 and 30000, HDFS clients will renew leases for files they are writing at this interval. + If dfs.client.lease.renewal.interval.ms is not set and ipc.client.rpc-timeout.ms is set between 0 and 60000, + then the value of ipc.client.rpc-timeout.ms / 2 will be used as the lease renewal interval. + + + dfs.client.refresh.read-block-locations.ms 0 From 940949ae5aebd8259108c1631f9cc353b6876048 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Sat, 7 Dec 2024 11:16:51 -0500 Subject: [PATCH 2/4] off-by-one bug --- .../org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java | 6 ++++-- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java index 6a3fccf931e2b..2b13ba8c7b9da 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java @@ -383,13 +383,15 @@ static long getNewRenewalIntervalMs(Collection dfsClients) { // 3. Default renewal time of HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2 // #2 exists because users with small timeouts want to find out quickly when a NameNode dies, and a small lease renewal interval will help to inform them quickly. See HDFS-278. long min = HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2; + boolean leaseOverrideSet = false; for (DFSClient c : dfsClients) { final int newLeaseRenewalInterval = c.getConf().getLeaseRenewalIntervalMs(); - if (newLeaseRenewalInterval > 0 && newLeaseRenewalInterval < min) { + if (newLeaseRenewalInterval > 0 && newLeaseRenewalInterval <= min) { min = newLeaseRenewalInterval; + leaseOverrideSet = true; } } - if (min <= HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2) { + if (leaseOverrideSet) { return min; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 969f89219484f..4cc5ae9636ff9 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -3448,7 +3448,7 @@ dfs.client.lease.renewal.interval.ms 0 - If set between 0 and 30000, HDFS clients will renew leases for files they are writing at this interval. + If set between 0 and 30000 inclusive, HDFS clients will renew leases for files they are writing at this interval. If dfs.client.lease.renewal.interval.ms is not set and ipc.client.rpc-timeout.ms is set between 0 and 60000, then the value of ipc.client.rpc-timeout.ms / 2 will be used as the lease renewal interval. From ab59f2d807b83a6c1540b912a326d31de20fc302 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Sat, 7 Dec 2024 11:19:03 -0500 Subject: [PATCH 3/4] checkstyle --- .../hadoop/hdfs/client/impl/LeaseRenewer.java | 9 ++++--- .../hdfs/client/impl/TestLeaseRenewer.java | 24 ++++++++++++------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java index 2b13ba8c7b9da..11a302085fdd3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java @@ -378,10 +378,13 @@ public synchronized void closeClient(final DFSClient dfsc) { @VisibleForTesting static long getNewRenewalIntervalMs(Collection dfsClients) { // Update renewal time to the first applicable of: - // 1. Requested renewal time amongst all DFSClients, if requested and smaller than the default renewal interval - // 2. Half the HDFS timeout amongst all DFSClients, if requested and smaller than the default renewal interval + // 1. Requested renewal time amongst all DFSClients, if requested and smaller than the default + // renewal interval + // 2. Half the HDFS timeout amongst all DFSClients, if requested and smaller than the default + // renewal interval // 3. Default renewal time of HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2 - // #2 exists because users with small timeouts want to find out quickly when a NameNode dies, and a small lease renewal interval will help to inform them quickly. See HDFS-278. + // #2 exists because users with small timeouts want to find out quickly when a NameNode dies, + // and a small lease renewal interval will help to inform them quickly. See HDFS-278. long min = HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2; boolean leaseOverrideSet = false; for (DFSClient c : dfsClients) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/client/impl/TestLeaseRenewer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/client/impl/TestLeaseRenewer.java index 62b41d7886430..172a8ac736ecc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/client/impl/TestLeaseRenewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/client/impl/TestLeaseRenewer.java @@ -297,7 +297,8 @@ public void testDefaultRenewalInterval() { DFSClient dfsClient = Mockito.mock(DFSClient.class); doReturn(mockConf).when(dfsClient).getConf(); // Use default renewal interval if both explicit renewal interval and HDFS timeout are not set - Assert.assertEquals(HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2, LeaseRenewer.getNewRenewalIntervalMs(ImmutableList.of(dfsClient))); + Assert.assertEquals(HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2, + LeaseRenewer.getNewRenewalIntervalMs(ImmutableList.of(dfsClient))); } @Test @@ -308,7 +309,8 @@ public void testShortExplicitRenewalInterval() { DFSClient dfsClient = Mockito.mock(DFSClient.class); doReturn(mockConf).when(dfsClient).getConf(); // Explicit renewal interval is shorter than half the HDFS timeout, renewer should use it - Assert.assertEquals(37, LeaseRenewer.getNewRenewalIntervalMs(ImmutableList.of(dfsClient))); + Assert.assertEquals(37, + LeaseRenewer.getNewRenewalIntervalMs(ImmutableList.of(dfsClient))); } @Test @@ -318,8 +320,10 @@ public void testMediumExplicitRenewalInterval() { doReturn(100).when(mockConf).getHdfsTimeout(); DFSClient dfsClient = Mockito.mock(DFSClient.class); doReturn(mockConf).when(dfsClient).getConf(); - // Explicit renewal interval is longer than half the HDFS timeout, renewer should use it regardless - Assert.assertEquals(370, LeaseRenewer.getNewRenewalIntervalMs(ImmutableList.of(dfsClient))); + // Explicit renewal interval is longer than half the HDFS timeout, + // renewer should use it regardless + Assert.assertEquals(370, + LeaseRenewer.getNewRenewalIntervalMs(ImmutableList.of(dfsClient))); } @Test @@ -329,8 +333,10 @@ public void testLongExplicitRenewalIntervalWithShortHdfsTimeout() { doReturn(100).when(mockConf).getHdfsTimeout(); DFSClient dfsClient = Mockito.mock(DFSClient.class); doReturn(mockConf).when(dfsClient).getConf(); - // Explicit renewal interval is longer than HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2, but half the HDFS timeout is shorter, so renewer should use the HDFS timeout - Assert.assertEquals(50, LeaseRenewer.getNewRenewalIntervalMs(ImmutableList.of(dfsClient))); + // Explicit renewal interval is longer than HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2, + // but half the HDFS timeout is shorter, so renewer should use the HDFS timeout + Assert.assertEquals(50, + LeaseRenewer.getNewRenewalIntervalMs(ImmutableList.of(dfsClient))); } @Test @@ -340,7 +346,9 @@ public void testLongExplicitRenewalIntervalWithLongHdfsTimeout() { doReturn(120000).when(mockConf).getHdfsTimeout(); DFSClient dfsClient = Mockito.mock(DFSClient.class); doReturn(mockConf).when(dfsClient).getConf(); - // Explicit renewal interval and HDFS timeout is longer than HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2, so renewer should use default renewal interval - Assert.assertEquals(HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2, LeaseRenewer.getNewRenewalIntervalMs(ImmutableList.of(dfsClient))); + // Explicit renewal interval and HDFS timeout is longer than + // HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2, so renewer should use default renewal interval + Assert.assertEquals(HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2, + LeaseRenewer.getNewRenewalIntervalMs(ImmutableList.of(dfsClient))); } } From 463a78f16fe92479e2fe818c650b7f69cf82423a Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Sun, 8 Dec 2024 09:49:26 -0500 Subject: [PATCH 4/4] no nice way to make this method shorter --- .../java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java index da3dac33161fe..1316e5e8f89c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java @@ -174,7 +174,6 @@ public DfsClientConf(Configuration conf) { leaseRenewalIntervalMs = conf.getInt( HdfsClientConfigKeys.DFS_CLIENT_LEASE_RENEWAL_INTERVAL_KEY, HdfsClientConfigKeys.DFS_CLIENT_LEASE_RENEWAL_INTERVAL_DEFAULT); - maxRetryAttempts = conf.getInt( Retry.MAX_ATTEMPTS_KEY, Retry.MAX_ATTEMPTS_DEFAULT); @@ -187,7 +186,6 @@ public DfsClientConf(Configuration conf) { retryIntervalForGetLastBlockLength = conf.getInt( Retry.INTERVAL_GET_LAST_BLOCK_LENGTH_KEY, Retry.INTERVAL_GET_LAST_BLOCK_LENGTH_DEFAULT); - maxFailoverAttempts = conf.getInt( Failover.MAX_ATTEMPTS_KEY, Failover.MAX_ATTEMPTS_DEFAULT);