Skip to content

Commit

Permalink
777777
Browse files Browse the repository at this point in the history
  • Loading branch information
@SWJTU-ZhangLei
SWJTU-ZhangLei committed Nov 23, 2023
1 parent b726036 commit c8155a9
Show file tree
Hide file tree
Showing 2 changed files with 51 additions and 100 deletions.
2 changes: 1 addition & 1 deletion fe/fe-common/src/main/java/org/apache/doris/common/Config.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2002,7 +2002,7 @@ public class Config extends ConfigBase {
* OFF, SEVERE, WARNING, INFO, CONFIG, FINE, FINER, FINEST, ALL
*/
@ConfField
public static String bdbje_file_logging_level = "ALL";
public static String bdbje_file_logging_level = "INFO";

/**
* When holding lock time exceeds the threshold, need to report it.
Expand Down
149 changes: 50 additions & 99 deletions fe/fe-core/src/test/java/org/apache/doris/journal/bdbje/BDBEnvironmentTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,9 @@
import com.google.common.base.Strings;
import com.sleepycat.je.Database;
import com.sleepycat.je.DatabaseEntry;
// import com.sleepycat.je.EnvironmentConfig;
// import com.sleepycat.je.EnvironmentMutableConfig;
import com.sleepycat.je.LockMode;
import com.sleepycat.je.OperationStatus;
import com.sleepycat.je.rep.ReplicatedEnvironment;
import com.sleepycat.je.rep.ReplicationConfig;
import com.sleepycat.je.rep.ReplicationMutableConfig;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand All @@ -52,7 +48,6 @@
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
// import java.util.concurrent.TimeUnit;

public class BDBEnvironmentTest {
private static final Logger LOG = LogManager.getLogger(BDBEnvironmentTest.class);
Expand All @@ -70,7 +65,7 @@ public static String createTmpDir() throws Exception {

@BeforeAll
public static void startUp() throws Exception {
Config.bdbje_heartbeat_timeout_second = 3;
Config.bdbje_heartbeat_timeout_second = 10;
}

@AfterAll
Expand Down Expand Up @@ -268,7 +263,7 @@ private Pair<BDBEnvironment, NodeInfo> findMaster(List<Pair<BDBEnvironment, Node
}

// @Test
@RepeatedTest(30)
@RepeatedTest(1)
public void testRollbackException() throws Exception {
LOG.info("start");
List<Pair<BDBEnvironment, NodeInfo>> followersInfo = new ArrayList<>();
Expand All @@ -293,45 +288,16 @@ public void testRollbackException() throws Exception {
followersInfo.add(Pair.of(followerEnvironment, new NodeInfo(nodeName, nodeHostPort, nodeDir)));
}

// followersInfo.stream().forEach(entryPair -> {
// EnvironmentMutableConfig config = entryPair.first.getReplicatedEnvironment().getMutableConfig();
// config.setConfigParam(EnvironmentConfig.ENV_RUN_CLEANER, "false");
// config.setConfigParam(EnvironmentConfig.ENV_RUN_CHECKPOINTER, "false");
// entryPair.first.getReplicatedEnvironment().setMutableConfig(config);

// ReplicationMutableConfig repConfig = entryPair.first.getReplicatedEnvironment().getRepMutableConfig();
// repConfig.setConfigParam(ReplicationConfig.ENV_UNKNOWN_STATE_TIMEOUT, "3 s");
// entryPair.first.getReplicatedEnvironment().setRepMutableConfig(repConfig);
// LOG.info("ReplicationConfig.ENV_UNKNOWN_STATE_TIMEOUT: {}", repConfig.getConfigParam(ReplicationConfig.ENV_UNKNOWN_STATE_TIMEOUT));

// entryPair.first.close();
// });


// for (Pair<BDBEnvironment, NodeInfo> entryPair : followersInfo) {
// entryPair.first.openReplicatedEnvironment(new File(entryPair.second.dir));
// EnvironmentMutableConfig config = entryPair.first.getReplicatedEnvironment().getMutableConfig();
// config.setConfigParam(EnvironmentConfig.ENV_RUN_CLEANER, "false");
// config.setConfigParam(EnvironmentConfig.ENV_RUN_CHECKPOINTER, "false");
// entryPair.first.getReplicatedEnvironment().setMutableConfig(config);

// ReplicationMutableConfig repConfig = entryPair.first.getReplicatedEnvironment().getRepMutableConfig();
// repConfig.setConfigParam(ReplicationConfig.ENV_UNKNOWN_STATE_TIMEOUT, "3 s");
// entryPair.first.getReplicatedEnvironment().setRepMutableConfig(repConfig);
// LOG.info("ReplicationConfig.ENV_UNKNOWN_STATE_TIMEOUT: {}", repConfig.getConfigParam(ReplicationConfig.ENV_UNKNOWN_STATE_TIMEOUT));
// }

Pair<BDBEnvironment, NodeInfo> masterPair = findMaster(followersInfo);

String beginDbName = String.valueOf(0L);
Database masterDb = masterPair.first.openDatabase(beginDbName);
DatabaseEntry key = new DatabaseEntry(randomBytes());
DatabaseEntry value = new DatabaseEntry(randomBytes());
Assertions.assertEquals(OperationStatus.SUCCESS, masterDb.put(null, key, value));
Assertions.assertEquals(1, masterEnvironment.getDatabaseNames().size());
LOG.info("master is {}|{}", masterPair.second.name, masterPair.second.dir);
LOG.info("master is {} | {}", masterPair.second.name, masterPair.second.dir);

followersInfo.stream().forEach(entryPair -> {
for (Pair<BDBEnvironment, NodeInfo> entryPair : followersInfo) {
if (entryPair.second.dir.equals(masterPair.second.dir)) {
LOG.info("skip {}", entryPair.second.name);
return;
Expand All @@ -343,7 +309,7 @@ public void testRollbackException() throws Exception {
Assertions.assertEquals(OperationStatus.SUCCESS, followerDb.get(null, key, readValue, LockMode.READ_COMMITTED));
Assertions.assertEquals(new String(value.getData()), new String(readValue.getData()));
followerDb.close();
});
}

masterDb.close();
masterEnvironment.getEpochDB().close();
Expand All @@ -352,74 +318,59 @@ public void testRollbackException() throws Exception {
entryPair.first.close();
LOG.info("close {} | {}", entryPair.second.name, entryPair.second.dir);
});
// masterEnvironment.getReplicatedEnvironment().shutdownGroup(20, TimeUnit.SECONDS);

// all follower closed
// for (Pair<BDBEnvironment, NodeInfo> entryPair : followersInfo) {
// String followerCopyDir = entryPair.second.dir + "_copy";
// LOG.info("Copy from {} to {}", entryPair.second.dir, followerCopyDir);
// FileUtils.copyDirectory(new File(entryPair.second.dir), new File(followerCopyDir));
// }

// followersInfo.stream().forEach(entryPair -> {
// entryPair.first.openReplicatedEnvironment(new File(entryPair.second.dir));
// });

// masterEnvironment = null;
// masterDir = null;
// for (Pair<BDBEnvironment, NodeInfo> entryPair : followersInfo) {
// if (entryPair.first.getReplicatedEnvironment().getState().equals(ReplicatedEnvironment.State.MASTER)) {
// masterEnvironment = entryPair.first;
// masterDir = entryPair.second.dir;
// LOG.info("masterDir transfer to {} | {}", entryPair.first.getReplicatedEnvironment().getNodeName(),
// entryPair.second.dir);
// }
// }
// Assertions.assertNotNull(masterEnvironment);
// Assertions.assertNotNull(masterDir);

// // masterDb = masterEnvironment.openDatabase(String.valueOf(1L));
// // // for (int i = 0; i < Config.txn_rollback_limit + 10; i++) {
// // for (int i = 0; i < 10; i++) {
// // OperationStatus status = masterDb.put(null, new DatabaseEntry(randomBytes()), new DatabaseEntry(randomBytes()));
// // Assertions.assertEquals(OperationStatus.SUCCESS, status);
// // }
// // Assertions.assertEquals(2, masterEnvironment.getDatabaseNames().size());
// // Assertions.assertEquals(0, masterEnvironment.getDatabaseNames().get(0));
// // Assertions.assertEquals(1, masterEnvironment.getDatabaseNames().get(1));

// // followersInfo.stream().forEach(entryPair -> {
// // entryPair.first.close();
// // });

// // Restore follower's (not new master) bdbje dir
// for (Pair<BDBEnvironment, NodeInfo> entryPair : followersInfo) {
// if (entryPair.second.dir.equals(masterDir)) {
// String masterCopyDir = entryPair.second.dir + "_copy";
// // FileUtils.deleteDirectory(new File(masterCopyDir));
// continue;
// }
// LOG.info("Delete followerDir {} ", entryPair.second.dir);
// // FileUtils.deleteDirectory(new File(entryPair.second.dir));
// FileUtils.moveDirectory(new File(entryPair.second.dir), new File(entryPair.second.dir + "_copy2"));
// String followerCopyDir = entryPair.second.dir + "_copy";
// LOG.info("Move {} to {}", followerCopyDir, entryPair.second.dir);
// FileUtils.moveDirectory(new File(followerCopyDir), new File(entryPair.second.dir));
// }
for (Pair<BDBEnvironment, NodeInfo> entryPair : followersInfo) {
String followerCopyDir = entryPair.second.dir + "_copy";
LOG.info("Copy from {} to {}", entryPair.second.dir, followerCopyDir);
FileUtils.copyDirectory(new File(entryPair.second.dir), new File(followerCopyDir));
}

followersInfo.stream().forEach(entryPair -> {
entryPair.first.openReplicatedEnvironment(new File(entryPair.second.dir));
LOG.info("open {} | {}", entryPair.second.name, entryPair.second.dir);
});

masterPair = findMaster(followersInfo);

masterDb = masterPair.first.openDatabase(String.valueOf(1L));
for (int i = 0; i < Config.txn_rollback_limit + 10; i++) {
// for (int i = 0; i < 10; i++) {
OperationStatus status = masterDb.put(null, new DatabaseEntry(randomBytes()), new DatabaseEntry(randomBytes()));
Assertions.assertEquals(OperationStatus.SUCCESS, status);
}
Assertions.assertEquals(2, masterPair.first.getDatabaseNames().size());
Assertions.assertEquals(0, masterPair.first.getDatabaseNames().get(0));
Assertions.assertEquals(1, masterPair.first.getDatabaseNames().get(1));

followersInfo.stream().forEach(entryPair -> {
entryPair.first.close();
LOG.info("close {} | {}", entryPair.second.name, entryPair.second.dir);
});

// Restore follower's (not new master) bdbje dir
for (Pair<BDBEnvironment, NodeInfo> entryPair : followersInfo) {
if (entryPair.second.dir.equals(masterDir)) {
String masterCopyDir = entryPair.second.dir + "_copy";
FileUtils.deleteDirectory(new File(masterCopyDir));
continue;
}
LOG.info("Delete followerDir {} ", entryPair.second.dir);
FileUtils.deleteDirectory(new File(entryPair.second.dir));
// FileUtils.moveDirectory(new File(entryPair.second.dir), new File(entryPair.second.dir + "_copy2"));
String followerCopyDir = entryPair.second.dir + "_copy";
LOG.info("Move {} to {}", followerCopyDir, entryPair.second.dir);
FileUtils.moveDirectory(new File(followerCopyDir), new File(entryPair.second.dir));
}

Thread.sleep(5000);
for (Pair<BDBEnvironment, NodeInfo> entryPair : followersInfo) {
if (entryPair.second.dir.equals(masterPair.second.dir)) {
LOG.info("skip open {}", entryPair.second.name, entryPair.second.dir);
LOG.info("skip open {} | {}", entryPair.second.name, entryPair.second.dir);
continue;
}
entryPair.first.openReplicatedEnvironment(new File(entryPair.second.dir));
LOG.info("open {}|", entryPair.second.name, entryPair.second.dir);

ReplicationMutableConfig repConfig = entryPair.first.getReplicatedEnvironment().getRepMutableConfig();
repConfig.setConfigParam(ReplicationConfig.ENV_UNKNOWN_STATE_TIMEOUT, "3 s");
entryPair.first.getReplicatedEnvironment().setRepMutableConfig(repConfig);
LOG.info("ReplicationConfig.ENV_UNKNOWN_STATE_TIMEOUT: {}", repConfig.getConfigParam(ReplicationConfig.ENV_UNKNOWN_STATE_TIMEOUT));
LOG.info("open {} | {}", entryPair.second.name, entryPair.second.dir);
}

BDBEnvironment newMasterEnvironment = null;
Expand Down Expand Up @@ -452,8 +403,8 @@ public void testRollbackException() throws Exception {
// // old master
masterEnvironment.openReplicatedEnvironment(new File(masterDir));
followersInfo.stream().forEach(entryPair -> {
LOG.info("close {} | {}", entryPair.second.name, entryPair.second.dir);
entryPair.first.close();
LOG.info("close {} | {}", entryPair.second.name, entryPair.second.dir);
});
LOG.info("end");
}
Expand Down

0 comments on commit c8155a9

Please sign in to comment.