feat(server): support new backend Hstore (apache#2560)

subtask of apache#2483

.github/workflows/pd-store-ci.yml

@@ -116,3 +116,73 @@ jobs:
  uses: codecov/codecov-action@v3.0.0
  with:
  file: ${{ env.REPORT_DIR }}/*.xml
+
+ hstore:
+ # TODO: avoid duplicated env setup
+ runs-on: ubuntu-latest
+ env:
+ USE_STAGE: 'true' # Whether to include the stage repository.
+ TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis
+ REPORT_DIR: target/site/jacoco
+ BACKEND: hstore
+ RELEASE_BRANCH: ${{ startsWith(github.ref_name, 'release-') || startsWith(github.ref_name, 'test-') || startsWith(github.base_ref, 'release-') }}
+
+ steps:
+ - name: Install JDK 11
+ uses: actions/setup-java@v3
+ with:
+ java-version: '11'
+ distribution: 'zulu'
+
+ - name: Cache Maven packages
+ uses: actions/cache@v3
+ with:
+ path: ~/.m2
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ runner.os }}-m2
+
+ - name: Checkout
+ uses: actions/checkout@v3
+ with:
+ fetch-depth: 2
+
+ - name: use staged maven repo settings
+ if: ${{ env.USE_STAGE == 'true' }}
+ run: |
+ cp $HOME/.m2/settings.xml /tmp/settings.xml
+ mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml
+
+ - name: Package
+ run: |
+ mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp
+
+ - name: Prepare env and service
+ run: |
+ $TRAVIS_DIR/install-backend.sh $BACKEND
+
+ - name: Run unit test
+ run: |
+ $TRAVIS_DIR/run-unit-test.sh $BACKEND
+
+ - name: Run core test
+ run: |
+ $TRAVIS_DIR/run-core-test.sh $BACKEND
+
+ - name: Run api test
+ run: |
+ $TRAVIS_DIR/run-api-test.sh $BACKEND $REPORT_DIR
+
+ - name: Run raft test
+ if: ${{ env.BACKEND == 'rocksdb' }}
+ run: |
+ $TRAVIS_DIR/run-api-test-for-raft.sh $BACKEND $REPORT_DIR
+
+ - name: Run TinkerPop test
+ if: ${{ env.RELEASE_BRANCH == 'true' }}
+ run: |
+ $TRAVIS_DIR/run-tinkerpop-test.sh $BACKEND tinkerpop
+
+ - name: Upload coverage to Codecov
+ uses: codecov/codecov-action@v3.0.0
+ with:
+ file: ${{ env.REPORT_DIR }}/*.xml

hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/job/TaskAPI.java

@@ -136,11 +136,12 @@ public Map<String, Object> get(@Context GraphManager manager,
  @RedirectFilter.RedirectMasterRole
  public void delete(@Context GraphManager manager,
  @PathParam("graph") String graph,
- @PathParam("id") long id) {
+ @PathParam("id") long id,
+ @DefaultValue("false") @QueryParam("force") boolean force) {
  LOG.debug("Graph [{}] delete task: {}", graph, id);
 
  TaskScheduler scheduler = graph(manager, graph).taskScheduler();
- HugeTask<?> task = scheduler.delete(IdGenerator.of(id));
+ HugeTask<?> task = scheduler.delete(IdGenerator.of(id), force);
  E.checkArgument(task != null, "There is no task with id '%s'", id);
  }
 

hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java

@@ -26,6 +26,7 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.Callable;
 import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadFactory;
@@ -71,6 +72,7 @@
 import org.apache.hugegraph.structure.HugeFeatures;
 import org.apache.hugegraph.structure.HugeVertex;
 import org.apache.hugegraph.task.HugeTask;
+import org.apache.hugegraph.task.ServerInfoManager;
 import org.apache.hugegraph.task.TaskManager;
 import org.apache.hugegraph.task.TaskScheduler;
 import org.apache.hugegraph.task.TaskStatus;
@@ -1085,10 +1087,10 @@ public <V> Iterator<HugeTask<V>> tasks(TaskStatus status,
  }
 
  @Override
- public <V> HugeTask<V> delete(Id id) {
+ public <V> HugeTask<V> delete(Id id, boolean force) {
  verifyTaskPermission(HugePermission.DELETE,
  this.taskScheduler.task(id));
- return this.taskScheduler.delete(id);
+ return this.taskScheduler.delete(id, force);
  }
 
  @Override
@@ -1124,6 +1126,36 @@ public void checkRequirement(String op) {
  this.taskScheduler.checkRequirement(op);
  }
 
+ @Override
+ public <V> V call(Callable<V> callable) {
+ verifyAnyPermission();
+ return this.taskScheduler.call(callable);
+ }
+
+ @Override
+ public <V> V call(Runnable runnable) {
+ verifyAnyPermission();
+ return this.taskScheduler.call(runnable);
+ }
+
+ @Override
+ public ServerInfoManager serverManager() {
+ verifyAnyPermission();
+ return this.taskScheduler.serverManager();
+ }
+
+ @Override
+ public String graphName() {
+ verifyAnyPermission();
+ return this.taskScheduler.graphName();
+ }
+
+ @Override
+ public void taskDone(HugeTask<?> task) {
+ verifyAnyPermission();
+ this.taskScheduler.taskDone(task);
+ }
+
  private void verifyTaskPermission(HugePermission actionPerm) {
  verifyPermission(actionPerm, ResourceType.TASK);
  }

hugegraph-server/hugegraph-cassandra/src/main/java/org/apache/hugegraph/backend/store/cassandra/CassandraStore.java

@@ -592,7 +592,7 @@ protected Collection<CassandraTable> tables() {
 
  @Override
  protected final CassandraTable table(HugeType type) {
- return this.table(type.string());
+ return this.table(convertTaskOrServerToVertex(type).string());
  }
 
  protected final CassandraTable table(String name) {

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeGraphParams.java

@@ -24,6 +24,7 @@
 import org.apache.hugegraph.backend.store.BackendStore;
 import org.apache.hugegraph.backend.store.ram.RamTable;
 import org.apache.hugegraph.backend.tx.GraphTransaction;
+import org.apache.hugegraph.backend.tx.ISchemaTransaction;
 import org.apache.hugegraph.backend.tx.SchemaTransaction;
 import org.apache.hugegraph.config.HugeConfig;
 import org.apache.hugegraph.event.EventHub;
@@ -47,7 +48,7 @@ public interface HugeGraphParams {
 
  GraphReadMode readMode();
 
- SchemaTransaction schemaTransaction();
+ ISchemaTransaction schemaTransaction();
 
  GraphTransaction systemTransaction();
 
@@ -94,4 +95,6 @@ public interface HugeGraphParams {
  RamTable ramtable();
 
  <T> void submitEphemeralJob(EphemeralJob<T> job);
+
+ String schedulerType();
 }

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java

@@ -37,6 +37,7 @@
 import org.apache.hugegraph.backend.cache.CacheNotifier.SchemaCacheNotifier;
 import org.apache.hugegraph.backend.cache.CachedGraphTransaction;
 import org.apache.hugegraph.backend.cache.CachedSchemaTransaction;
+import org.apache.hugegraph.backend.cache.CachedSchemaTransactionV2;
 import org.apache.hugegraph.backend.id.Id;
 import org.apache.hugegraph.backend.id.IdGenerator;
 import org.apache.hugegraph.backend.id.SnowflakeIdGenerator;
@@ -52,7 +53,7 @@
 import org.apache.hugegraph.backend.store.raft.RaftGroupManager;
 import org.apache.hugegraph.backend.store.ram.RamTable;
 import org.apache.hugegraph.backend.tx.GraphTransaction;
-import org.apache.hugegraph.backend.tx.SchemaTransaction;
+import org.apache.hugegraph.backend.tx.ISchemaTransaction;
 import org.apache.hugegraph.config.CoreOptions;
 import org.apache.hugegraph.config.HugeConfig;
 import org.apache.hugegraph.config.TypedOption;
@@ -69,6 +70,7 @@
 import org.apache.hugegraph.masterelection.RoleElectionStateMachine;
 import org.apache.hugegraph.masterelection.StandardClusterRoleStore;
 import org.apache.hugegraph.masterelection.StandardRoleElectionStateMachine;
+import org.apache.hugegraph.meta.MetaManager;
 import org.apache.hugegraph.perf.PerfUtil.Watched;
 import org.apache.hugegraph.rpc.RpcServiceConfig4Client;
 import org.apache.hugegraph.rpc.RpcServiceConfig4Server;
@@ -176,6 +178,8 @@ public class StandardHugeGraph implements HugeGraph {
 
  private final RamTable ramtable;
 
+ private final String schedulerType;
+
  public StandardHugeGraph(HugeConfig config) {
  this.params = new StandardHugeGraphParams();
  this.configuration = config;
@@ -209,6 +213,7 @@ public StandardHugeGraph(HugeConfig config) {
  this.closed = false;
  this.mode = GraphMode.NONE;
  this.readMode = GraphReadMode.OLTP_ONLY;
+ this.schedulerType = config.get(CoreOptions.SCHEDULER_TYPE);
 
  LockUtil.init(this.name);
 
@@ -221,6 +226,13 @@ public StandardHugeGraph(HugeConfig config) {
  throw new HugeException(message, e);
  }
 
+ if (isHstore()) {
+ // TODO: parameterize the remaining configurations
+ MetaManager.instance().connect("hg", MetaManager.MetaDriverType.PD,
+ "ca", "ca", "ca",
+ config.get(CoreOptions.PD_PEERS));
+ }
+
  try {
  this.tx = new TinkerPopTransaction(this);
  boolean supportsPersistence = this.backendStoreFeatures().supportsPersistence();
@@ -457,9 +469,18 @@ private void clearVertexCache() {
  }
  }
 
- private SchemaTransaction openSchemaTransaction() throws HugeException {
+ private boolean isHstore() {
+ return this.storeProvider.isHstore();
+ }
+
+ private ISchemaTransaction openSchemaTransaction() throws HugeException {
  this.checkGraphNotClosed();
  try {
+ if (isHstore()) {
+ return new CachedSchemaTransactionV2(
+ MetaManager.instance().metaDriver(),
+ MetaManager.instance().cluster(), this.params);
+ }
  return new CachedSchemaTransaction(this.params, loadSchemaStore());
  } catch (BackendException e) {
  String message = "Failed to open schema transaction";
@@ -504,11 +525,14 @@ private BackendStore loadGraphStore() {
  }
 
  private BackendStore loadSystemStore() {
+ if (isHstore()) {
+ return this.storeProvider.loadGraphStore(this.configuration);
+ }
  return this.storeProvider.loadSystemStore(this.configuration);
  }
 
  @Watched
- private SchemaTransaction schemaTransaction() {
+ private ISchemaTransaction schemaTransaction() {
  this.checkGraphNotClosed();
  /*
  * NOTE: each schema operation will be auto committed,
@@ -1196,7 +1220,7 @@ public GraphReadMode readMode() {
  }
 
  @Override
- public SchemaTransaction schemaTransaction() {
+ public ISchemaTransaction schemaTransaction() {
  return StandardHugeGraph.this.schemaTransaction();
  }
 
@@ -1316,6 +1340,11 @@ public RamTable ramtable() {
  public <T> void submitEphemeralJob(EphemeralJob<T> job) {
  this.ephemeralJobQueue.add(job);
  }
+
+ @Override
+ public String schedulerType() {
+ return StandardHugeGraph.this.schedulerType;
+ }
  }
 
  private class TinkerPopTransaction extends AbstractThreadLocalTransaction {
@@ -1447,7 +1476,7 @@ private void setClosed() {
  }
  }
 
- private SchemaTransaction schemaTransaction() {
+ private ISchemaTransaction schemaTransaction() {
  return this.getOrNewTransaction().schemaTx;
  }
 
@@ -1468,7 +1497,7 @@ private Txs getOrNewTransaction() {
 
  Txs txs = this.transactions.get();
  if (txs == null) {
- SchemaTransaction schemaTransaction = null;
+ ISchemaTransaction schemaTransaction = null;
  SysTransaction sysTransaction = null;
  GraphTransaction graphTransaction = null;
  try {
@@ -1511,12 +1540,12 @@ private void destroyTransaction() {
 
  private static final class Txs {
 
- private final SchemaTransaction schemaTx;
+ private final ISchemaTransaction schemaTx;
  private final SysTransaction systemTx;
  private final GraphTransaction graphTx;
  private long openedTime;
 
- public Txs(SchemaTransaction schemaTx, SysTransaction systemTx,
+ public Txs(ISchemaTransaction schemaTx, SysTransaction systemTx,
  GraphTransaction graphTx) {
  assert schemaTx != null && systemTx != null && graphTx != null;
  this.schemaTx = schemaTx;

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/auth/HugeAccess.java

@@ -228,4 +228,9 @@ private String[] initProperties() {
  return super.initProperties(props);
  }
  }
+
+ public static HugeAccess fromMap(Map<String, Object> map) {
+ HugeAccess access = new HugeAccess(null, null, null);
+ return fromMap(map, access);
+ }
 }

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/auth/HugeBelong.java

@@ -32,10 +32,15 @@
 
 public class HugeBelong extends Relationship {
 
+ public static final String UG = "ug";
+ public static final String UR = "ur";
+ public static final String GR = "gr";
+ public static final String ALL = "*";
  private static final long serialVersionUID = -7242751631755533423L;
 
  private final Id user;
  private final Id group;
+ private String link;
  private String description;
 
  public HugeBelong(Id user, Id group) {
@@ -74,6 +79,10 @@ public Id target() {
  return this.group;
  }
 
+ public String link() {
+ return this.link;
+ }
+
  public String description() {
  return this.description;
  }
@@ -193,4 +202,9 @@ private String[] initProperties() {
  return super.initProperties(props);
  }
  }
+
+ public static HugeBelong fromMap(Map<String, Object> map) {
+ HugeBelong belong = new HugeBelong(null, null);
+ return fromMap(map, belong);
+ }
 }