From 9d8b6920b70d4e428ada54fa65135daf988e7ab4 Mon Sep 17 00:00:00 2001 From: Andreas Eberhart Date: Mon, 7 Oct 2024 19:31:40 +0200 Subject: [PATCH] Analytics can be based on queries and tables --- .../main/java/org/dashjoin/service/Data.java | 47 +++++++----- .../java/org/dashjoin/service/Database.java | 2 +- .../org/dashjoin/service/SQLDatabase.java | 6 +- .../org/dashjoin/service/SQLDatabaseTest.java | 72 +++++++++---------- 4 files changed, 67 insertions(+), 60 deletions(-) diff --git a/dashjoin-core/src/main/java/org/dashjoin/service/Data.java b/dashjoin-core/src/main/java/org/dashjoin/service/Data.java index 71bfba8d4..aa2c3e23c 100644 --- a/dashjoin-core/src/main/java/org/dashjoin/service/Data.java +++ b/dashjoin-core/src/main/java/org/dashjoin/service/Data.java @@ -29,7 +29,6 @@ import org.eclipse.microprofile.openapi.annotations.parameters.Parameter; import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; @@ -309,10 +308,11 @@ public List> queryInternal(SecurityContext sc, String databa if (arguments == null) arguments = new HashMap<>(); - if (arguments.get("__dj_analytics") instanceof List) { + if ("__dj_analytics".equals(queryId)) { AbstractDatabase db = services.getConfig().getDatabase(dj(database)); - String query = getAnalytics(sc, db, queryId, arguments); - return db.query(QueryMeta.ofQuery(query, false), null); + Analytics analytics = om.convertValue(arguments, Analytics.class); + String query = getAnalytics(sc, db, analytics); + return db.query(QueryMeta.ofQuery(query, false), analytics.arguments); } QueryMeta info = services.getConfig().getQueryMeta(queryId); @@ -325,17 +325,22 @@ public List> queryInternal(SecurityContext sc, String databa return db.query(info, arguments); } - String getAnalytics(SecurityContext sc, AbstractDatabase db, String table, - Map arguments) { - List cols = - om.convertValue(arguments.get("__dj_analytics"), new TypeReference>() {}); - Table m = db.tables.get(table); + String getAnalytics(SecurityContext sc, AbstractDatabase db, Analytics a) throws Exception { - if (m == null) - throw new IllegalArgumentException("Unknown table: " + table); + QueryMeta info = null; + if (a.query != null) { + info = services.getConfig().getQueryMeta(a.query); + if ("write".equals(info.type)) + return null; + ACLContainerRequestFilter.check(sc, info); + } + Table m = db.tables.get(a.table); + if (m == null) + throw new IllegalArgumentException("Unknown table: " + a.table); ACLContainerRequestFilter.check(sc, db, m); - for (ColInfo e : cols) { + + for (ColInfo e : a.cols) { if (e.arg1 != null) { Map tmp = MapUtil.of(e.name, e.arg1); db.cast(m, tmp); @@ -355,10 +360,10 @@ String getAnalytics(SecurityContext sc, AbstractDatabase db, String table, ci.name = e.getKey(); ci.filter = Filter.EQUALS; ci.arg1 = e.getValue(); - cols.add(ci); + a.cols.add(ci); } - return db.analytics(m, cols); + return db.analytics(info, m, a.cols); } /** @@ -378,10 +383,11 @@ public Map queryMeta(@Context SecurityContext sc, if (arguments == null) arguments = new HashMap<>(); - if (arguments.get("__dj_analytics") instanceof List) { + if ("__dj_analytics".equals(queryId)) { AbstractDatabase db = services.getConfig().getDatabase(dj(database)); - String query = getAnalytics(sc, db, queryId, arguments); - return db.queryMeta(QueryMeta.ofQuery(query, false), null); + Analytics analytics = om.convertValue(arguments, Analytics.class); + String query = getAnalytics(sc, db, analytics); + return db.queryMeta(QueryMeta.ofQuery(query, false), analytics.arguments); } QueryMeta info = services.getConfig().getQueryMeta(queryId); @@ -869,6 +875,13 @@ public static enum Filter { EQUALS, NOT_EQUALS, LIKE, IS_NULL, IS_NOT_NULL, SMALLER_EQUAL, GREATER_EQUAL, BETWEEN } + public static class Analytics { + public List cols; + public String table; + public String query; + public Map arguments; + } + /** * analytics query column info */ diff --git a/dashjoin-core/src/main/java/org/dashjoin/service/Database.java b/dashjoin-core/src/main/java/org/dashjoin/service/Database.java index 363aab661..94fdaaf00 100644 --- a/dashjoin-core/src/main/java/org/dashjoin/service/Database.java +++ b/dashjoin-core/src/main/java/org/dashjoin/service/Database.java @@ -71,7 +71,7 @@ public List> all(Table s, Integer offset, Integer limit, Str /** * generates ad hoc analytics queries without having to rely on the query editor */ - default public String analytics(Table s, List arguments) { + default public String analytics(QueryMeta info, Table s, List arguments) { return null; } diff --git a/dashjoin-core/src/main/java/org/dashjoin/service/SQLDatabase.java b/dashjoin-core/src/main/java/org/dashjoin/service/SQLDatabase.java index 33d8a7d2d..62a94e5bb 100644 --- a/dashjoin-core/src/main/java/org/dashjoin/service/SQLDatabase.java +++ b/dashjoin-core/src/main/java/org/dashjoin/service/SQLDatabase.java @@ -536,7 +536,7 @@ public List search(SecurityContext sc, Table filter, String search } @Override - public String analytics(Table s, List arguments) { + public String analytics(QueryMeta info, Table s, List arguments) { List project = new ArrayList<>(); boolean isGroupBy = false; @@ -639,7 +639,9 @@ else if (isGroupBy) } } - String select = "select " + String.join(", ", project) + " from " + schema() + q(s.name); + String subselect = info == null ? "" : "(" + info.query + ") as "; + String select = + "select " + String.join(", ", project) + " from " + subselect + schema() + q(s.name); if (!where.isEmpty()) select += " where " + String.join(" and ", where); diff --git a/dashjoin-core/src/test/java/org/dashjoin/service/SQLDatabaseTest.java b/dashjoin-core/src/test/java/org/dashjoin/service/SQLDatabaseTest.java index 651a2c873..9c4ffd60f 100644 --- a/dashjoin-core/src/test/java/org/dashjoin/service/SQLDatabaseTest.java +++ b/dashjoin-core/src/test/java/org/dashjoin/service/SQLDatabaseTest.java @@ -60,95 +60,87 @@ ColInfo col(String name, Filter filter, Object arg1, Aggregation aggregation) { public void testAnalyticsDate() throws Exception { Assertions.assertEquals( "select \"HIRE_DATE\" from \"EMP\" where \"HIRE_DATE\" = '1970-01-01' limit 1000", - db().analytics(Table.ofName("EMP"), + db().analytics(null, Table.ofName("EMP"), Arrays.asList(col("HIRE_DATE", Filter.EQUALS, new Date(0), null)))); } @Test public void testAnalytics() throws Exception { // project - Assertions - .assertEquals("[{EMP.ID=1}, {EMP.ID=2}]", - "" + db().query(QueryMeta.ofQuery( - db().analytics(Table.ofName("EMP"), Arrays.asList(col("ID", null, null, null)))), - null)); - // filter - Assertions.assertEquals("[{EMP.ID=1}]", + Assertions.assertEquals("[{EMP.ID=1}, {EMP.ID=2}]", "" + db().query(QueryMeta.ofQuery( - db().analytics(Table.ofName("EMP"), Arrays.asList(col("ID", Filter.EQUALS, 1, null)))), + db().analytics(null, Table.ofName("EMP"), Arrays.asList(col("ID", null, null, null)))), null)); + // filter + Assertions.assertEquals("[{EMP.ID=1}]", "" + db().query(QueryMeta.ofQuery(db().analytics(null, + Table.ofName("EMP"), Arrays.asList(col("ID", Filter.EQUALS, 1, null)))), null)); Assertions.assertEquals("[{EMP.ID=1}, {EMP.ID=2}]", - "" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"), + "" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"), Arrays.asList(col("ID", Filter.GREATER_EQUAL, 1, null)))), null)); - Assertions.assertEquals("[{EMP.ID=1}]", - "" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"), - Arrays.asList(col("ID", Filter.SMALLER_EQUAL, 1, null)))), null)); - Assertions.assertEquals("[{EMP.ID=2}]", "" + db().query(QueryMeta.ofQuery( - db().analytics(Table.ofName("EMP"), Arrays.asList(col("ID", Filter.NOT_EQUALS, 1, null)))), - null)); - Assertions.assertEquals("[{EMP.NAME=mike}]", "" + db().query(QueryMeta.ofQuery( - db().analytics(Table.ofName("EMP"), Arrays.asList(col("NAME", Filter.LIKE, "%IK%", null)))), - null)); - Assertions.assertEquals("[{EMP.ID=1}]", - "" + db().query(QueryMeta.ofQuery( - db().analytics(Table.ofName("EMP"), Arrays.asList(col("ID", Filter.LIKE, 1, null)))), - null)); - Assertions.assertEquals("[{EMP.ID=1}, {EMP.ID=2}]", "" + db().query(QueryMeta.ofQuery( - db().analytics(Table.ofName("EMP"), Arrays.asList(col("ID", Filter.IS_NOT_NULL, 1, null)))), + Assertions.assertEquals("[{EMP.ID=1}]", "" + db().query(QueryMeta.ofQuery(db().analytics(null, + Table.ofName("EMP"), Arrays.asList(col("ID", Filter.SMALLER_EQUAL, 1, null)))), null)); + Assertions.assertEquals("[{EMP.ID=2}]", "" + db().query(QueryMeta.ofQuery(db().analytics(null, + Table.ofName("EMP"), Arrays.asList(col("ID", Filter.NOT_EQUALS, 1, null)))), null)); + Assertions.assertEquals("[{EMP.NAME=mike}]", + "" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"), + Arrays.asList(col("NAME", Filter.LIKE, "%IK%", null)))), null)); + Assertions.assertEquals("[{EMP.ID=1}]", "" + db().query(QueryMeta.ofQuery( + db().analytics(null, Table.ofName("EMP"), Arrays.asList(col("ID", Filter.LIKE, 1, null)))), null)); - Assertions.assertEquals("[]", - "" + db().query(QueryMeta.ofQuery( - db().analytics(Table.ofName("EMP"), Arrays.asList(col("ID", Filter.IS_NULL, 1, null)))), - null)); + Assertions.assertEquals("[{EMP.ID=1}, {EMP.ID=2}]", + "" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"), + Arrays.asList(col("ID", Filter.IS_NOT_NULL, 1, null)))), null)); + Assertions.assertEquals("[]", "" + db().query(QueryMeta.ofQuery(db().analytics(null, + Table.ofName("EMP"), Arrays.asList(col("ID", Filter.IS_NULL, 1, null)))), null)); // aggregation Assertions.assertEquals("[{COUNT(ID)=2, EMP.WORKSON=1000}]", - "" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"), + "" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"), Arrays.asList(col("ID", null, null, Aggregation.COUNT), col("WORKSON", null, null, Aggregation.GROUP_BY)))), null)); Assertions.assertEquals("[{AVG(ID)=1.5, EMP.WORKSON=1000}]", - "" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"), + "" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"), Arrays.asList(col("ID", null, null, Aggregation.AVG), col("WORKSON", null, null, Aggregation.GROUP_BY)))), null)); Assertions.assertEquals("[{SUM(ID)=3, EMP.WORKSON=1000}]", - "" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"), + "" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"), Arrays.asList(col("ID", null, null, Aggregation.SUM), col("WORKSON", null, null, Aggregation.GROUP_BY)))), null)); Assertions.assertEquals("[{MIN(ID)=1, EMP.WORKSON=1000}]", - "" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"), + "" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"), Arrays.asList(col("ID", null, null, Aggregation.MIN), col("WORKSON", null, null, Aggregation.GROUP_BY)))), null)); Assertions.assertEquals("[{MAX(ID)=2, EMP.WORKSON=1000}]", - "" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"), + "" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"), Arrays.asList(col("ID", null, null, Aggregation.MAX), col("WORKSON", null, null, Aggregation.GROUP_BY)))), null)); Assertions.assertEquals("[{COUNT(DISTINCT ID)=2, EMP.WORKSON=1000}]", - "" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"), + "" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"), Arrays.asList(col("ID", null, null, Aggregation.COUNT_DISTINCT), col("WORKSON", null, null, Aggregation.GROUP_BY)))), null)); Assertions.assertEquals("[{X=1,2, EMP.WORKSON=1000}]", - "" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"), + "" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"), Arrays.asList(alias("X", "ID", null, null, Aggregation.GROUP_CONCAT_DISTINCT), col("WORKSON", null, null, Aggregation.GROUP_BY)))), null)); Assertions.assertEquals("[{STDDEV_SAMP(ID)=0.7071067811865476, EMP.WORKSON=1000}]", - "" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"), + "" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"), Arrays.asList(col("ID", null, null, Aggregation.STDDEV), col("WORKSON", null, null, Aggregation.GROUP_BY)))), null)); Assertions.assertEquals("[{Y=1,2, EMP.WORKSON=1000}]", - "" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"), + "" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"), Arrays.asList(alias("Y", "ID", null, null, Aggregation.GROUP_CONCAT), col("WORKSON", null, null, Aggregation.GROUP_BY)))), null)); // filter and aggregation Assertions.assertEquals("[{COUNT(ID)=1, EMP.WORKSON=1000}]", - "" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"), + "" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"), Arrays.asList(col("ID", Filter.EQUALS, 1, Aggregation.COUNT), col("WORKSON", null, null, Aggregation.GROUP_BY)))), null)); @@ -159,7 +151,7 @@ public void testAnalyticsWhereNull() throws Exception { for (Filter f : Filter.values()) { if (!f.toString().contains("NULL")) Assertions.assertFalse( - db().analytics(Table.ofName("EMP"), Arrays.asList(col("ID", f, null, null))) + db().analytics(null, Table.ofName("EMP"), Arrays.asList(col("ID", f, null, null))) .contains("where")); } }