Skip to content

Commit

Permalink
Analytics can be based on queries and tables
Browse files Browse the repository at this point in the history
  • Loading branch information
@aeberhart
aeberhart committed Oct 7, 2024
1 parent 9b5c3e1 commit 9d8b692
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 60 deletions.
47 changes: 30 additions & 17 deletions dashjoin-core/src/main/java/org/dashjoin/service/Data.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import org.eclipse.microprofile.openapi.annotations.parameters.Parameter;
import org.eclipse.microprofile.openapi.annotations.responses.APIResponse;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
Expand Down Expand Up @@ -309,10 +308,11 @@ public List<Map<String, Object>> queryInternal(SecurityContext sc, String databa
if (arguments == null)
arguments = new HashMap<>();

if (arguments.get("__dj_analytics") instanceof List) {
if ("__dj_analytics".equals(queryId)) {
AbstractDatabase db = services.getConfig().getDatabase(dj(database));
String query = getAnalytics(sc, db, queryId, arguments);
return db.query(QueryMeta.ofQuery(query, false), null);
Analytics analytics = om.convertValue(arguments, Analytics.class);
String query = getAnalytics(sc, db, analytics);
return db.query(QueryMeta.ofQuery(query, false), analytics.arguments);
}

QueryMeta info = services.getConfig().getQueryMeta(queryId);
Expand All @@ -325,17 +325,22 @@ public List<Map<String, Object>> queryInternal(SecurityContext sc, String databa
return db.query(info, arguments);
}

String getAnalytics(SecurityContext sc, AbstractDatabase db, String table,
Map<String, Object> arguments) {
List<ColInfo> cols =
om.convertValue(arguments.get("__dj_analytics"), new TypeReference<List<ColInfo>>() {});
Table m = db.tables.get(table);
String getAnalytics(SecurityContext sc, AbstractDatabase db, Analytics a) throws Exception {

if (m == null)
throw new IllegalArgumentException("Unknown table: " + table);
QueryMeta info = null;
if (a.query != null) {
info = services.getConfig().getQueryMeta(a.query);
if ("write".equals(info.type))
return null;
ACLContainerRequestFilter.check(sc, info);
}

Table m = db.tables.get(a.table);
if (m == null)
throw new IllegalArgumentException("Unknown table: " + a.table);
ACLContainerRequestFilter.check(sc, db, m);
for (ColInfo e : cols) {

for (ColInfo e : a.cols) {
if (e.arg1 != null) {
Map<String, Object> tmp = MapUtil.of(e.name, e.arg1);
db.cast(m, tmp);
Expand All @@ -355,10 +360,10 @@ String getAnalytics(SecurityContext sc, AbstractDatabase db, String table,
ci.name = e.getKey();
ci.filter = Filter.EQUALS;
ci.arg1 = e.getValue();
cols.add(ci);
a.cols.add(ci);
}

return db.analytics(m, cols);
return db.analytics(info, m, a.cols);
}

/**
Expand All @@ -378,10 +383,11 @@ public Map<String, Property> queryMeta(@Context SecurityContext sc,
if (arguments == null)
arguments = new HashMap<>();

if (arguments.get("__dj_analytics") instanceof List) {
if ("__dj_analytics".equals(queryId)) {
AbstractDatabase db = services.getConfig().getDatabase(dj(database));
String query = getAnalytics(sc, db, queryId, arguments);
return db.queryMeta(QueryMeta.ofQuery(query, false), null);
Analytics analytics = om.convertValue(arguments, Analytics.class);
String query = getAnalytics(sc, db, analytics);
return db.queryMeta(QueryMeta.ofQuery(query, false), analytics.arguments);
}

QueryMeta info = services.getConfig().getQueryMeta(queryId);
Expand Down Expand Up @@ -869,6 +875,13 @@ public static enum Filter {
EQUALS, NOT_EQUALS, LIKE, IS_NULL, IS_NOT_NULL, SMALLER_EQUAL, GREATER_EQUAL, BETWEEN
}

public static class Analytics {
public List<ColInfo> cols;
public String table;
public String query;
public Map<String, Object> arguments;
}

/**
* analytics query column info
*/
Expand Down
2 changes: 1 addition & 1 deletion dashjoin-core/src/main/java/org/dashjoin/service/Database.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ public List<Map<String, Object>> all(Table s, Integer offset, Integer limit, Str
/**
* generates ad hoc analytics queries without having to rely on the query editor
*/
default public String analytics(Table s, List<ColInfo> arguments) {
default public String analytics(QueryMeta info, Table s, List<ColInfo> arguments) {
return null;
}

Expand Down
6 changes: 4 additions & 2 deletions dashjoin-core/src/main/java/org/dashjoin/service/SQLDatabase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -536,7 +536,7 @@ public List<SearchResult> search(SecurityContext sc, Table filter, String search
}

@Override
public String analytics(Table s, List<ColInfo> arguments) {
public String analytics(QueryMeta info, Table s, List<ColInfo> arguments) {
List<String> project = new ArrayList<>();

boolean isGroupBy = false;
Expand Down Expand Up @@ -639,7 +639,9 @@ else if (isGroupBy)
}
}

String select = "select " + String.join(", ", project) + " from " + schema() + q(s.name);
String subselect = info == null ? "" : "(" + info.query + ") as ";
String select =
"select " + String.join(", ", project) + " from " + subselect + schema() + q(s.name);

if (!where.isEmpty())
select += " where " + String.join(" and ", where);
Expand Down
72 changes: 32 additions & 40 deletions dashjoin-core/src/test/java/org/dashjoin/service/SQLDatabaseTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,95 +60,87 @@ ColInfo col(String name, Filter filter, Object arg1, Aggregation aggregation) {
public void testAnalyticsDate() throws Exception {
Assertions.assertEquals(
"select \"HIRE_DATE\" from \"EMP\" where \"HIRE_DATE\" = '1970-01-01' limit 1000",
db().analytics(Table.ofName("EMP"),
db().analytics(null, Table.ofName("EMP"),
Arrays.asList(col("HIRE_DATE", Filter.EQUALS, new Date(0), null))));
}

@Test
public void testAnalytics() throws Exception {
// project
Assertions
.assertEquals("[{EMP.ID=1}, {EMP.ID=2}]",
"" + db().query(QueryMeta.ofQuery(
db().analytics(Table.ofName("EMP"), Arrays.asList(col("ID", null, null, null)))),
null));
// filter
Assertions.assertEquals("[{EMP.ID=1}]",
Assertions.assertEquals("[{EMP.ID=1}, {EMP.ID=2}]",
"" + db().query(QueryMeta.ofQuery(
db().analytics(Table.ofName("EMP"), Arrays.asList(col("ID", Filter.EQUALS, 1, null)))),
db().analytics(null, Table.ofName("EMP"), Arrays.asList(col("ID", null, null, null)))),
null));
// filter
Assertions.assertEquals("[{EMP.ID=1}]", "" + db().query(QueryMeta.ofQuery(db().analytics(null,
Table.ofName("EMP"), Arrays.asList(col("ID", Filter.EQUALS, 1, null)))), null));
Assertions.assertEquals("[{EMP.ID=1}, {EMP.ID=2}]",
"" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"),
"" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"),
Arrays.asList(col("ID", Filter.GREATER_EQUAL, 1, null)))), null));
Assertions.assertEquals("[{EMP.ID=1}]",
"" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"),
Arrays.asList(col("ID", Filter.SMALLER_EQUAL, 1, null)))), null));
Assertions.assertEquals("[{EMP.ID=2}]", "" + db().query(QueryMeta.ofQuery(
db().analytics(Table.ofName("EMP"), Arrays.asList(col("ID", Filter.NOT_EQUALS, 1, null)))),
null));
Assertions.assertEquals("[{EMP.NAME=mike}]", "" + db().query(QueryMeta.ofQuery(
db().analytics(Table.ofName("EMP"), Arrays.asList(col("NAME", Filter.LIKE, "%IK%", null)))),
null));
Assertions.assertEquals("[{EMP.ID=1}]",
"" + db().query(QueryMeta.ofQuery(
db().analytics(Table.ofName("EMP"), Arrays.asList(col("ID", Filter.LIKE, 1, null)))),
null));
Assertions.assertEquals("[{EMP.ID=1}, {EMP.ID=2}]", "" + db().query(QueryMeta.ofQuery(
db().analytics(Table.ofName("EMP"), Arrays.asList(col("ID", Filter.IS_NOT_NULL, 1, null)))),
Assertions.assertEquals("[{EMP.ID=1}]", "" + db().query(QueryMeta.ofQuery(db().analytics(null,
Table.ofName("EMP"), Arrays.asList(col("ID", Filter.SMALLER_EQUAL, 1, null)))), null));
Assertions.assertEquals("[{EMP.ID=2}]", "" + db().query(QueryMeta.ofQuery(db().analytics(null,
Table.ofName("EMP"), Arrays.asList(col("ID", Filter.NOT_EQUALS, 1, null)))), null));
Assertions.assertEquals("[{EMP.NAME=mike}]",
"" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"),
Arrays.asList(col("NAME", Filter.LIKE, "%IK%", null)))), null));
Assertions.assertEquals("[{EMP.ID=1}]", "" + db().query(QueryMeta.ofQuery(
db().analytics(null, Table.ofName("EMP"), Arrays.asList(col("ID", Filter.LIKE, 1, null)))),
null));
Assertions.assertEquals("[]",
"" + db().query(QueryMeta.ofQuery(
db().analytics(Table.ofName("EMP"), Arrays.asList(col("ID", Filter.IS_NULL, 1, null)))),
null));
Assertions.assertEquals("[{EMP.ID=1}, {EMP.ID=2}]",
"" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"),
Arrays.asList(col("ID", Filter.IS_NOT_NULL, 1, null)))), null));
Assertions.assertEquals("[]", "" + db().query(QueryMeta.ofQuery(db().analytics(null,
Table.ofName("EMP"), Arrays.asList(col("ID", Filter.IS_NULL, 1, null)))), null));
// aggregation
Assertions.assertEquals("[{COUNT(ID)=2, EMP.WORKSON=1000}]",
"" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"),
"" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"),
Arrays.asList(col("ID", null, null, Aggregation.COUNT),
col("WORKSON", null, null, Aggregation.GROUP_BY)))),
null));
Assertions.assertEquals("[{AVG(ID)=1.5, EMP.WORKSON=1000}]",
"" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"),
"" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"),
Arrays.asList(col("ID", null, null, Aggregation.AVG),
col("WORKSON", null, null, Aggregation.GROUP_BY)))),
null));
Assertions.assertEquals("[{SUM(ID)=3, EMP.WORKSON=1000}]",
"" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"),
"" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"),
Arrays.asList(col("ID", null, null, Aggregation.SUM),
col("WORKSON", null, null, Aggregation.GROUP_BY)))),
null));
Assertions.assertEquals("[{MIN(ID)=1, EMP.WORKSON=1000}]",
"" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"),
"" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"),
Arrays.asList(col("ID", null, null, Aggregation.MIN),
col("WORKSON", null, null, Aggregation.GROUP_BY)))),
null));
Assertions.assertEquals("[{MAX(ID)=2, EMP.WORKSON=1000}]",
"" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"),
"" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"),
Arrays.asList(col("ID", null, null, Aggregation.MAX),
col("WORKSON", null, null, Aggregation.GROUP_BY)))),
null));
Assertions.assertEquals("[{COUNT(DISTINCT ID)=2, EMP.WORKSON=1000}]",
"" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"),
"" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"),
Arrays.asList(col("ID", null, null, Aggregation.COUNT_DISTINCT),
col("WORKSON", null, null, Aggregation.GROUP_BY)))),
null));
Assertions.assertEquals("[{X=1,2, EMP.WORKSON=1000}]",
"" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"),
"" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"),
Arrays.asList(alias("X", "ID", null, null, Aggregation.GROUP_CONCAT_DISTINCT),
col("WORKSON", null, null, Aggregation.GROUP_BY)))),
null));
Assertions.assertEquals("[{STDDEV_SAMP(ID)=0.7071067811865476, EMP.WORKSON=1000}]",
"" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"),
"" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"),
Arrays.asList(col("ID", null, null, Aggregation.STDDEV),
col("WORKSON", null, null, Aggregation.GROUP_BY)))),
null));
Assertions.assertEquals("[{Y=1,2, EMP.WORKSON=1000}]",
"" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"),
"" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"),
Arrays.asList(alias("Y", "ID", null, null, Aggregation.GROUP_CONCAT),
col("WORKSON", null, null, Aggregation.GROUP_BY)))),
null));
// filter and aggregation
Assertions.assertEquals("[{COUNT(ID)=1, EMP.WORKSON=1000}]",
"" + db().query(QueryMeta.ofQuery(db().analytics(Table.ofName("EMP"),
"" + db().query(QueryMeta.ofQuery(db().analytics(null, Table.ofName("EMP"),
Arrays.asList(col("ID", Filter.EQUALS, 1, Aggregation.COUNT),
col("WORKSON", null, null, Aggregation.GROUP_BY)))),
null));
Expand All @@ -159,7 +151,7 @@ public void testAnalyticsWhereNull() throws Exception {
for (Filter f : Filter.values()) {
if (!f.toString().contains("NULL"))
Assertions.assertFalse(
db().analytics(Table.ofName("EMP"), Arrays.asList(col("ID", f, null, null)))
db().analytics(null, Table.ofName("EMP"), Arrays.asList(col("ID", f, null, null)))
.contains("where"));
}
}
Expand Down

0 comments on commit 9d8b692

Please sign in to comment.