Skip to content

Commit

Permalink
Merge branch 'improve_filters_with_and_or'
Browse files Browse the repository at this point in the history
  • Loading branch information
@jlolling
jlolling committed May 11, 2023
2 parents a42f431 + 22a9ac2 commit 94c98ad
Show file tree
Hide file tree
Showing 4 changed files with 217 additions and 21 deletions.
Binary file modified doc/tGoogleAnalytics4Input.docx
View file
Binary file not shown.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>de.jlo.talendcomp.google</groupId>
<artifactId>jlo-talendcomp-google-analytics4</artifactId>
<version>1.3</version>
<version>2.0</version>
<name>GoogleAnalytics Talend Component for GA4</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
Expand Down
144 changes: 144 additions & 0 deletions src/main/java/de/jlo/talendcomp/google/analytics/ga4/v1/FilterExpressionBuilder.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
package de.jlo.talendcomp.google.analytics.ga4.v1;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import com.google.analytics.data.v1beta.Filter;
import com.google.analytics.data.v1beta.FilterExpression;
import com.google.analytics.data.v1beta.FilterExpressionList;
import com.google.analytics.data.v1beta.NumericValue;
import com.google.analytics.data.v1beta.Filter.NumericFilter;
import com.google.analytics.data.v1beta.Filter.StringFilter;
Expand All @@ -10,6 +14,7 @@

public class FilterExpressionBuilder {

private static Logger log = LogManager.getLogger(FilterExpressionBuilder.class);
/* Filter operators for String and Numeric filters */

private static final String FILTER_OP_EXACT = "==";
Expand All @@ -21,9 +26,145 @@ public class FilterExpressionBuilder {
private static final String STRING_FILTER_OP_REGEX_EXCL = "!~";
private static final String STRING_FILTER_OP_CONTAINS = "=@";
private static final String STRING_FILTER_OP_CONTAINS_NOT = "!@";

/**
* Setup filter expressions in th UA style
* Examples:
* dim1==value1,dim2==value2,dim3!=value3
* dim1==value1;dim2==value2;dim3!=value3
*
* comma means OR
* semicolon means AND
*
* @param completeFilter
* @param isNumericFilter if true, otherwise it is a String filter
* @return FilterExpression object
* @throws Exception
*/
public static FilterExpression buildCombinedFilterExpression(String completeFilter, boolean isNumericFilter) throws Exception {
FilterExpression.Builder fb = FilterExpression.newBuilder();
FilterExpressionList.Builder list = FilterExpressionList.newBuilder();
boolean or = false;
boolean and = false;
StringBuilder oneFilterTerm = new StringBuilder();
for (int i = 0, n = completeFilter.length(); i < n; i++) {
char c = completeFilter.charAt(i);
if (c == '(') {
// start of group
throw new Exception("Invalid filter: " + completeFilter + ". Groups are not allowed yet! Position: " + i);
} else if (c == ')') {
throw new Exception("Invalid filter: " + completeFilter + ". Groups are not allowed yet! Position: " + i);
} else if (c == ',') {
// OR found
if (i == 0) {
throw new Exception("Invalid filter: " + completeFilter + ". <,> cannot be the first char! Position: " + i);
}
if (and) {
throw new Exception("Invalid filter: " + completeFilter + ". OR(,) found with previous AND. Within one filter you can only combine all with OR (,) or all with AND(;). Position: " + i);
}
if (oneFilterTerm.length() == 0) {
throw new Exception("Invalid filter: " + completeFilter + ". OR(,) found but no filter term as operand before. Position: " + i);
}
or = true;
String term = oneFilterTerm.toString();
oneFilterTerm.setLength(0);
if (isNumericFilter) {
if (log.isDebugEnabled()) {
log.debug("add numeric filter term: " + term);
}
list.addExpressions(buildNumericFilterExpression(term));
} else {
if (log.isDebugEnabled()) {
log.debug("add string filter term: " + term);
}
list.addExpressions(buildStringFilterExpression(term));
}
} else if (c == ';') {
// AND found
if (i == 0) {
throw new Exception("Invalid filter: " + completeFilter + ". <;> cannot be the first char! Position: " + i);
}
if (or) {
throw new Exception("Invalid filter: " + completeFilter + ". AND(;) found with previous OR. Within one filter you can only combine all with OR (,) or all with AND(;). Position: " + i);
}
if (oneFilterTerm.length() == 0) {
throw new Exception("Invalid filter: " + completeFilter + ". AND(;) found but no filter term as operand before. Position: " + i);
}
and = true;
String term = oneFilterTerm.toString();
oneFilterTerm.setLength(0);
if (isNumericFilter) {
if (log.isDebugEnabled()) {
log.debug("add numeric filter term: " + term);
}
list.addExpressions(buildNumericFilterExpression(term));
} else {
if (log.isDebugEnabled()) {
log.debug("add string filter term: " + term);
}
list.addExpressions(buildStringFilterExpression(term));
}
} else if (c == ' ') {
// skip this
continue;
} else if (i == n-1) {
// we are at the very last char
if (oneFilterTerm.length() == 0) {
throw new Exception("Invalid filter: " + completeFilter + ". Found end of filter but no filter term as operand before. Position: " + i);
}
if (and == false && or == false) {
String term = oneFilterTerm.toString();
oneFilterTerm.setLength(0);
if (isNumericFilter) {
if (log.isDebugEnabled()) {
log.debug("add numeric filter term: " + term);
}
list.addExpressions(buildNumericFilterExpression(term));
} else {
if (log.isDebugEnabled()) {
log.debug("add string filter term: " + term);
}
list.addExpressions(buildStringFilterExpression(term));
}
} else {
String term = oneFilterTerm.toString();
oneFilterTerm.setLength(0);
if (isNumericFilter) {
if (log.isDebugEnabled()) {
log.debug("add numeric filter term: " + term);
}
list.addExpressions(buildNumericFilterExpression(term));
} else {
if (log.isDebugEnabled()) {
log.debug("add string filter term: " + term);
}
list.addExpressions(buildStringFilterExpression(term));
}
if (and) {
if (log.isDebugEnabled()) {
log.debug("set AND filter list with count expressions: " + list.getExpressionsCount());
}
fb.setAndGroup(list);
} else if (or) {
if (log.isDebugEnabled()) {
log.debug("set OR filter list with count expressions: " + list.getExpressionsCount());
}
fb.setOrGroup(list);
}
}
} else {
// we must be in a filter term
oneFilterTerm.append(c);
}
}
return fb.build();
}

public static FilterExpression buildStringFilterExpression(String oneFilterTerm) throws Exception {
if (oneFilterTerm != null && oneFilterTerm.trim().isEmpty() == false) {
if (log.isDebugEnabled()) {
log.debug("Build string filter for term: " + oneFilterTerm);
}
String operand1 = null;
String operand2 = null;
FilterExpression fe = null;
Expand Down Expand Up @@ -160,6 +301,9 @@ public static FilterExpression buildStringFilterExpression(String oneFilterTerm)

public static FilterExpression buildNumericFilterExpression(String oneFilterTerm) throws Exception {
if (oneFilterTerm != null && oneFilterTerm.trim().isEmpty() == false) {
if (log.isDebugEnabled()) {
log.debug("Build numeric filter for term: " + oneFilterTerm);
}
String operand1 = null;
String operand2 = null;
FilterExpression fe = null;
Expand Down
92 changes: 72 additions & 20 deletions src/main/java/de/jlo/talendcomp/google/analytics/ga4/v1/GoogleAnalyticsInput.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import com.google.analytics.data.v1beta.DateRange;
import com.google.analytics.data.v1beta.Dimension;
Expand All @@ -44,6 +46,7 @@

public class GoogleAnalyticsInput extends GoogleAnalyticsBase {

private static Logger log = LogManager.getLogger(GoogleAnalyticsInput.class);
private static final Map<String, GoogleAnalyticsInput> clientCache = new HashMap<String, GoogleAnalyticsInput>();
private SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
private String startDate = null;
Expand Down Expand Up @@ -539,36 +542,85 @@ private void setupDimensions() {
}
}


private void setupDimensionFilters() throws Exception {
reportRequestBuilder.clearDimensionFilter();
if (dimensionFilters != null) {
String[] filters = dimensionFilters.split(",");
if (filters.length > 1) {
for (String f : filters) {
reportRequestBuilder.setDimensionFilter(FilterExpression.newBuilder()
.setAndGroup(FilterExpressionList.newBuilder()
.addExpressions(FilterExpressionBuilder.buildStringFilterExpression(f))));
if (dimensionFilters != null && dimensionFilters.trim().isEmpty() == false) {
if (dimensionFilters.contains(";")) {
String[] filters = dimensionFilters.split(";");
if (filters.length > 1) {
FilterExpressionList.Builder flb = FilterExpressionList.newBuilder();
for (String f : filters) {
if (f.contains(",")) {
throw new Exception("DimensionFilter cannot have AND(;) and OR(,) operators together. Invalid filter term: " + f);
}
flb.addExpressions(FilterExpressionBuilder.buildStringFilterExpression(f));
}
reportRequestBuilder.setDimensionFilter(FilterExpression
.newBuilder()
.setAndGroup(flb.build()));
} else if (filters.length == 1) {
reportRequestBuilder.setDimensionFilter(FilterExpressionBuilder.buildStringFilterExpression(filters[0]));
}
} else if (dimensionFilters.contains(",")) {
String[] filters = dimensionFilters.split(",");
if (filters.length > 1) {
FilterExpressionList.Builder flb = FilterExpressionList.newBuilder();
for (String f : filters) {
if (f.contains(";")) {
throw new Exception("DimensionFilter cannot have OR(,) and AND(;) operators together. Invalid filter term: " + f);
}
flb.addExpressions(FilterExpressionBuilder.buildStringFilterExpression(f));
}
reportRequestBuilder.setDimensionFilter(FilterExpression
.newBuilder()
.setOrGroup(flb.build()));
} else if (filters.length == 1) {
reportRequestBuilder.setDimensionFilter(FilterExpressionBuilder.buildStringFilterExpression(filters[0]));
}
} else if (filters.length == 1) {
reportRequestBuilder.setDimensionFilter(FilterExpressionBuilder.buildStringFilterExpression(filters[0]));
} else {
reportRequestBuilder.setDimensionFilter(FilterExpressionBuilder.buildStringFilterExpression(dimensionFilters));
}
}
}


private void setupMetricsFilters() throws Exception {
reportRequestBuilder.clearMetricFilter();
if (metricFilters != null) {
String[] filters = metricFilters.split(",");
if (filters.length > 1) {
for (String f : filters) {
reportRequestBuilder.setMetricFilter(FilterExpression.newBuilder()
.setAndGroup(FilterExpressionList.newBuilder()
.addExpressions(FilterExpressionBuilder.buildNumericFilterExpression(f))));
if (metricFilters != null && metricFilters.trim().isEmpty() == false) {
if (metricFilters.contains(";")) {
String[] filters = metricFilters.split(";");
if (filters.length > 1) {
FilterExpressionList.Builder flb = FilterExpressionList.newBuilder();
for (String f : filters) {
if (f.contains(",")) {
throw new Exception("MetricFilter cannot have AND(;) and OR(,) operators together. Invalid filter term: " + f);
}
flb.addExpressions(FilterExpressionBuilder.buildNumericFilterExpression(f));
}
reportRequestBuilder.setMetricFilter(FilterExpression
.newBuilder()
.setAndGroup(flb.build()));
} else if (filters.length == 1) {
reportRequestBuilder.setMetricFilter(FilterExpressionBuilder.buildNumericFilterExpression(filters[0]));
}
} else if (filters.length == 1) {
reportRequestBuilder.setMetricFilter(FilterExpressionBuilder.buildNumericFilterExpression(filters[0]));
} else if (metricFilters.contains(",")) {
String[] filters = metricFilters.split(",");
if (filters.length > 1) {
FilterExpressionList.Builder flb = FilterExpressionList.newBuilder();
for (String f : filters) {
if (f.contains(";")) {
throw new Exception("MetricFilter cannot have OR(,) and AND(;) operators together. Invalid filter term: " + f);
}
flb.addExpressions(FilterExpressionBuilder.buildNumericFilterExpression(f));
}
reportRequestBuilder.setMetricFilter(FilterExpression
.newBuilder()
.setOrGroup(flb.build()));
} else if (filters.length == 1) {
reportRequestBuilder.setMetricFilter(FilterExpressionBuilder.buildNumericFilterExpression(filters[0]));
}
} else {
reportRequestBuilder.setMetricFilter(FilterExpressionBuilder.buildNumericFilterExpression(metricFilters));
}
}
}
Expand Down

0 comments on commit 94c98ad

Please sign in to comment.