diff --git a/doc/tGoogleAnalytics4Input.docx b/doc/tGoogleAnalytics4Input.docx index 66cc743..cb29de1 100644 Binary files a/doc/tGoogleAnalytics4Input.docx and b/doc/tGoogleAnalytics4Input.docx differ diff --git a/pom.xml b/pom.xml index 1986797..0df6d7a 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 de.jlo.talendcomp.google jlo-talendcomp-google-analytics4 - 1.3 + 2.0 GoogleAnalytics Talend Component for GA4 UTF-8 diff --git a/src/main/java/de/jlo/talendcomp/google/analytics/ga4/v1/FilterExpressionBuilder.java b/src/main/java/de/jlo/talendcomp/google/analytics/ga4/v1/FilterExpressionBuilder.java index ea0a7b1..3817505 100644 --- a/src/main/java/de/jlo/talendcomp/google/analytics/ga4/v1/FilterExpressionBuilder.java +++ b/src/main/java/de/jlo/talendcomp/google/analytics/ga4/v1/FilterExpressionBuilder.java @@ -1,7 +1,11 @@ package de.jlo.talendcomp.google.analytics.ga4.v1; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import com.google.analytics.data.v1beta.Filter; import com.google.analytics.data.v1beta.FilterExpression; +import com.google.analytics.data.v1beta.FilterExpressionList; import com.google.analytics.data.v1beta.NumericValue; import com.google.analytics.data.v1beta.Filter.NumericFilter; import com.google.analytics.data.v1beta.Filter.StringFilter; @@ -10,6 +14,7 @@ public class FilterExpressionBuilder { + private static Logger log = LogManager.getLogger(FilterExpressionBuilder.class); /* Filter operators for String and Numeric filters */ private static final String FILTER_OP_EXACT = "=="; @@ -21,9 +26,145 @@ public class FilterExpressionBuilder { private static final String STRING_FILTER_OP_REGEX_EXCL = "!~"; private static final String STRING_FILTER_OP_CONTAINS = "=@"; private static final String STRING_FILTER_OP_CONTAINS_NOT = "!@"; + + /** + * Setup filter expressions in th UA style + * Examples: + * dim1==value1,dim2==value2,dim3!=value3 + * dim1==value1;dim2==value2;dim3!=value3 + * + * comma means OR + * semicolon means AND + * + * @param completeFilter + * @param isNumericFilter if true, otherwise it is a String filter + * @return FilterExpression object + * @throws Exception + */ + public static FilterExpression buildCombinedFilterExpression(String completeFilter, boolean isNumericFilter) throws Exception { + FilterExpression.Builder fb = FilterExpression.newBuilder(); + FilterExpressionList.Builder list = FilterExpressionList.newBuilder(); + boolean or = false; + boolean and = false; + StringBuilder oneFilterTerm = new StringBuilder(); + for (int i = 0, n = completeFilter.length(); i < n; i++) { + char c = completeFilter.charAt(i); + if (c == '(') { + // start of group + throw new Exception("Invalid filter: " + completeFilter + ". Groups are not allowed yet! Position: " + i); + } else if (c == ')') { + throw new Exception("Invalid filter: " + completeFilter + ". Groups are not allowed yet! Position: " + i); + } else if (c == ',') { + // OR found + if (i == 0) { + throw new Exception("Invalid filter: " + completeFilter + ". <,> cannot be the first char! Position: " + i); + } + if (and) { + throw new Exception("Invalid filter: " + completeFilter + ". OR(,) found with previous AND. Within one filter you can only combine all with OR (,) or all with AND(;). Position: " + i); + } + if (oneFilterTerm.length() == 0) { + throw new Exception("Invalid filter: " + completeFilter + ". OR(,) found but no filter term as operand before. Position: " + i); + } + or = true; + String term = oneFilterTerm.toString(); + oneFilterTerm.setLength(0); + if (isNumericFilter) { + if (log.isDebugEnabled()) { + log.debug("add numeric filter term: " + term); + } + list.addExpressions(buildNumericFilterExpression(term)); + } else { + if (log.isDebugEnabled()) { + log.debug("add string filter term: " + term); + } + list.addExpressions(buildStringFilterExpression(term)); + } + } else if (c == ';') { + // AND found + if (i == 0) { + throw new Exception("Invalid filter: " + completeFilter + ". <;> cannot be the first char! Position: " + i); + } + if (or) { + throw new Exception("Invalid filter: " + completeFilter + ". AND(;) found with previous OR. Within one filter you can only combine all with OR (,) or all with AND(;). Position: " + i); + } + if (oneFilterTerm.length() == 0) { + throw new Exception("Invalid filter: " + completeFilter + ". AND(;) found but no filter term as operand before. Position: " + i); + } + and = true; + String term = oneFilterTerm.toString(); + oneFilterTerm.setLength(0); + if (isNumericFilter) { + if (log.isDebugEnabled()) { + log.debug("add numeric filter term: " + term); + } + list.addExpressions(buildNumericFilterExpression(term)); + } else { + if (log.isDebugEnabled()) { + log.debug("add string filter term: " + term); + } + list.addExpressions(buildStringFilterExpression(term)); + } + } else if (c == ' ') { + // skip this + continue; + } else if (i == n-1) { + // we are at the very last char + if (oneFilterTerm.length() == 0) { + throw new Exception("Invalid filter: " + completeFilter + ". Found end of filter but no filter term as operand before. Position: " + i); + } + if (and == false && or == false) { + String term = oneFilterTerm.toString(); + oneFilterTerm.setLength(0); + if (isNumericFilter) { + if (log.isDebugEnabled()) { + log.debug("add numeric filter term: " + term); + } + list.addExpressions(buildNumericFilterExpression(term)); + } else { + if (log.isDebugEnabled()) { + log.debug("add string filter term: " + term); + } + list.addExpressions(buildStringFilterExpression(term)); + } + } else { + String term = oneFilterTerm.toString(); + oneFilterTerm.setLength(0); + if (isNumericFilter) { + if (log.isDebugEnabled()) { + log.debug("add numeric filter term: " + term); + } + list.addExpressions(buildNumericFilterExpression(term)); + } else { + if (log.isDebugEnabled()) { + log.debug("add string filter term: " + term); + } + list.addExpressions(buildStringFilterExpression(term)); + } + if (and) { + if (log.isDebugEnabled()) { + log.debug("set AND filter list with count expressions: " + list.getExpressionsCount()); + } + fb.setAndGroup(list); + } else if (or) { + if (log.isDebugEnabled()) { + log.debug("set OR filter list with count expressions: " + list.getExpressionsCount()); + } + fb.setOrGroup(list); + } + } + } else { + // we must be in a filter term + oneFilterTerm.append(c); + } + } + return fb.build(); + } public static FilterExpression buildStringFilterExpression(String oneFilterTerm) throws Exception { if (oneFilterTerm != null && oneFilterTerm.trim().isEmpty() == false) { + if (log.isDebugEnabled()) { + log.debug("Build string filter for term: " + oneFilterTerm); + } String operand1 = null; String operand2 = null; FilterExpression fe = null; @@ -160,6 +301,9 @@ public static FilterExpression buildStringFilterExpression(String oneFilterTerm) public static FilterExpression buildNumericFilterExpression(String oneFilterTerm) throws Exception { if (oneFilterTerm != null && oneFilterTerm.trim().isEmpty() == false) { + if (log.isDebugEnabled()) { + log.debug("Build numeric filter for term: " + oneFilterTerm); + } String operand1 = null; String operand2 = null; FilterExpression fe = null; diff --git a/src/main/java/de/jlo/talendcomp/google/analytics/ga4/v1/GoogleAnalyticsInput.java b/src/main/java/de/jlo/talendcomp/google/analytics/ga4/v1/GoogleAnalyticsInput.java index efb17c8..ba42756 100644 --- a/src/main/java/de/jlo/talendcomp/google/analytics/ga4/v1/GoogleAnalyticsInput.java +++ b/src/main/java/de/jlo/talendcomp/google/analytics/ga4/v1/GoogleAnalyticsInput.java @@ -23,7 +23,9 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.concurrent.TimeUnit; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import com.google.analytics.data.v1beta.DateRange; import com.google.analytics.data.v1beta.Dimension; @@ -44,6 +46,7 @@ public class GoogleAnalyticsInput extends GoogleAnalyticsBase { + private static Logger log = LogManager.getLogger(GoogleAnalyticsInput.class); private static final Map clientCache = new HashMap(); private SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); private String startDate = null; @@ -539,19 +542,43 @@ private void setupDimensions() { } } - private void setupDimensionFilters() throws Exception { reportRequestBuilder.clearDimensionFilter(); - if (dimensionFilters != null) { - String[] filters = dimensionFilters.split(","); - if (filters.length > 1) { - for (String f : filters) { - reportRequestBuilder.setDimensionFilter(FilterExpression.newBuilder() - .setAndGroup(FilterExpressionList.newBuilder() - .addExpressions(FilterExpressionBuilder.buildStringFilterExpression(f)))); + if (dimensionFilters != null && dimensionFilters.trim().isEmpty() == false) { + if (dimensionFilters.contains(";")) { + String[] filters = dimensionFilters.split(";"); + if (filters.length > 1) { + FilterExpressionList.Builder flb = FilterExpressionList.newBuilder(); + for (String f : filters) { + if (f.contains(",")) { + throw new Exception("DimensionFilter cannot have AND(;) and OR(,) operators together. Invalid filter term: " + f); + } + flb.addExpressions(FilterExpressionBuilder.buildStringFilterExpression(f)); + } + reportRequestBuilder.setDimensionFilter(FilterExpression + .newBuilder() + .setAndGroup(flb.build())); + } else if (filters.length == 1) { + reportRequestBuilder.setDimensionFilter(FilterExpressionBuilder.buildStringFilterExpression(filters[0])); + } + } else if (dimensionFilters.contains(",")) { + String[] filters = dimensionFilters.split(","); + if (filters.length > 1) { + FilterExpressionList.Builder flb = FilterExpressionList.newBuilder(); + for (String f : filters) { + if (f.contains(";")) { + throw new Exception("DimensionFilter cannot have OR(,) and AND(;) operators together. Invalid filter term: " + f); + } + flb.addExpressions(FilterExpressionBuilder.buildStringFilterExpression(f)); + } + reportRequestBuilder.setDimensionFilter(FilterExpression + .newBuilder() + .setOrGroup(flb.build())); + } else if (filters.length == 1) { + reportRequestBuilder.setDimensionFilter(FilterExpressionBuilder.buildStringFilterExpression(filters[0])); } - } else if (filters.length == 1) { - reportRequestBuilder.setDimensionFilter(FilterExpressionBuilder.buildStringFilterExpression(filters[0])); + } else { + reportRequestBuilder.setDimensionFilter(FilterExpressionBuilder.buildStringFilterExpression(dimensionFilters)); } } } @@ -559,16 +586,41 @@ private void setupDimensionFilters() throws Exception { private void setupMetricsFilters() throws Exception { reportRequestBuilder.clearMetricFilter(); - if (metricFilters != null) { - String[] filters = metricFilters.split(","); - if (filters.length > 1) { - for (String f : filters) { - reportRequestBuilder.setMetricFilter(FilterExpression.newBuilder() - .setAndGroup(FilterExpressionList.newBuilder() - .addExpressions(FilterExpressionBuilder.buildNumericFilterExpression(f)))); + if (metricFilters != null && metricFilters.trim().isEmpty() == false) { + if (metricFilters.contains(";")) { + String[] filters = metricFilters.split(";"); + if (filters.length > 1) { + FilterExpressionList.Builder flb = FilterExpressionList.newBuilder(); + for (String f : filters) { + if (f.contains(",")) { + throw new Exception("MetricFilter cannot have AND(;) and OR(,) operators together. Invalid filter term: " + f); + } + flb.addExpressions(FilterExpressionBuilder.buildNumericFilterExpression(f)); + } + reportRequestBuilder.setMetricFilter(FilterExpression + .newBuilder() + .setAndGroup(flb.build())); + } else if (filters.length == 1) { + reportRequestBuilder.setMetricFilter(FilterExpressionBuilder.buildNumericFilterExpression(filters[0])); } - } else if (filters.length == 1) { - reportRequestBuilder.setMetricFilter(FilterExpressionBuilder.buildNumericFilterExpression(filters[0])); + } else if (metricFilters.contains(",")) { + String[] filters = metricFilters.split(","); + if (filters.length > 1) { + FilterExpressionList.Builder flb = FilterExpressionList.newBuilder(); + for (String f : filters) { + if (f.contains(";")) { + throw new Exception("MetricFilter cannot have OR(,) and AND(;) operators together. Invalid filter term: " + f); + } + flb.addExpressions(FilterExpressionBuilder.buildNumericFilterExpression(f)); + } + reportRequestBuilder.setMetricFilter(FilterExpression + .newBuilder() + .setOrGroup(flb.build())); + } else if (filters.length == 1) { + reportRequestBuilder.setMetricFilter(FilterExpressionBuilder.buildNumericFilterExpression(filters[0])); + } + } else { + reportRequestBuilder.setMetricFilter(FilterExpressionBuilder.buildNumericFilterExpression(metricFilters)); } } }