Enhanced statistics, added hana driver, added query element based commit scope

core/src/conf/scriptella/dtd/etl.dtd

@@ -215,11 +215,13 @@ Queries a table and writes the result into CSV file
  @attr if expression to test. If result of evaluation is true - the element is evaluated, otherwise skipped.
         Use <a href="http://jakarta.apache.org/commons/jexl">JEXL</a> syntax for expressions.
  @attr id script identifier(name).
+ @attr commit if set to true, it will commit all connections on query exit
 -->
 <!ATTLIST query
 connection-id IDREF #IMPLIED
 if CDATA #IMPLIED
 id ID #IMPLIED
+commit (true | false) "false"
 >
 
 <!--

core/src/java/scriptella/configuration/ScriptingElement.java

@@ -28,6 +28,7 @@
 public abstract class ScriptingElement extends XmlConfigurableBase {
     private String connectionId;
     private String ifExpr;
+    private String commit;
     private DialectBasedContentEl contentEl;
     private ScriptingElement parent;
 
@@ -55,7 +56,15 @@ public String getIf() {
     public void setIf(final String ifExpr) {
         this.ifExpr = ifExpr;
     }
-
+
+    public String getCommit(){
+    		return commit;
+    }
+
+    public void setCommit(final String commit){
+    		this.commit = commit;
+    }
+
     public Resource getContent() {
         return contentEl.getContent(null);
     }
@@ -73,6 +82,7 @@ public void configure(final XmlElement element) {
         setLocation(element);
         setProperty(element, "connection-id", "connectionId");
         setProperty(element, "if");
+        setProperty(element, "commit");
         contentEl = new DialectBasedContentEl(element);
         contentEl.setLocation(getLocation());
     }
@@ -81,7 +91,7 @@ public void configure(final XmlElement element) {
     public String toString() {
         return "connectionId='" + connectionId + '\'' +
                 ", location=" + getLocation() +
-                ", ifExpr='" + ifExpr + '\'';
-
+                ", ifExpr='" + ifExpr + '\'' + 
+                ", commit='" + commit + '\'';
     }
 }

core/src/java/scriptella/core/DynamicContext.java

@@ -16,6 +16,7 @@
 package scriptella.core;
 
 import scriptella.execution.EtlContext;
+import scriptella.execution.ExecutionStatisticsBuilder;
 import scriptella.spi.Connection;
 import scriptella.spi.ParametersCallback;
 
@@ -32,12 +33,14 @@
 public class DynamicContext implements ParametersCallback {
     protected EtlContext globalContext;
     protected EtlVariable etlVariable;
+    protected ExecutionStatisticsBuilder statisticsBuilder;
 
     protected DynamicContext() {
     }
 
     public DynamicContext(EtlContext globalContext) {
         this.globalContext = globalContext;
+        this.statisticsBuilder = globalContext.getStatisticsBuilder();
     }
 
     public Object getParameter(final String name) {
@@ -56,7 +59,11 @@ public Connection getConnection() {
     protected EtlContext getGlobalContext() {
         return globalContext;
     }
-
+
+    protected ExecutionStatisticsBuilder getStatisticsBuilder(){
+    		return statisticsBuilder;
+    }
+
     EtlVariable getEtlVariable() {
         if (etlVariable == null) {
             etlVariable = new EtlVariable(this, globalContext);

core/src/java/scriptella/core/DynamicContextDecorator.java

@@ -56,14 +56,15 @@ public Connection getConnection() {
 
 
     /**
-     * Dynamically changes context beign decorated.
-     * <p>Should be used with caution mostly for performance reasosns.
+     * Dynamically changes context being decorated.
+     * <p>Should be used with caution mostly for performance reasons.
      *
      * @param context new context to decorate.
      */
     void setContext(final DynamicContext context) {
         this.context = context;
         globalContext = context.getGlobalContext();
+    		statisticsBuilder = globalContext.getStatisticsBuilder();
         cachedConnection = null;
     }
 

core/src/java/scriptella/core/IfInterceptor.java

@@ -54,7 +54,6 @@ public IfInterceptor(ExecutableElement next, ScriptingElement scr) {
 
     public void execute(final DynamicContext ctx) {
         boolean ok = false;
-
         try {
             final Object res = expression.evaluate(ctx);
 
@@ -75,6 +74,7 @@ public void execute(final DynamicContext ctx) {
         if (ok) { //if expr evaluated to true
             executeNext(ctx);
         } else {
+        		ctx.statisticsBuilder.incrementFilteredExecutionCount(location);
             if (LOG.isLoggable(Level.FINE)) {
                 LOG.fine("if=\""+expression.getExpression()+"\" is false, element body is skipped.");
             }

core/src/java/scriptella/core/QueryExecutor.java

@@ -61,14 +61,16 @@ private void initNestedExecutors() {
             }
         }
     }
-
-
+
     protected void execute(Connection connection, Resource resource, DynamicContext ctx) {
         final QueryCtxDecorator ctxDecorator = new QueryCtxDecorator(ctx);
         if (debug) {
             log.fine("Executing query " + getLocation());
         }
         connection.executeQuery(resource, ctx, ctxDecorator);
+        if (getElement().getCommit().equalsIgnoreCase("TRUE")){
+    			ctx.getGlobalContext().getSession().commit();
+        }
         if (debug) {
             if (ctxDecorator.rownum == 0) {
                 log.fine("Query " + getLocation() + " returned no results.");
@@ -103,6 +105,10 @@ public QueryCtxDecorator(DynamicContext context) {
 
         public void processRow(final ParametersCallback parameters) {
             EtlCancelledException.checkEtlCancelled();
+            if (rownum == 0){
+              statisticsBuilder.elementProcessingStarted();
+            }
+            statisticsBuilder.incrementNestedExecutionCount();
             rownum++;
             params = parameters;
             cachedParams.clear();

core/src/java/scriptella/core/ScriptExecutor.java

@@ -37,6 +37,7 @@
  * @version 1.0
  */
 public final class ScriptExecutor extends ContentExecutor<ScriptEl> {
+		private boolean firstExecution = true;
     public ScriptExecutor(ScriptEl scriptEl) {
         super(scriptEl);
     }
@@ -45,6 +46,10 @@ protected void execute(Connection connection, Resource resource, DynamicContext
         if (debug) {
             log.fine("Executing script " + getLocation());
         }
+	      if (firstExecution){
+	      	firstExecution = false;
+	        ctx.statisticsBuilder.elementProcessingStarted();
+	      }
         boolean repeat;
         OnErrorHandler onErrorHandler = null;
         do {

core/src/java/scriptella/core/StatisticInterceptor.java

@@ -16,7 +16,6 @@
 package scriptella.core;
 
 import scriptella.configuration.Location;
-import scriptella.execution.ExecutionStatisticsBuilder;
 
 
 /**
@@ -35,16 +34,15 @@ public StatisticInterceptor(ExecutableElement next, Location location) {
 
     public void execute(final DynamicContext ctx) {
         boolean ok = false;
-        final ExecutionStatisticsBuilder statisticsBuilder = ctx.getGlobalContext().getStatisticsBuilder();
-        statisticsBuilder.elementStarted(location, ctx.getConnection());
+        ctx.statisticsBuilder.elementStarted(location, ctx.getConnection());
         try {
             executeNext(ctx);
             ok = true;
         } finally {
             if (ok) {
-                statisticsBuilder.elementExecuted();
+                ctx.statisticsBuilder.elementExecuted();
             } else {
-                statisticsBuilder.elementFailed();
+                ctx.statisticsBuilder.elementFailed();
             }
         }
 

core/src/java/scriptella/execution/ExecutionStatistics.java

@@ -104,11 +104,11 @@ public String toString() {
         for (ElementInfo ei : elements) {
             sb.append(ei.id).append(":");
 
-            if (ei.okCount > 0) {
+            if (ei.successfulExecutionCount > 0) {
                 sb.append(" Element successfully executed");
-                if (ei.okCount > 1) {
+                if (ei.successfulExecutionCount > 1) {
                     sb.append(' ');
-                    appendPlural(sb, ei.okCount, "time");
+                    appendPlural(sb, ei.successfulExecutionCount, "time");
                 }
                 if (ei.statements > 0) {
                     sb.append(" (");
@@ -117,9 +117,9 @@ public String toString() {
                 sb.append('.');
             }
 
-            if (ei.failedCount > 0) {
+            if (ei.failedExecutionCount > 0) {
                 sb.append(" Element failed ");
-                appendPlural(sb, ei.failedCount, "time");
+                appendPlural(sb, ei.failedExecutionCount, "time");
                 sb.append('.');
             }
             sb.append(" Working time ").append(ei.workingTime / 1000000).append(" milliseconds.");
@@ -248,21 +248,50 @@ void setFinished(Date finished) {
 
 
     public static class ElementInfo {
-        int okCount;
-        Connection connection;
+    		Connection connection;
+        int successfulExecutionCount;
+        int filteredExecutionCount;
+        int failedExecutionCount;
+        int nestedExecutionCount;
         long statementsOnStart;
         long statements;
-        int failedCount;
         long started;
+        Date execStart;
+        Date processingStart;
+        Date execEnd;
         long workingTime;
         String id;
-
+
+        public Date getProcessingStart(){
+        		return processingStart;
+        }
+
+        public Date getExecStart(){
+        	return execStart;
+        }
+
+        public Date getExecEnd(){
+        	return execEnd;
+        }
+
+        public int getTotalExecutionCount(){
+        	return filteredExecutionCount + successfulExecutionCount + failedExecutionCount;
+        }
+
+        public int getFilteredExecutionCount(){
+        	return filteredExecutionCount;
+        }
+
         public int getSuccessfulExecutionCount() {
-            return okCount;
+            return successfulExecutionCount;
         }
 
         public int getFailedExecutionCount() {
-            return failedCount;
+            return failedExecutionCount;
+        }
+
+        public int getNestedExecutionCount(){
+        	return nestedExecutionCount;
         }
 
         /**
@@ -284,15 +313,23 @@ public long getStatementsCount() {
         public long getWorkingTime() {
             return workingTime;
         }
+
+        /**
+         * 
+         * @return total execution time in seconds.
+         */
+        public long getExecTime() {
+        		return (execEnd.getTime()-execStart.getTime())/1000;
+        }
 
         /**
-         * Returns throughput t=statements/workingTimeSeconds. The
-         * throughput has statement/second unit.
+         * Returns throughput t=statements/execTimeSeconds. The
+         * throughput has statements/second unit.
          *
-         * @return statement/second thoughput or -1 if no statements info available or working time is zero.
+         * @return stmt/second thoughput or -1 if no statements info available or execution time is zero.
          */
         public double getThroughput() {
-            return statements <= 0 || workingTime <= 0 ? -1 : 1000000000d * statements / workingTime;
+            return statements <= 0 || getExecTime() <= 0 ? -1 : getStatementsCount() / getExecTime();
         }
 
         public String getId() {

core/src/java/scriptella/execution/ExecutionStatisticsBuilder.java

@@ -46,18 +46,37 @@ public class ExecutionStatisticsBuilder {
 
     //Execution stack for nested elements
     protected ElementsStack executionStack = new ElementsStack();
+
+    //current element (shortcut for performance reasons)
+    protected ExecutionStatistics.ElementInfo currentEInfo;
 
     /**
      * Called when new element execution started.
      *
      * @param loc element location.
      */
     public void elementStarted(final Location loc, Connection connection) {
-        ExecutionStatistics.ElementInfo ei = getInfo(loc);
-        executionStack.push(ei);
-        ei.statementsOnStart = connection.getExecutedStatementsCount();
-        ei.connection = connection;
-        ei.started = System.nanoTime();
+    		currentEInfo = getInfo(loc);
+        currentEInfo.started = System.nanoTime();
+        // check, because script elements will call elementStarted every time in the query loop
+        if (currentEInfo.execStart == null){
+        	currentEInfo.execStart = new Date();
+        }
+        executionStack.push(currentEInfo);
+        currentEInfo.statementsOnStart = connection.getExecutedStatementsCount();
+        currentEInfo.connection = connection;
+    }
+
+    public void incrementNestedExecutionCount(){
+    	currentEInfo.nestedExecutionCount++;
+    }
+
+    public void incrementFilteredExecutionCount(final Location loc){
+    	getInfo(loc).filteredExecutionCount++;
+    }
+
+    public void elementProcessingStarted(){
+    	currentEInfo.processingStart = new Date();
     }
 
     /**
@@ -74,7 +93,8 @@ public void etlStarted() {
         executionStatistics = new ExecutionStatistics();
         executionStatistics.setStarted(new Date());
     }
-
+
+
     /**
      * Invoked on ETL completion.
      */
@@ -93,6 +113,11 @@ public void etlComplete() {
      */
     private void setElementState(boolean ok) {
         ExecutionStatistics.ElementInfo ended = executionStack.pop();
+        if (executionStack.size() > 0){
+        	currentEInfo = executionStack.get(executionStack.size() -1);
+        }
+
+        ended.execEnd = new Date();
         long ti = System.nanoTime() - ended.started;
         ended.workingTime += ti; //increase the total working time for element
         if (ended.workingTime < 0) {
@@ -103,9 +128,9 @@ private void setElementState(boolean ok) {
         long conStatements = con.getExecutedStatementsCount();
         long elStatements = conStatements - ended.statementsOnStart;
         if (ok) {
-            ended.okCount++;
+            ended.successfulExecutionCount++;
         } else {
-            ended.failedCount++;
+            ended.failedExecutionCount++;
         }
 
         if (elStatements > 0) {