Merge pull request #1223 from ashitsalesforce/master

perf optimization - cache data from upload csv or db

src/main/java/com/salesforce/dataloader/dao/DAORowCache.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2015, salesforce.com, inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted provided
+ * that the following conditions are met:
+ *
+ *    Redistributions of source code must retain the above copyright notice, this list of conditions and the
+ *    following disclaimer.
+ *
+ *    Redistributions in binary form must reproduce the above copyright notice, this list of conditions and
+ *    the following disclaimer in the documentation and/or other materials provided with the distribution.
+ *
+ *    Neither the name of salesforce.com, inc. nor the names of its contributors may be used to endorse or
+ *    promote products derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+package com.salesforce.dataloader.dao;
+
+import java.util.ArrayList;
+
+import com.salesforce.dataloader.config.Config;
+import com.salesforce.dataloader.model.Row;
+
+public class DAORowCache {
+    private ArrayList<Row> rowList = new ArrayList<Row>();
+    private int currentRowIndex = 0;
+    private int totalRows = 0;
+
+    public DAORowCache() {
+    }
+
+    public void resetCurrentRowIndex() {
+        currentRowIndex = 0;
+    }
+
+    public Row getCurrentRow() {
+        Config config = Config.getCurrentConfig();
+        if (currentRowIndex >= totalRows
+            || !config.getBoolean(Config.PROCESS_BULK_CACHE_DATA_FROM_DAO)) {
+            return null;
+        }
+        return rowList.get(currentRowIndex++);
+    }
+
+    public void addRow(Row row) {
+        rowList.add(row);
+        currentRowIndex++;
+        totalRows++;
+    }
+}

src/main/java/com/salesforce/dataloader/dao/csv/CSVFileReader.java

@@ -43,6 +43,7 @@
 
 import com.salesforce.dataloader.config.Config;
 import com.salesforce.dataloader.config.Messages;
+import com.salesforce.dataloader.dao.DAORowCache;
 import com.salesforce.dataloader.dao.DataReader;
 import com.salesforce.dataloader.exception.DataAccessObjectException;
 import com.salesforce.dataloader.exception.DataAccessObjectInitializationException;
@@ -70,6 +71,8 @@ public class CSVFileReader implements DataReader {
     private boolean isOpen;
     private char[] csvDelimiters;
     private Config config;
+    private DAORowCache rowCache = new DAORowCache();
+    private boolean endOfFileReached = false;
 
     // Handles 3 types of CSV files:
     // 1. CSV files provided by the user for upload operations: ignoreDelimiterConfig = false, isQueryOperationResult = false
@@ -122,6 +125,7 @@ public void open() throws DataAccessObjectInitializationException {
             close();
         }
         currentRowNumber = 0;
+        rowCache.resetCurrentRowIndex();
 
         initalizeInput(csvDelimiters);
         readHeaderRow();
@@ -167,7 +171,18 @@ public Row readRow() throws DataAccessObjectException {
         if (!isOpen) {
             open();
         }
-
+
+        Row row = rowCache.getCurrentRow();
+        if (row != null) {
+            currentRowNumber++;
+            return row;
+        }
+
+        if (config.getBoolean(Config.PROCESS_BULK_CACHE_DATA_FROM_DAO)
+            && endOfFileReached) {
+            return null;
+        }
+
         List<String> record;
         synchronized (lock) {
             try {
@@ -178,6 +193,7 @@ record = csvReader.nextRecord();
         }
 
         if (!DAORowUtil.isValidRow(record)) {
+            endOfFileReached = true;
             return null;
         }
 
@@ -191,7 +207,7 @@ record = csvReader.nextRecord();
             throw new DataAccessRowException(errMsg);
         }
 
-        Row row = new Row(record.size());
+        row = new Row(record.size());
 
         for (int i = 0; i < headerRow.size(); i++) {
             String value = record.get(i);
@@ -201,6 +217,7 @@ record = csvReader.nextRecord();
             row.put(headerRow.get(i), value);
         }
         currentRowNumber++;
+        rowCache.addRow(row);
         return row;
     }
 

src/main/java/com/salesforce/dataloader/dao/database/DatabaseReader.java

@@ -36,6 +36,7 @@
 
 import com.salesforce.dataloader.config.Config;
 import com.salesforce.dataloader.config.Messages;
+import com.salesforce.dataloader.dao.DAORowCache;
 import com.salesforce.dataloader.dao.DataReader;
 import com.salesforce.dataloader.exception.*;
 import com.salesforce.dataloader.util.DAORowUtil;
@@ -60,6 +61,8 @@ public class DatabaseReader implements DataReader {
     private int currentRowNumber = 0;
     private final SqlConfig sqlConfig;
     private final DatabaseContext dbContext;
+    private boolean endOfDBReached = false;
+    private DAORowCache rowCache = new DAORowCache();
 
     /**
      * Get an instance of database reader for the data access object name from configuration
@@ -98,6 +101,7 @@ public DatabaseReader(Config config, String dbConfigName) throws DataAccessObjec
     @Override
     public void open() throws DataAccessObjectInitializationException {
         open(null);
+        rowCache.resetCurrentRowIndex();
     }
 
     /**
@@ -174,12 +178,20 @@ public List<Row> readRowList(int maxRows) throws DataAccessObjectException {
 
     @Override
     public Row readRow() throws DataAccessObjectException {
-        Row row = null;
-
         if (!dbContext.isOpen()) {
             open();
         }
-
+
+        Row row = rowCache.getCurrentRow();
+        if (row != null) {
+            currentRowNumber++;
+            return row;
+        }
+        if (config.getBoolean(Config.PROCESS_BULK_CACHE_DATA_FROM_DAO)
+            && endOfDBReached) {
+            return null;
+        }
+
         String currentColumnName = "";
         try {
             ResultSet rs = dbContext.getDataResultSet();
@@ -191,8 +203,12 @@ public Row readRow() throws DataAccessObjectException {
                     Object value = rs.getObject(columnName);
                     row.put(columnName, value);
                 }
+                rowCache.addRow(row);
                 currentRowNumber++;
             }
+            if (row == null) {
+                endOfDBReached = true;
+            }
             return row;
         } catch (SQLException sqe) {
             String errMsg = Messages.getFormattedString("DatabaseDAO.sqlExceptionReadRow", new String[] {

src/main/java/com/salesforce/dataloader/ui/DataSelectionDialog.java

@@ -123,7 +123,6 @@ protected void processingWithBusyIndicator(Shell shell) {
         List<String> header = null;
         int totalRows = 0;
         try {
-            dataReader.checkConnection();
             dataReader.open();
 
             String error = DAORowUtil.validateColumns(dataReader);