From 0b8660e411dafae74ed393c2525b4f986b34d451 Mon Sep 17 00:00:00 2001 From: ashitsalesforce Date: Thu, 15 Aug 2024 08:27:53 -0700 Subject: [PATCH] perf optimization - cache data from upload csv or db avoid reading entire csv file or db entries to be uploaded multiple times by caching it. --- .../dataloader/dao/DAORowCache.java | 59 +++++++++++++++++++ .../dataloader/dao/csv/CSVFileReader.java | 21 ++++++- .../dao/database/DatabaseReader.java | 22 ++++++- .../dataloader/ui/DataSelectionDialog.java | 1 - 4 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 src/main/java/com/salesforce/dataloader/dao/DAORowCache.java diff --git a/src/main/java/com/salesforce/dataloader/dao/DAORowCache.java b/src/main/java/com/salesforce/dataloader/dao/DAORowCache.java new file mode 100644 index 00000000..df38a198 --- /dev/null +++ b/src/main/java/com/salesforce/dataloader/dao/DAORowCache.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2015, salesforce.com, inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided + * that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this list of conditions and the + * following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and + * the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * Neither the name of salesforce.com, inc. nor the names of its contributors may be used to endorse or + * promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.salesforce.dataloader.dao; + +import java.util.ArrayList; + +import com.salesforce.dataloader.config.Config; +import com.salesforce.dataloader.model.Row; + +public class DAORowCache { + private ArrayList rowList = new ArrayList(); + private int currentRowIndex = 0; + private int totalRows = 0; + + public DAORowCache() { + } + + public void resetCurrentRowIndex() { + currentRowIndex = 0; + } + + public Row getCurrentRow() { + Config config = Config.getCurrentConfig(); + if (currentRowIndex >= totalRows + || !config.getBoolean(Config.PROCESS_BULK_CACHE_DATA_FROM_DAO)) { + return null; + } + return rowList.get(currentRowIndex++); + } + + public void addRow(Row row) { + rowList.add(row); + currentRowIndex++; + totalRows++; + } +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileReader.java b/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileReader.java index f9c930ec..d49fb7a4 100644 --- a/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileReader.java +++ b/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileReader.java @@ -43,6 +43,7 @@ import com.salesforce.dataloader.config.Config; import com.salesforce.dataloader.config.Messages; +import com.salesforce.dataloader.dao.DAORowCache; import com.salesforce.dataloader.dao.DataReader; import com.salesforce.dataloader.exception.DataAccessObjectException; import com.salesforce.dataloader.exception.DataAccessObjectInitializationException; @@ -70,6 +71,8 @@ public class CSVFileReader implements DataReader { private boolean isOpen; private char[] csvDelimiters; private Config config; + private DAORowCache rowCache = new DAORowCache(); + private boolean endOfFileReached = false; // Handles 3 types of CSV files: // 1. CSV files provided by the user for upload operations: ignoreDelimiterConfig = false, isQueryOperationResult = false @@ -122,6 +125,7 @@ public void open() throws DataAccessObjectInitializationException { close(); } currentRowNumber = 0; + rowCache.resetCurrentRowIndex(); initalizeInput(csvDelimiters); readHeaderRow(); @@ -167,7 +171,18 @@ public Row readRow() throws DataAccessObjectException { if (!isOpen) { open(); } - + + Row row = rowCache.getCurrentRow(); + if (row != null) { + currentRowNumber++; + return row; + } + + if (config.getBoolean(Config.PROCESS_BULK_CACHE_DATA_FROM_DAO) + && endOfFileReached) { + return null; + } + List record; synchronized (lock) { try { @@ -178,6 +193,7 @@ record = csvReader.nextRecord(); } if (!DAORowUtil.isValidRow(record)) { + endOfFileReached = true; return null; } @@ -191,7 +207,7 @@ record = csvReader.nextRecord(); throw new DataAccessRowException(errMsg); } - Row row = new Row(record.size()); + row = new Row(record.size()); for (int i = 0; i < headerRow.size(); i++) { String value = record.get(i); @@ -201,6 +217,7 @@ record = csvReader.nextRecord(); row.put(headerRow.get(i), value); } currentRowNumber++; + rowCache.addRow(row); return row; } diff --git a/src/main/java/com/salesforce/dataloader/dao/database/DatabaseReader.java b/src/main/java/com/salesforce/dataloader/dao/database/DatabaseReader.java index ec7b8600..69092165 100644 --- a/src/main/java/com/salesforce/dataloader/dao/database/DatabaseReader.java +++ b/src/main/java/com/salesforce/dataloader/dao/database/DatabaseReader.java @@ -36,6 +36,7 @@ import com.salesforce.dataloader.config.Config; import com.salesforce.dataloader.config.Messages; +import com.salesforce.dataloader.dao.DAORowCache; import com.salesforce.dataloader.dao.DataReader; import com.salesforce.dataloader.exception.*; import com.salesforce.dataloader.util.DAORowUtil; @@ -60,6 +61,8 @@ public class DatabaseReader implements DataReader { private int currentRowNumber = 0; private final SqlConfig sqlConfig; private final DatabaseContext dbContext; + private boolean endOfDBReached = false; + private DAORowCache rowCache = new DAORowCache(); /** * Get an instance of database reader for the data access object name from configuration @@ -98,6 +101,7 @@ public DatabaseReader(Config config, String dbConfigName) throws DataAccessObjec @Override public void open() throws DataAccessObjectInitializationException { open(null); + rowCache.resetCurrentRowIndex(); } /** @@ -174,12 +178,20 @@ public List readRowList(int maxRows) throws DataAccessObjectException { @Override public Row readRow() throws DataAccessObjectException { - Row row = null; - if (!dbContext.isOpen()) { open(); } - + + Row row = rowCache.getCurrentRow(); + if (row != null) { + currentRowNumber++; + return row; + } + if (config.getBoolean(Config.PROCESS_BULK_CACHE_DATA_FROM_DAO) + && endOfDBReached) { + return null; + } + String currentColumnName = ""; try { ResultSet rs = dbContext.getDataResultSet(); @@ -191,8 +203,12 @@ public Row readRow() throws DataAccessObjectException { Object value = rs.getObject(columnName); row.put(columnName, value); } + rowCache.addRow(row); currentRowNumber++; } + if (row == null) { + endOfDBReached = true; + } return row; } catch (SQLException sqe) { String errMsg = Messages.getFormattedString("DatabaseDAO.sqlExceptionReadRow", new String[] { diff --git a/src/main/java/com/salesforce/dataloader/ui/DataSelectionDialog.java b/src/main/java/com/salesforce/dataloader/ui/DataSelectionDialog.java index c8a8e1a8..5a90790f 100644 --- a/src/main/java/com/salesforce/dataloader/ui/DataSelectionDialog.java +++ b/src/main/java/com/salesforce/dataloader/ui/DataSelectionDialog.java @@ -123,7 +123,6 @@ protected void processingWithBusyIndicator(Shell shell) { List header = null; int totalRows = 0; try { - dataReader.checkConnection(); dataReader.open(); String error = DAORowUtil.validateColumns(dataReader);