Smart handling of new/updated entries when importing (#559)

* Main updates to UI and to the handling of determining entry IDs and the insert/update behaviour

* Fixing up detection of user names, etc

* Instantiating some variables we need

* handling linked checkboxes, and interpreting uitexts

* Turning on Import Data button even when there's a relationship

* Improving filename of blank template

* Force tab char when generating templates, presuming Excel is the platform

* Adding tip about importing only to the main form

modules/formulize/include/entriesdisplay.php

@@ -1343,7 +1343,7 @@ function drawInterface($settings, $fid, $frid, $groups, $mid, $gperm_handler, $l
 			$screenButtonText['addProxyButton'] = "";
 		}
 		$screenButtonText['exportButton'] = !$lockcontrols ? $screen->getVar('useexport') : "";
-		$screenButtonText['importButton'] = ($import_data = $gperm_handler->checkRight("import_data", $fid, $groups, $mid) AND !$frid) ? $screen->getVar('useimport') : "";
+		$screenButtonText['importButton'] = $import_data = $gperm_handler->checkRight("import_data", $fid, $groups, $mid) ? $screen->getVar('useimport') : "";
 		$screenButtonText['notifButton'] = $screen->getVar('usenotifications');
 		$screenButtonText['currentViewList'] = $screen->getVar('usecurrentviewlist');
 		$screenButtonText['saveButton'] = !$lockcontrols ? $screen->getVar('desavetext') : "";

modules/formulize/include/functions.php

@@ -1228,7 +1228,7 @@ function prepExport($headers, $cols, $data, $fdchoice, $custdel, $template, $fid
             $lineStarted = true;
         } else {
             if ($template == "update") {
-                $csvfile = "\"" . _formulize_DE_IMPORT_IDREQCOL . "\"$fd\"" . _formulize_DE_CALC_CREATOR . "\"";
+                $csvfile = "\"" . _formulize_ENTRY_ID . "\"$fd\"" . _formulize_DE_CALC_CREATOR . "\"";
                 $lineStarted = true;
             } else {
                 $csvfile = "\"" . _formulize_DE_CALC_CREATOR . "\"";
@@ -1358,8 +1358,15 @@ function prepExport($headers, $cols, $data, $fdchoice, $custdel, $template, $fid
     // grab and output any secondary data for the last entry, if there was any
     $csvfile = prepExportSecondaryData($csvfile, $cols, $fd, $secondaryData);
 
-    $tempfold = microtime(true);
-    $exfilename = _formulize_DE_XF . $tempfold . $fxt;
+		$form_handler = xoops_getmodulehandler('forms','formulize');
+    $formObject = $form_handler->get($fid);
+    if (is_object($formObject)) {
+        $formTitle = "'".str_replace(array(" ", "-", "/", "'", "`", "\\", ".", "?", ",", ")", "(", "[", "]"), "_", trans(undoAllHTMLChars($formObject->getVar('title'))))."'";
+    } else {
+        $formTitle = "a_form";
+    }
+
+    $exfilename = _formulize_EXPORT_FILENAME_TEXT."_".$formTitle."_".date("M_j_Y_Hi").".csv";
 
     // open the output file for writing
     $wpath = XOOPS_ROOT_PATH. SPREADSHEET_EXPORT_FOLDER . "$exfilename";
@@ -1369,15 +1376,7 @@ function prepExport($headers, $cols, $data, $fdchoice, $custdel, $template, $fid
     fclose($exportfile);
 
     // garbage collection. delete files older than 6 hours
-    formulize_scandirAndClean(XOOPS_ROOT_PATH . SPREADSHEET_EXPORT_FOLDER, _formulize_DE_XF);
-
-    // write id_reqs and tempfold to the DB if we're making an update template
-    if ($template == "update") {
-        $sql = "INSERT INTO " . $xoopsDB->prefix("formulize_valid_imports") . " (file, id_reqs) VALUES (\"$tempfold\", \"" . serialize($id_req) . "\")";
-        if (!$res = $xoopsDB->queryF($sql)) {
-            exit("Error: could not write import information to the database.  SQL: $sql<br>".$xoopsDB->error());
-        }
-    }
+    formulize_scandirAndClean(XOOPS_ROOT_PATH . SPREADSHEET_EXPORT_FOLDER, _formulize_EXPORT_FILENAME_TEXT);
 
     return XOOPS_URL . SPREADSHEET_EXPORT_FOLDER . "$exfilename";
 }
@@ -1410,7 +1409,8 @@ function prepareCellForSpreadsheetExport($column, $entry) {
             case 1:
             default:
                 // Google wants a ' and Excel wants a tab...assume makecsv is going to be imported into Google, and otherwise we're downloading for Excel - default preference for handling strings in csv's, so they import without being mangled. Setting for no intro char may be useful when exporting to other programs that suck in raw data.
-                $exportIntroChar = strstr(getCurrentURL(),'makecsv') ? "'" : "\t";
+								// Exception: if makecsv is called from the import.php popup, because we need to use it there to make a simple spreadsheet, that will probably be edited on a desktop with Excel
+                $exportIntroChar = (strstr(getCurrentURL(),'makecsv') AND !strstr($_SERVER['HTTP_REFERER'], '/modules/formulize/include/import.php')) ? "'" : "\t";
         }
     }
 

modules/formulize/language/english/main.php

@@ -1,5 +1,6 @@
 <?php
 // Module main
+define("_formulize_ENTRY_ID", "Entry ID");
 define("_formulize_FORM_TITLE", "Contact us by filling out this form.");
 define("_AM_CATGENERAL", "General Forms");
 define("_AM_NOFORMS_AVAIL", "There are no forms currently available.");
@@ -657,36 +658,37 @@
 // import
 define("_formulize_DE_IMPORT", "Import Data");
 define("_formulize_DE_IMPORT_RESULTS", "Results...");
-define("_formulize_DE_IMPORT_STEP1", "Step 1: download a blank template file or a file with data");
-define("_formulize_DE_IMPORT_STEP2", "Step 2: modify the file you downloaded");
-define("_formulize_DE_IMPORT_STEP3", "Step 3: upload the modified file");
-define("_formulize_DE_IMPORT_FILE", "Select the file you modified and upload it.");
+define("_formulize_DE_IMPORT_STEP1", "Download Spreadsheets:");
+define("_formulize_DE_IMPORT_STEP3", "Tips");
+define("_formulize_DE_IMPORT_STEP2", "Upload a spreadsheet with data to import:");
 define("_formulize_DE_IMPORT_GO", "Upload");
 define("_formulize_DE_IMPORT_VALIDATEDATA", "Automatically validate the structure and data in the .csv file before actually importing (highly recommended!)");
 define("_formulize_DE_IMPORT_UPDATEDERIVED", "Automatically updated derived values in the form");
 define("_formulize_DE_IMPORT_SENDNOTIFICATIONS", "Automatically send notifications to users based on the imported data");
+define("_formulize_DE_IMPORT_IDENTIFIER_COLUMN", "This column uniquely identifies each entry in the spreadsheet:");
+define("_formulize_DE_IMPORT_USEPKASID", "Advanced option: the value of the unique identifier column is also the internal entry id");
 define("_formulize_DE_IMPORT_INSTRUCTIONS", "<p><b>Open the file you downloaded in a spreadsheet program, like Excel, and modify it</b> so it contains the data you want to upload.  You don't have to do this right now, you can modify the file and come back later to upload it.</p>
 
-<p>When you save the file, <b>make sure you save it in .csv format.</b>  If you save it in a different format, like .xls, then the import process won't work!</p>");
+<p>When you save the file, <b>make sure you save it in .csv format.</b>  If you save it in a different format, like .xls, then the import process won't work!</p>
+
+<p><b>You can only import data to the main form</b>, not any subforms or other connected forms. To import data to a connected form, you need to use a screen that is based on that form. You should import data to the primary form first, so that when you import data into other forms, they can link to the entries you have already imported. ie: Import the 'Countries' data first, then import the 'Cities.' That way, if the City form contains a link to its Country entry, the Country data will already be in place.</p>
+");
 
 define("_formulize_DE_IMPORT_INSTNEWPROFILE","<p><b>If you are creating new entries in the user profile form,</b> you must include a unique username, a full name, a password, a unique e-mail address and a valid registration code for each entry.  A new user account will be created for each entry, based on the information you provide.</p>");
 
-define("_formulize_DE_IMPORT_INSTUPDATE", "<p><b>If you are updating existing entries, do not change or remove the '_1148849956' part of the file name!</b>  Do not add any other '_' characters either.  Also, <b>do not alter the ID numbers</b> in each row of the file.  All that information uniquely identifies the entries associated with each row.</p>
+define("_formulize_DE_IMPORT_INSTUPDATE", "<p>To update existing entries, <b>make sure you leave the values in the '"._formulize_ENTRY_ID."' column unchanged</b>, because that is how Formulize can tell which entry you are editing. You can also choose to use a different column to identify the entries, as long as it has a unique value for every entry (such as a student number, employee number, etc).</p>
 
-<p><b>Every row in the spreadsheet (after the headings) represents one entry in the form.</b>  So if you want to import three entries, then you need to have three rows of data in the spreadsheet. The order of the rows does not matter. If you are updating entries and you delete rows from the spreadsheet, that <i>will not</i> delete those entries from the database.</p>
+<p><b>Every row in the spreadsheet (after the headings) represents one entry in the form.</b>  So if you want to import three entries, then you need to have three rows of data in the spreadsheet. The order of the rows does not matter. If you download a spreadsheet with all your data, and you delete rows from the spreadsheet and then upload it, that <i>will not</i> delete those entries from the database. That will simply update the entries that you did include in the spreadsheet.</p>
 
 <p><b>If a question has a choice of answers,</b> the information in your spreadsheet must match exactly with the options in the form. This includes spelling, capitalization and spacing.  Some questions in some forms allow you to select more than one answer; for instance, a series of checkboxes.  <b>To include multiple answers in your spreadsheet,</b> each answer must be in the same cell with a line break between them.  In Excel, press ALT-Enter after each answer to add a line break.</p>");
 
 define("_formulize_DE_IMPORT_NEWENTRYID", "Use this entry id");
-define("_formulize_DE_IMPORT_INSTNEW", "<p><b>If you are creating new entries,</b> then the column called \"" . _formulize_DE_CALC_CREATOR . "\" can have the username or full name of the person who should be recorded as the entry's creator.  If you leave that column blank, then you will be recorded as the creator.  If you are updating existing entries, then this column is ignored.</p><p><b>If you are creating new entries,</b> and you want to override the primary key that the database gives to each entry, then you can include a column called \"" . _formulize_DE_IMPORT_NEWENTRYID . "\" and put the entry ids you want to use in that column.  If you don't understand what this means, then simply don't add this column to your spreadsheet.</p>");
+define("_formulize_DE_IMPORT_INSTNEW", "<p><b>If you are creating new entries,</b> then the column called '" . _formulize_DE_CALC_CREATOR . "' can have the username or full name of the person who should be recorded as the entry's creator.  If you leave that column blank, then you will be recorded as the creator.  If you are updating existing entries, then this column is ignored.</p>");
 
 define("_formulize_DE_IMPORT_BACK", "Go Back");
-define("_formulize_DE_IMPORT_EITHEROR", "You can either add new entries to a form, or update existing entries.  You <b>cannot</b> do both at once.");
 define("_formulize_DE_IMPORT_OR", "OR");
-define("_formulize_DE_IMPORT_BLANK", "If you want to add new entries to this form...");
-define("_formulize_DE_IMPORT_BLANK2", "Right-click here and save the necessary template.");
-define("_formulize_DE_IMPORT_DATATEMP", "If you want to update entries in this form...");
-define("_formulize_DE_IMPORT_DATATEMP2", "Click here to get a template with your entries in it.");
+define("_formulize_DE_IMPORT_BLANK2", "Blank for making new entries");
+define("_formulize_DE_IMPORT_DATATEMP2", "With data for editing entries");
 define("_formulize_DE_IMPORT_DATATEMP3", "Templates always include all columns regardless of what columns are currently selected.  Templates only include the rows (entries) that are currently visible.  To include all entries, turn off all searches and other filters.");
 define("_formulize_DE_IMPORT_DATATEMP4", "After you have downloaded the template, and made your changes,");
 define("_formulize_DE_IMPORT_DATATEMP5", "go back to the import page and upload your template.");
@@ -762,8 +764,6 @@
 
 define("_formulize_CLOSE_FORM_ELEMENT", "You need to close the form element that is open first before you edit this one");
 
-define("_formulize_ENTRY_ID", "Entry ID");
-
 define("_formulize_FROM", "From:");
 define("_formulize_TO", "To:");
 

modules/formulize/language/french/main.php

@@ -313,17 +313,13 @@
 define("_formulize_DE_IMPORT_FULLNAME", "Nom complet");
 define("_formulize_DE_IMPORT_GO", "Unvoyer");
 define("_formulize_DE_IMPORT_IDREQCOL", "ID number of this entry (do NOT remove or modify this column)");
-define("_formulize_DE_IMPORT_INSTNEW", "<p><b>If you are creating new entries,</b> then the column called \"" . _formulize_DE_CALC_CREATOR . "\" can have the username or full name of the person who should be recorded as the entry's creator.  If you leave that column blank, then you will be recorded as the creator.  If you are updating existing entries, then this column is ignored.</p>");
-define("_formulize_DE_IMPORT_INSTNEWPROFILE","<p><b>If you are creating new entries in the user profile form,</b> you must include a unique username, a full name, a password, a unique e-mail address and a valid registration code for each entry.  A new user account will be created for each entry, based on the information you provide.</p>");
-define("_formulize_DE_IMPORT_INSTRUCTIONS", "<p><b>Open the file you downloaded in a spreadsheet program, like Excel, and modify it</b> so it contains the data you want to upload.  You don't have to do this right now, you can modify the file and come back later to upload it.</p><p>When you save the file, <b>make sure you save it in .csv format.</b>  If you save it in a different format, like .xls, then the import process won't work!</p>");
-define("_formulize_DE_IMPORT_INSTUPDATE", "<p><b>If you are updating existing entries, do not change or remove the '_1148849956' part of the file name!</b>  Do not add any other '_' characters either.  Also, <b>do not alter the ID numbers</b> in each row of the file.  All that information uniquely identifies the entries associated with each row.</p><p><b>Every row in the spreadsheet (after the headings) represents one entry in the form.</b>  So if you want to import three entries, then you need to have three rows of data in the spreadsheet. The order of the rows does not matter. If you are updating entries and you delete rows from the spreadsheet, that <i>will not</i> delete those entries from the database.</p><p><b>If a question has a choice of answers,</b> the information in your spreadsheet must match exactly with the options in the form. This includes spelling, capitalization and spacing.  Some questions in some forms allow you to select more than one answer; for instance, a series of checkboxes.  <b>To include multiple answers in your spreadsheet,</b> each answer must be in the same cell with a line break between them.  In Excel, press ALT-Enter after each answer to add a line break.</p>");
 define("_formulize_DE_IMPORT_OR", "OR");
 define("_formulize_DE_IMPORT_PASSWORD", "Mot de passe");
 define("_formulize_DE_IMPORT_REGCODE", "Registration Code");
 define("_formulize_DE_IMPORT_RESULTS", "Resultats...");
 define("_formulize_DE_IMPORT_STEP1", "Etape 1: télécharger un fichier blanc ou avec des données");
-define("_formulize_DE_IMPORT_STEP2", "Etape 2: modifier le fichier téléchargé");
-define("_formulize_DE_IMPORT_STEP3", "Etape 3: envoyer le fichier modifié");
+define("_formulize_DE_IMPORT_STEP2", "");
+define("_formulize_DE_IMPORT_STEP3", "Etape 2: envoyer le fichier modifié");
 define("_formulize_DE_IMPORT_USERNAME", "Nom d'utilisateur");
 define("_formulize_DE_IMPORT_VALIDATEDATA", "Faire une validation automatique de la structure et des données dans le fichier .csv avant l'import actuel (vraiment recommandé!)");
 define("_formulize_DE_IMPORTDATA", "Importer les entrées");

modules/formulize/sql/mysql.sql

@@ -270,14 +270,6 @@ CREATE TABLE `formulize_screen` (
 
 ) ENGINE=InnoDB;
 
-CREATE TABLE formulize_valid_imports (
-  import_id smallint(5) NOT NULL auto_increment,
-  file varchar(255) NOT NULL default '',
-  id_reqs text NOT NULL,
-  fid int(5),
-  PRIMARY KEY (`import_id`)
-) ENGINE=InnoDB;
-
 CREATE TABLE formulize_notification_conditions (
   not_cons_id smallint(5) NOT NULL auto_increment,
   not_cons_fid smallint(5) NOT NULL default 0,

modules/formulize/xoops_version.php

@@ -67,7 +67,7 @@
 	"formulize_screen_multipage",
 	"formulize_screen_listofentries",
 	"formulize_screen_template",
-    "formulize_screen_calendar",
+  "formulize_screen_calendar",
 	"formulize_entry_owner_groups",
 	"formulize_application_form_link",
 	"formulize_applications",
@@ -78,11 +78,11 @@
 	"formulize_procedure_logs",
 	"formulize_procedure_logs_params",
 	"formulize_deletion_logs",
-    "formulize_apikeys",
+  "formulize_apikeys",
 	"formulize_tokens",
-    "formulize_digest_data",
-    "formulize_passcodes",
-    "tfa_codes"
+  "formulize_digest_data",
+  "formulize_passcodes",
+  "tfa_codes"
 );
 
 $modversion['formulize_exportable_tables'] = array(
@@ -238,7 +238,6 @@
             )
         )
     ),
-    "formulize_valid_imports" => array(),
     "formulize_screen" => array(
         "fields" => array("title", "type"),
         "joins" => array(