Recover deleted draft submissions by soft-deleting them when updating…

… the form draft (#1299)

* wip: soft-delete draft submissions when drafts get replaced

* Fixing tests

* renaming and removing unused functions

* adding more tests

* Migration to support deleted draft subs with shared instanceIds

* More experimental tests

* Revert migration and let reused sub instance ids result in conflicts

* Update comment in lib/model/query/forms.js

Co-authored-by: Matthew White <matthew-white@users.noreply.github.com>

* another test

* exclude draft subs from submission getdeleted query for restoring

---------

Co-authored-by: Matthew White <matthew-white@users.noreply.github.com>

lib/model/query/forms.js

@@ -486,14 +486,20 @@ const _draftFilter = (form, project) =>
       ? sql`and forms."projectId" = ${project.id}`
       : sql``));
 
-// NOTE: copypasta alert! The following SQL also appears in 20220209-01-purge-unneeded-drafts.js
+// NOTE: copypasta alert! Similar SQL also appears in 20220209-01-purge-unneeded-drafts.js
+// Purges draft form defs that are not referenced by the form as either currentDefId or draftDefId AND have no associated submission defs.
 const clearUnneededDrafts = (form = null, project = null) => ({ run }) =>
   run(sql`
 DELETE FROM form_defs
   USING forms
 WHERE form_defs."formId" = forms.id
   AND form_defs."publishedAt" IS NULL
   AND form_defs.id IS DISTINCT FROM forms."draftDefId"
+  AND NOT EXISTS (
+      SELECT 1
+      FROM submission_defs
+      WHERE submission_defs."formDefId" = form_defs.id
+  )
 ${_draftFilter(form, project)}`)
     .then(() => run(sql`
 DELETE FROM form_schemas

lib/model/query/keys.js

@@ -37,7 +37,7 @@ INNER JOIN form_defs
 INNER JOIN submission_defs
   ON submission_defs."formDefId" = form_defs.id
 INNER JOIN submissions
-  ON submissions.id = submission_defs."submissionId"
+  ON submissions.id = submission_defs."submissionId" AND submissions."deletedAt" IS NULL
 WHERE submission_defs.current = true
   AND submission_defs."localKey" IS NOT NULL
   AND submissions.draft = ${draft}

lib/model/query/submissions.js

@@ -115,12 +115,12 @@ createVersion.audit.withResult = true;
 const update = (form, submission, data) => ({ one }) => one(updater(submission, data)).then(construct(Submission));
 update.audit = (form, submission, data) => (log) => log('submission.update', form, Object.assign({ submissionId: submission.id, submissionDefId: submission.def.id, instanceId: submission.def.instanceId }, data));
 
-const clearDraftSubmissions = (formId) => ({ run }) =>
-  run(sql`delete from submissions where "formId"=${formId} and draft=true`);
-
 const clearDraftSubmissionsForProject = (projectId) => ({ run }) =>
   run(sql`DELETE FROM submissions USING forms WHERE submissions."formId" = forms.id AND forms."projectId" = ${projectId} AND submissions.draft=true`);
 
+const deleteDraftSubmissions = (formId) => ({ run }) =>
+  run(sql`UPDATE submissions SET "deletedAt"=now() WHERE "formId"=${formId} AND "draft"=true AND "deletedAt" IS NULL`);
+
 ////////////////////////////////////////////////////////////////////////////////
 // SELECT-MULTIPLE VALUES
 
@@ -209,6 +209,7 @@ const getDeleted = (projectId, formId, instanceId) => ({ maybeOne }) =>
     and submissions."formId" = ${formId}
     and submissions."instanceId" = ${instanceId}
     and submissions."deletedAt" IS NOT NULL
+    and submissions."draft" = false
 `)
     .then(map(construct(Submission)));
 
@@ -475,7 +476,7 @@ select count(*) from deleted_submissions`);
 
 module.exports = {
   createNew, createVersion,
-  update, del, restore, purge, clearDraftSubmissions, clearDraftSubmissionsForProject,
+  update, del, restore, purge, clearDraftSubmissionsForProject, deleteDraftSubmissions,
   setSelectMultipleValues, getSelectMultipleValuesForExport,
   getByIdsWithDef, getSubAndDefById,
   getByIds, getAllForFormByIds, getById, countByFormId, verifyVersion,

lib/resources/forms.js

@@ -137,7 +137,7 @@ module.exports = (service, endpoint) => {
           : getPartial(Forms, request, project, Keys)))
         .then((partial) => Promise.all([
           Forms.createVersion(partial, form, false),
-          Submissions.clearDraftSubmissions(form.id)
+          Submissions.deleteDraftSubmissions(form.id)
         ]))
         .then(() => Forms.clearUnneededDrafts(form))) // remove drafts made obsolete by new draft
       .then(success)));
@@ -162,7 +162,7 @@ module.exports = (service, endpoint) => {
           .then(() => Forms.getByProjectAndXmlFormId(params.projectId, params.id, false, Form.DraftVersion))
           .then(getOrNotFound)
         : resolve(form)))
-      .then(((form) => Promise.all([ Forms.publish(form), Submissions.clearDraftSubmissions(form.id) ])))
+      .then(((form) => Promise.all([ Forms.publish(form), Submissions.deleteDraftSubmissions(form.id) ])))
       .then(success)));
 
   // Entity/Dataset-specific endpoint that is used to show how publishing
@@ -244,7 +244,7 @@ module.exports = (service, endpoint) => {
       .then(rejectIf(((form) => form.draftDefId == null), noargs(Problem.user.notFound)))
       .then((form) => Promise.all([
         Forms.clearDraft(form).then(() => Forms.clearUnneededDrafts(form)),
-        Submissions.clearDraftSubmissions(form.id),
+        Submissions.deleteDraftSubmissions(form.id),
         Audits.log(auth.actor, 'form.update.draft.delete', form, { oldDraftDefId: form.draftDefId })
       ]))
       .then(success)));

lib/task/purge.js

@@ -21,8 +21,13 @@ const purgeTask = task.withContainer((container) => async (options = {}) => {
         const count = await Forms.purge(options.force, options.formId, options.projectId, options.xmlFormId);
         return `Forms purged: ${count}`;
       } else {
+        // Purge both Forms and Submissions according to options
         const formCount = await Forms.purge(options.force, options.formId, options.projectId, options.xmlFormId);
         const submissionCount = await Submissions.purge(options.force, options.projectId, options.xmlFormId, options.instanceId);
+
+        // Related to form purging: deletes draft form defs that are not in use by any form and have no associated submission defs
+        await Forms.clearUnneededDrafts();
+
         return `Forms purged: ${formCount}, Submissions purged: ${submissionCount}`;
       }
     } catch (error) {