DKAN-4287 Make harvest_run id not a primary key.

modules/harvest/harvest.install

@@ -1,5 +1,7 @@
 <?php
 
+use Drupal\harvest\HarvestUtility;
+
 /**
  * @file
  */
@@ -95,8 +97,102 @@ function harvest_update_8007(&$sandbox) {
  * This finishes the process started by harvest_update_8007.
 */
 function harvest_update_8008(&$sandbox) {
+  // Moved and repeated to 8010.
+}
+
+/**
+ * Update harvest_run schema to add timestamp, uuid, and true id.
+ *
+ * @see https://github.com/GetDKAN/dkan/issues/4287
+ */
+function harvest_update_8009(&$sandbox) {
+  $table_name = 'harvest_runs';
+  $table_name_temp = "{$table_name}_temp";
+  $entity_type_name = 'harvest_run';
+
+  $definition_update_manager = \Drupal::entityDefinitionUpdateManager();
+  $entity_type_manager = \Drupal::entityTypeManager();
+  $schema = \Drupal::database()->schema();
+
+  // Move the table so we can rebuild from it.
+  $schema->renameTable($table_name, $table_name_temp);
+  $messages = "Table {$table_name} moved to {$table_name_temp}. " . PHP_EOL;
+  // Uninstall the the original entity.
+  $original_type = $definition_update_manager->getEntityType($entity_type_name);
+  $definition_update_manager->uninstallEntityType($original_type);
+  $messages .= "Old harvest_run entity removed. " . PHP_EOL;
+  $entity_type_manager->clearCachedDefinitions();
+  // Install the new entity.
+  //$entity_type = $entity_type_manager->get($entity_type_name);
+  $entity_type_manager->clearCachedDefinitions();
+  $entity_type_def = $entity_type_manager->getDefinition($entity_type_name);
+  $definition_update_manager->installEntityType($entity_type_def);
+  $messages .= "New harvest_run entity installed. " . PHP_EOL;
+
+  return $messages;
+}
+
+/**
+ * Move data from temp table back into harvest_run.
+ *
+ * @see https://github.com/GetDKAN/dkan/issues/4287
+ */
+function harvest_update_8010(&$sandbox) {
+  $table_name = 'harvest_runs';
+  $table_name_temp = "{$table_name}_temp";
+  $messages = '';
+  $schema = \Drupal::database()->schema();
   /** @var \Drupal\harvest\HarvestUtility $harvest_utility */
   $harvest_utility = \Drupal::service('dkan.harvest.utility');
-  $harvest_utility->harvestRunsUpdate();
-  return 'Harvest runs coalesced into table harvest_runs.';
+
+  if (!isset($sandbox['total'])) {
+    // Sandbox has not been initiated, so initiate it.
+    $sandbox['items_to_process'] = $harvest_utility->getTempRunIdsForUpdate($table_name_temp);
+    $sandbox['total'] = count($sandbox['items_to_process']);
+    $sandbox['current'] = 0;
+  }
+  // Process them in batches of 25.
+  $harvest_runs_batch = array_slice($sandbox['items_to_process'], 0, 25, TRUE);
+  // Loop through all the entries in temp table and save them new.
+  foreach ($harvest_runs_batch as $key => $time_id) {
+    // Load the old row.
+    $row = $harvest_utility->readTempHarvestRunForUpdate($table_name_temp, $time_id);
+    // Write the new harvest run.
+    $harvest_utility->writeHarvestRunFromUpdate($row['id'], $row['harvest_plan_id'], $row['data'], $row['extract_status']);
+    // The item has been processed, remove it from the array.
+    unset($sandbox['items_to_process'][$key]);
+  }
+
+  // Determine when to stop batching.
+  $sandbox['current'] = ($sandbox['total'] - count($sandbox['items_to_process']));
+  $sandbox['#finished'] = (empty($sandbox['total'])) ? 1 : ($sandbox['current'] / $sandbox['total']);
+  $vars = [
+    '@completed' => $sandbox['current'],
+    '@total' => $sandbox['total'],
+  ];
+
+  $messages = t('Processed: @completed/@total.', $vars) . PHP_EOL;
+  // Log the all finished notice.
+  if ($sandbox['#finished'] === 1) {
+    // The update of the harvest_runs is complete.
+    $messages .= t('Data in harvest_runs updated to new schema:') . PHP_EOL;
+    $dropped = $schema->dropTable($table_name_temp);
+    if ($dropped) {
+      $messages .= t('Temporary table dropped.') . PHP_EOL;
+    }
+  }
+
+  return $messages;
+}
+
+/**
+ * Move entries from harvest_[ID]_runs to harvest_runs.
+ *
+ * This finishes the process started by harvest_update_8007 and re-runs 8008.
+*/
+function harvest_update_8011(&$sandbox) {
+  /** @var \Drupal\harvest\HarvestUtility $harvest_utility */
+    $harvest_utility = \Drupal::service('dkan.harvest.utility');
+    $harvest_utility->harvestRunsUpdate();
+    return 'Harvest plan specific run tables coalesced into table harvest_runs.';
 }

modules/harvest/harvest.services.yml

@@ -19,6 +19,7 @@ services:
       - '@dkan.harvest.storage.harvest_run_repository'
       - '@database'
       - '@dkan.harvest.logger_channel'
+      - '@uuid'
   dkan.harvest.harvest_plan_repository:
     class: Drupal\harvest\Entity\HarvestPlanRepository
     arguments:

modules/harvest/src/Commands/HarvestCommands.php

@@ -221,11 +221,6 @@ public function runAll($options = ['new' => FALSE]) {
     foreach ($plan_ids as $plan_id) {
       $result = $this->harvestService->runHarvest($plan_id);
       $runs_info[] = $result;
-      // Since run IDs are also one-second-resolution timestamps, we must wait
-      // one second before running the next harvest.
-      // @todo Remove this sleep when we've switched to a better system for
-      //   timestamps.
-      sleep(1);
     }
     $this->renderHarvestRunsInfo($runs_info);
   }

modules/harvest/src/Entity/HarvestRun.php

@@ -52,9 +52,6 @@
  *     "canonical" = "/harvest-run/{harvest_run}",
  *   },
  * )
- *
- * @todo Convert to using microtime() or other better system for the timestamp/
- *   id.
  */
 final class HarvestRun extends HarvestEntityBase implements HarvestRunInterface {
 
@@ -64,11 +61,25 @@ final class HarvestRun extends HarvestEntityBase implements HarvestRunInterface
   public static function baseFieldDefinitions(EntityTypeInterface $entity_type) {
     $base_fields = parent::baseFieldDefinitions($entity_type);
 
-    // The id is the unique ID for the harvest run, and also the timestamp at
-    // which the run occurred, generated by time().
-    $base_fields['id'] = static::getBaseFieldIdentifier(
-      new TranslatableMarkup('Harvest Run')
-    );
+    $base_fields['id'] = BaseFieldDefinition::create('integer')
+      ->setLabel(t('ID'))
+      ->setDescription(t('The ID of the Harvest Run entity.'))
+      ->setDisplayOptions('view', [
+        'label' => 'inline',
+        'weight' => 0,
+      ])
+      ->setReadOnly(TRUE);
+
+    $base_fields['timestamp'] = BaseFieldDefinition::create('timestamp')
+      ->setLabel(t('timestamp'))
+      ->setDescription(t('The timestamp of when this harvest was run.'))
+      ->setRequired(TRUE);
+
+    $base_fields['uuid'] = BaseFieldDefinition::create('uuid')
+      ->setLabel(t('UUID'))
+      ->setDescription(t('The unique identifier for this harvest_run'))
+      ->setRequired(TRUE)
+      ->setReadOnly(TRUE);
 
     // Harvest plan id. This is the name of the harvest plan as seen in the UI.
     $base_fields['harvest_plan_id'] = BaseFieldDefinition::create('string')

modules/harvest/src/Entity/HarvestRunRepository.php

@@ -24,7 +24,7 @@ class HarvestRunRepository {
    *
    * @var \Drupal\Core\Entity\EntityStorageInterface
    */
-  private EntityStorageInterface $runStorage;
+  protected EntityStorageInterface $runStorage;
 
   /**
    * Database connection service.
@@ -89,8 +89,8 @@ public function destructForPlanId(string $plan_id) {
    *   Run data. Usually the result returned by Harvester::harvest().
    * @param string $plan_id
    *   The plan identifier.
-   * @param string $run_id
-   *   The run identifier, which is also a timestamp.
+   * @param string $timestamp
+   *   The run timestamp.
    *
    * @return string
    *   The run identifier.
@@ -100,9 +100,9 @@ public function destructForPlanId(string $plan_id) {
    * @todo Eventually all the subsystems will be able to understand the entity
    *   rather than needing conversion to and from the array format.
    */
-  public function storeRun(array $run_data, string $plan_id, string $run_id): string {
+  public function storeRun(array $run_data, string $plan_id, string $timestamp): string {
     $field_values = [
-      'id' => $run_id,
+      'timestamp' => (int) $timestamp,
       'harvest_plan_id' => $plan_id,
     ];
     $field_values['extract_status'] = $run_data['status']['extract'] ?? 'FAILURE';
@@ -139,23 +139,26 @@ public function storeRun(array $run_data, string $plan_id, string $run_id): stri
     // JSON encode remaining run data.
     $field_values['data'] = json_encode($run_data);
 
-    return $this->writeEntity($field_values, $plan_id, $run_id);
+    return $this->writeEntity($field_values, $plan_id, $timestamp);
   }
 
   /**
    * Retrieve the JSON-encoded data for the given plan and run IDs.
    *
    * @param string $plan_id
    *   The harvest plan identifier.
-   * @param string $run_id
-   *   The harvest run identifier.
+   * @param string|null $timestamp
+   *   The harvest run timestamp. No longer used.  Retained for BC.
    *
    * @return string|null
    *   JSON-encoded run result data, or NULL if none could be found.
    */
-  public function retrieveRunJson(string $plan_id, string $run_id): ?string {
-    if ($entity = $this->loadEntity($plan_id, $run_id)) {
-      return json_encode($entity->toResult());
+  public function retrieveRunJson(string $plan_id, $timestamp = NULL): ?string {
+    $run_ids = $this->retrieveAllRunIds($plan_id);
+    $run_id = reset($run_ids);
+    if ($run_id) {
+      $run_entity = $this->runStorage->load($run_id);
+      return json_encode($run_entity->toResult());
     }
     return NULL;
   }
@@ -198,7 +201,7 @@ public function retrieveAllRunsJson(string $plan_id): array {
   }
 
   /**
-   * Get all the harvet plan ids available in the harvest runs table.
+   * Get all the harvest plan ids available in the harvest runs table.
    *
    * @return array
    *   All the harvest plan ids present in the harvest runs table, as both key
@@ -218,18 +221,18 @@ public function getUniqueHarvestPlanIds(): array {
   /**
    * Get the extracted UUIDs from the given harvest run.
    *
-   * @param string $plan_id
-   *   The harvest plan ID.
-   * @param string $run_id
-   *   The harvest run ID.
+   * @param string $planId
+   *   The harvest plan ID.  deprecated: no longer needed but kept for BC.
+   * @param string $runId
+   *   The harvest_run entity id.
    *
    * @return string[]
    *   Array of UUIDs, keyed by UUID. Note that these are UUIDs by convention;
    *   they could be any string value.
    */
-  public function getExtractedUuids(string $plan_id, string $run_id): array {
+  public function getExtractedUuids(string $planId, string $runId): array {
     $extracted = [];
-    if ($entity = $this->loadEntity($plan_id, $run_id)) {
+    if ($entity = $this->runStorage->load($runId)) {
       foreach ($entity->get('extracted_uuid')->getValue() as $field) {
         $uuid = $field['value'];
         $extracted[$uuid] = $uuid;
@@ -243,16 +246,17 @@ public function getExtractedUuids(string $plan_id, string $run_id): array {
    *
    * @param string $plan_id
    *   Plan ID.
-   * @param string $run_id
-   *   Run ID, which is a timestamp.
+   * @param string $timestamp
+   *   The timestamp for the run. Formerly the id.
    *
    * @return \Drupal\harvest\HarvestRunInterface|\Drupal\Core\Entity\EntityInterface|null
    *   The loaded entity or NULL if none could be loaded.
    */
-  public function loadEntity(string $plan_id, string $run_id): ?HarvestRunInterface {
+  public function loadEntity(string $plan_id, string $timestamp): ?HarvestRunInterface {
     if ($ids = $this->runStorage->getQuery()
-      ->condition('id', $run_id)
+      ->condition('timestamp', $timestamp)
       ->condition('harvest_plan_id', $plan_id)
+      ->sort('id', 'DESC')
       ->range(0, 1)
       ->accessCheck(FALSE)
       ->execute()
@@ -262,28 +266,59 @@ public function loadEntity(string $plan_id, string $run_id): ?HarvestRunInterfac
     return NULL;
   }
 
+  /**
+   * Helper method to load the most recent harvest_run entity given a plan ID.
+   *
+   * @param string $harvest_plan_id
+   *   Plan ID.
+   *
+   * @return \Drupal\harvest\HarvestRunInterface|null
+   *   The loaded harvest_run entity or NULL if none could be loaded.
+   */
+  public function loadRunByPlan($harvest_plan_id): ?HarvestRunInterface {
+    $run_id = $this->getLastHarvestRunId($harvest_plan_id);
+    return ($run_id) ? $this->runStorage->load($run_id) : NULL;
+  }
+
+  /**
+   * Get a harvest's most recent run identifier.
+   *
+   * Since the run record id is a timestamp, we can sort on the id.
+   *
+   * @param string $plan_id
+   *   The harvest plan identifier.
+   *
+   * @return string
+   *   The entity id of the most recent harvest run.
+   */
+  public function getLastHarvestRunId(string $plan_id): string {
+    $run_ids = $this->retrieveAllRunIds($plan_id);
+    return reset($run_ids);
+  }
+
   /**
    * Write a harvest_run entity, updating or saving as needed.
    *
    * @param array $field_values
    *   Structured data ready to send to entity_storage->create().
    * @param string $plan_id
    *   Harvest plan identifier.
-   * @param string $run_id
-   *   Harvest run identifier.
+   * @param mixed $timestamp
+   *   Harvest run timestamp.
    *
    * @return string
-   *   Harvest plan identifier for the entity that was written.
+   *   Harvest run id.
    */
-  public function writeEntity(array $field_values, string $plan_id, string $run_id) {
+  public function writeEntity(array $field_values, string $plan_id, mixed $timestamp) {
+    $timestamp = (int) $timestamp;
     /** @var \Drupal\harvest\HarvestRunInterface $entity */
-    $entity = $this->loadEntity($plan_id, $run_id);
+    $entity = $this->loadEntity($plan_id, $timestamp);
     if ($entity) {
       // Modify entity.
-      unset($field_values['id']);
       foreach ($field_values as $key => $value) {
         $entity->set($key, $value);
       }
+      $field_values['id'] = $entity->id();
     }
     else {
       // Create new entity.