From 53b8cd8a905c95c706041371b674477409e2c98d Mon Sep 17 00:00:00 2001
From: Keith Lawrence <keith.lawrence@digital.cabinet-office.gov.uk>
Date: Thu, 27 Apr 2023 13:25:23 +0100
Subject: [PATCH] Update OnsImportWorker to use OnsBaseWorker

---
 app/workers/ons_import_worker.rb | 60 ++++++++------------------------
 1 file changed, 14 insertions(+), 46 deletions(-)

diff --git a/app/workers/ons_import_worker.rb b/app/workers/ons_import_worker.rb
index 07ac1e9..638ad00 100644
--- a/app/workers/ons_import_worker.rb
+++ b/app/workers/ons_import_worker.rb
@@ -1,25 +1,16 @@
-class OnsImportWorker
-  include Sidekiq::Worker
-  sidekiq_options queue: :queue_ons, lock: :until_executed, lock_timeout: nil
+class OnsImportWorker < OnsBaseWorker
+  def perform(s3_key_name)
+    temp_csv_file = Tempfile.new("tmp/ONSPD.csv")
 
-  def perform(_s3_key_name)
-    # 1. Open CSV File
-    # 2. For each line, check if we have an OS Places record for it
-    # 3. If not, create a record in the DB with the ONS style
+    s3_client.get_object(
+      response_target: temp_csv_file.path,
+      bucket: BUCKET_NAME,
+      key: s3_key_name,
+    )
 
-    read_count = 0
-    imported_count = 0
-    retired = 0
-    large = 0
-    other = 0
-    missing_postcodes = []
-    CSV.foreach(args[:path_to_csv], headers: true) do |row|
-      read_count += 1
-      puts("Read #{read_count} records") if (read_count % 1000).zero?
-      break if read_count > 100_000
-
-      normal_pc = PostcodeHelper.normalise(row["pcds"])
-      next if Postcode.where(postcode: normal_pc).count.positive?
+    CSV.foreach(temp_csv_file.path, headers: true) do |row|
+      postcode = PostcodeHelper.normalise(row["pcds"])
+      next if Postcode.where(postcode:).count.positive?
 
       results = [
         {
@@ -32,32 +23,9 @@ def perform(_s3_key_name)
         },
       ]
 
-      Postcode.create(postcode: normal_pc, results:)
-
-      imported_count += 1
-      if row["doterm"] != ""
-        retired += 1
-        # puts("Found retired postcode: #{row['pcds']}")
-      elsif row["usertype"] == "1"
-        large += 1
-        # puts("Found large postcode: #{row['pcds']}")
-      else
-        other += 1
-        missing_postcodes << row["pcds"]
-      end
-      # puts("Postcode:  #{row['pcds']}")
-      # puts("Status:    #{row['doterm'] == '' ? 'active' : 'retired'}")
-      # puts("Type:      #{row['usertype'] == '0' ? 'small' : 'large'}")
-      # puts("Latitude:  #{row['lat']}")
-      # puts("Longitude: #{row['long']}")
+      Postcode.create(postcode:, results:)
     end
-
-    puts("Total Records Read: #{read_count}")
-    puts("Total Records Imported: #{imported_count}")
-    puts("Retired Records Imported: #{retired}")
-    puts("Large Records Imported: #{large}")
-    puts("Other Records Imported: #{other}")
-    puts("Other Postcodes: ")
-    puts(missing_postcodes.join(", "))
+  rescue StandardError => e
+    puts "Error getting object: #{e.message}"
   end
 end