From f45eec4b39dbe88c2f20f099faeb354e8859ea6b Mon Sep 17 00:00:00 2001 From: Jim Bailey Date: Tue, 17 Dec 2024 16:41:14 +0000 Subject: [PATCH] chore: Add the preprod refresh versions script MAP-1975 --- .../01-create-temp-tables-sql.yaml | 2 +- .../preprod-refresh/03-copy-temp-to-dest.yaml | 2 +- .../preprod-refresh/04-drop-temp-tables.yaml | 2 +- .../05-preprod-refresh-script.yaml | 6 +- .../preprod-refresh/06-refresh-cronjob.yaml | 4 +- .../07-preprod-refresh-versions-script.yaml | 9 +- .../08-refresh-versions-cronjob.yaml | 11 +- .../hmpps-book-secure-move-api/values.yaml | 10 +- helm_deploy/values-production.yaml | 10 +- .../01-create-temp-tables-sql.yaml | 43 -- preprod-refresh/03-copy-temp-to-dest.yaml | 441 ------------------ preprod-refresh/04-drop-temp-tables.yaml | 43 -- .../05-preprod-refresh-script.yaml | 114 ----- preprod-refresh/06-refresh-cronjob.yaml | 71 --- 14 files changed, 38 insertions(+), 730 deletions(-) rename {preprod-refresh => helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh}/07-preprod-refresh-versions-script.yaml (81%) rename {preprod-refresh => helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh}/08-refresh-versions-cronjob.yaml (74%) delete mode 100644 preprod-refresh/01-create-temp-tables-sql.yaml delete mode 100644 preprod-refresh/03-copy-temp-to-dest.yaml delete mode 100644 preprod-refresh/04-drop-temp-tables.yaml delete mode 100644 preprod-refresh/05-preprod-refresh-script.yaml delete mode 100644 preprod-refresh/06-refresh-cronjob.yaml diff --git a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/01-create-temp-tables-sql.yaml b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/01-create-temp-tables-sql.yaml index df3de0826..bd6c545dd 100644 --- a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/01-create-temp-tables-sql.yaml +++ b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/01-create-temp-tables-sql.yaml @@ -1,5 +1,5 @@ {{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "create-temp-tables" | trunc 52 }} -{{- if .Values.scripts.preprodRefresh.enabled }} +{{- if .Values.scripts.preprodRefresh.main.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/03-copy-temp-to-dest.yaml b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/03-copy-temp-to-dest.yaml index 62cfad232..834ecaaa2 100644 --- a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/03-copy-temp-to-dest.yaml +++ b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/03-copy-temp-to-dest.yaml @@ -1,5 +1,5 @@ {{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "copy-from-temp-sql" | trunc 52 }} -{{- if .Values.scripts.preprodRefresh.enabled }} +{{- if .Values.scripts.preprodRefresh.main.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/04-drop-temp-tables.yaml b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/04-drop-temp-tables.yaml index 64c5a9542..496e813a5 100644 --- a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/04-drop-temp-tables.yaml +++ b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/04-drop-temp-tables.yaml @@ -1,5 +1,5 @@ {{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "drop-temp-tables" | trunc 52 }} -{{- if .Values.scripts.preprodRefresh.enabled }} +{{- if .Values.scripts.preprodRefresh.main.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/05-preprod-refresh-script.yaml b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/05-preprod-refresh-script.yaml index 466bf207b..e9e913807 100644 --- a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/05-preprod-refresh-script.yaml +++ b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/05-preprod-refresh-script.yaml @@ -1,5 +1,5 @@ {{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "preprod-refresh-script" | trunc 52 }} -{{- if .Values.scripts.preprodRefresh.enabled }} +{{- if .Values.scripts.preprodRefresh.main.enabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -7,8 +7,8 @@ metadata: data: entrypoint.sh: |- #!/bin/bash - from=$(date --date="2 days ago" '+%Y-%m-%d') - to=$(date '+%Y-%m-%d') + from=$(date --date="{{ .Values.scripts.preprodRefresh.main.dataSince }}" '+%Y-%m-%d %T') + to=$(date '+%Y-%m-%d %T') echo "Refreshing data from ${from} to ${to}" createTemp='/tmp/sql/createTables.sql' diff --git a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/06-refresh-cronjob.yaml b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/06-refresh-cronjob.yaml index f56183b40..d8f45689f 100644 --- a/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/06-refresh-cronjob.yaml +++ b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/06-refresh-cronjob.yaml @@ -4,14 +4,14 @@ {{- $script2 := printf "%s-%s" (include "generic-service.fullname" $) "copy-from-temp-sql" | trunc 52 }} {{- $script3 := printf "%s-%s" (include "generic-service.fullname" $) "drop-temp-tables" | trunc 52 }} {{- $script4 := printf "%s-%s" (include "generic-service.fullname" $) "preprod-refresh-script" | trunc 52 }} -{{- if .Values.scripts.preprodRefresh.enabled }} +{{- if .Values.scripts.preprodRefresh.main.enabled }} --- apiVersion: batch/v1 kind: CronJob metadata: name: {{ $fullName }} spec: - schedule: {{ .Values.scripts.preprodRefresh.schedule }} + schedule: {{ .Values.scripts.preprodRefresh.main.schedule }} concurrencyPolicy: "Forbid" successfulJobsHistoryLimit: 5 failedJobsHistoryLimit: 3 diff --git a/preprod-refresh/07-preprod-refresh-versions-script.yaml b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/07-preprod-refresh-versions-script.yaml similarity index 81% rename from preprod-refresh/07-preprod-refresh-versions-script.yaml rename to helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/07-preprod-refresh-versions-script.yaml index 0e94f5b4f..9ec8d93ac 100644 --- a/preprod-refresh/07-preprod-refresh-versions-script.yaml +++ b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/07-preprod-refresh-versions-script.yaml @@ -1,11 +1,13 @@ +{{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "preprod-refresh-versions-script" | trunc 52 }} +{{- if .Values.scripts.preprodRefresh.versions.enabled }} apiVersion: v1 kind: ConfigMap metadata: - name: preprod-refresh-versions-script + name: {{ $fullName }} data: entrypoint.sh: |- #!/bin/bash - from=$(date --date="26 hours ago" '+%Y-%m-%d %T') + from=$(date --date="{{ .Values.scripts.preprodRefresh.versions.dataSince }}" '+%Y-%m-%d %T') to=$(date '+%Y-%m-%d %T') echo "Refreshing versions from ${from} to ${to}" @@ -38,4 +40,5 @@ data: echo "Dropping temporary tables" execDest "DROP TABLE IF EXISTS versions_temp CASCADE;" execSource "DROP TABLE IF EXISTS versions_temp CASCADE;" - echo "All done!" \ No newline at end of file + echo "All done!" +{{- end }} diff --git a/preprod-refresh/08-refresh-versions-cronjob.yaml b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/08-refresh-versions-cronjob.yaml similarity index 74% rename from preprod-refresh/08-refresh-versions-cronjob.yaml rename to helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/08-refresh-versions-cronjob.yaml index 6a97f9c7b..2025edd9a 100644 --- a/preprod-refresh/08-refresh-versions-cronjob.yaml +++ b/helm_deploy/hmpps-book-secure-move-api/templates/preprod-refresh/08-refresh-versions-cronjob.yaml @@ -1,10 +1,14 @@ +{{- $genericService := index .Values "generic-service" -}} +{{- $fullName := printf "%s-%s" (include "generic-service.fullname" $) "preprod-refresh" | trunc 52 }} +{{- $script1 := printf "%s-%s" (include "generic-service.fullname" $) "preprod-refresh-versions-script" | trunc 52 }} +{{- if .Values.scripts.preprodRefresh.versions.enabled }} --- apiVersion: batch/v1 kind: CronJob metadata: name: preprod-refresh-versions-job spec: - schedule: "15 7 * * *" + schedule: {{ .Values.scripts.preprodRefresh.versions.schedule }} concurrencyPolicy: "Forbid" successfulJobsHistoryLimit: 5 failedJobsHistoryLimit: 3 @@ -43,5 +47,6 @@ spec: volumes: - name: preprod-refresh-versions-script configMap: - name: preprod-refresh-versions-script - defaultMode: 0755 \ No newline at end of file + name: {{ $script1 }} + defaultMode: 0755 +{{- end }} diff --git a/helm_deploy/hmpps-book-secure-move-api/values.yaml b/helm_deploy/hmpps-book-secure-move-api/values.yaml index 14c74a280..e75326928 100644 --- a/helm_deploy/hmpps-book-secure-move-api/values.yaml +++ b/helm_deploy/hmpps-book-secure-move-api/values.yaml @@ -62,8 +62,14 @@ dashboards: scripts: preprodRefresh: - enabled: false - # schedule: "15 6 * * *" + main: + enabled: false + # schedule: "0 */4 * * *" + # dataSince: "6 hours ago" + versions: + enabled: false + # schedule: "15 7 * * *" + # dataSince: "26 hours ago" reports: enabled: false dailyIpt: diff --git a/helm_deploy/values-production.yaml b/helm_deploy/values-production.yaml index 97e39901a..04e95b6ee 100644 --- a/helm_deploy/values-production.yaml +++ b/helm_deploy/values-production.yaml @@ -119,8 +119,14 @@ dashboards: scripts: preprodRefresh: - enabled: true - schedule: "15 6 * * *" + main: + enabled: true + schedule: "0 */4 * * *" + dataSince: "6 hours ago" + versions: + enabled: true + schedule: "15 7 * * *" + dataSince: "26 hours ago" reports: enabled: true dailyIpt: diff --git a/preprod-refresh/01-create-temp-tables-sql.yaml b/preprod-refresh/01-create-temp-tables-sql.yaml deleted file mode 100644 index 1b5f03a33..000000000 --- a/preprod-refresh/01-create-temp-tables-sql.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: create-temp-tables -data: - createTables.sql: |- - BEGIN; - CREATE TABLE framework_flags_temp (LIKE framework_flags); - CREATE TABLE framework_nomis_codes_temp (LIKE framework_nomis_codes); - CREATE TABLE framework_nomis_mappings_temp (LIKE framework_nomis_mappings); - CREATE TABLE framework_questions_temp (LIKE framework_questions); - CREATE TABLE framework_responses_temp (LIKE framework_responses); - CREATE TABLE frameworks_temp (LIKE frameworks); - CREATE TABLE framework_flags_responses_temp (LIKE framework_flags_responses); - CREATE TABLE framework_nomis_codes_questions_temp (LIKE framework_nomis_codes_questions); - CREATE TABLE framework_nomis_mappings_responses_temp (LIKE framework_nomis_mappings_responses); - CREATE TABLE locations_temp (LIKE locations); - CREATE TABLE moves_temp (LIKE moves); - CREATE TABLE journeys_temp (LIKE journeys); - CREATE TABLE people_temp (LIKE people); - CREATE TABLE profiles_temp (LIKE profiles); - CREATE TABLE person_escort_records_temp (LIKE person_escort_records); - CREATE TABLE court_hearings_temp (LIKE court_hearings); - CREATE TABLE allocation_complex_cases_temp (LIKE allocation_complex_cases); - CREATE TABLE allocations_temp (LIKE allocations); - CREATE TABLE assessment_questions_temp (LIKE assessment_questions); - CREATE TABLE categories_temp (LIKE categories); - CREATE TABLE documents_temp (LIKE documents); - CREATE TABLE ethnicities_temp (LIKE ethnicities); - CREATE TABLE genders_temp (LIKE genders); - CREATE TABLE locations_regions_temp (LIKE locations_regions); - CREATE TABLE nationalities_temp (LIKE nationalities); - CREATE TABLE nomis_alerts_temp (LIKE nomis_alerts); - CREATE TABLE notifications_temp (LIKE notifications); - CREATE TABLE populations_temp (LIKE populations); - CREATE TABLE regions_temp (LIKE regions); - CREATE TABLE subscriptions_temp (LIKE subscriptions); - CREATE TABLE supplier_locations_temp (LIKE supplier_locations); - CREATE TABLE suppliers_temp (LIKE suppliers); - CREATE TABLE youth_risk_assessments_temp (LIKE youth_risk_assessments); - CREATE TABLE generic_events_temp (LIKE generic_events); - -- CREATE TABLE versions_temp (LIKE versions); - COMMIT; diff --git a/preprod-refresh/03-copy-temp-to-dest.yaml b/preprod-refresh/03-copy-temp-to-dest.yaml deleted file mode 100644 index b668bc948..000000000 --- a/preprod-refresh/03-copy-temp-to-dest.yaml +++ /dev/null @@ -1,441 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: copy-from-temp-sql -data: - copyFromTemp.sql: |- - BEGIN; - INSERT INTO frameworks SELECT * FROM frameworks_temp ON CONFLICT (id) DO UPDATE - SET - name = EXCLUDED.name, - version = EXCLUDED.version, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at; - - INSERT INTO framework_questions SELECT * FROM framework_questions_temp ON CONFLICT (id) DO UPDATE - SET - framework_id = EXCLUDED.framework_id, - key = EXCLUDED.key, - section = EXCLUDED.section, - required = EXCLUDED.required, - question_type = EXCLUDED.question_type, - options = EXCLUDED.options, - dependent_value = EXCLUDED.dependent_value, - followup_comment = EXCLUDED.followup_comment, - followup_comment_options = EXCLUDED.followup_comment_options, - parent_id = EXCLUDED.parent_id, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, - prefill = EXCLUDED.prefill; - - INSERT INTO framework_flags SELECT * FROM framework_flags_temp ON CONFLICT (id) DO UPDATE - SET - framework_question_id = EXCLUDED.framework_question_id, - flag_type = EXCLUDED.flag_type, - title = EXCLUDED.title, - question_value = EXCLUDED.question_value, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at; - - - INSERT INTO framework_nomis_codes SELECT * FROM framework_nomis_codes_temp ON CONFLICT (id) DO UPDATE - SET - code_type = EXCLUDED.code_type, - code = EXCLUDED.code, - fallback = EXCLUDED.fallback, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at; - - INSERT INTO framework_nomis_mappings SELECT * FROM framework_nomis_mappings_temp ON CONFLICT (id) DO UPDATE - SET - raw_nomis_mapping = EXCLUDED.raw_nomis_mapping, - code = EXCLUDED.code, - code_type = EXCLUDED.code_type, - code_description = EXCLUDED.code_description, - comments = EXCLUDED.comments, - start_date = EXCLUDED.start_date, - end_date = EXCLUDED.end_date, - creation_date = EXCLUDED.creation_date, - expiry_date = EXCLUDED.expiry_date, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, - approval_date = EXCLUDED.approval_date, - next_review_date = EXCLUDED.next_review_date; - - - INSERT INTO framework_responses SELECT * FROM framework_responses_temp ON CONFLICT (id) DO UPDATE - SET - framework_question_id = EXCLUDED.framework_question_id, - value_text = EXCLUDED.value_text, - value_json = EXCLUDED.value_json, - type = EXCLUDED.type, - parent_id = EXCLUDED.parent_id, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, - responded = EXCLUDED.responded, - prefilled = EXCLUDED.prefilled, - assessmentable_id = EXCLUDED.assessmentable_id, - assessmentable_type = EXCLUDED.assessmentable_type, - value_type = EXCLUDED.value_type, - section = EXCLUDED.section, - responded_by = EXCLUDED.responded_by, - responded_at = EXCLUDED.responded_at; - - - - INSERT INTO framework_flags_responses_temp SELECT * FROM framework_flags_responses ON CONFLICT DO NOTHING; - TRUNCATE table framework_flags_responses; - INSERT INTO framework_flags_responses select DISTINCT framework_response_id, framework_flag_id from framework_flags_responses_temp; - - INSERT INTO framework_nomis_codes_questions_temp SELECT * FROM framework_nomis_codes_questions ON CONFLICT DO NOTHING; - TRUNCATE TABLE framework_nomis_codes_questions; - INSERT INTO framework_nomis_codes_questions SELECT DISTINCT framework_question_id, framework_nomis_code_id FROM framework_nomis_codes_questions_temp ON CONFLICT DO NOTHING; - - INSERT INTO framework_nomis_mappings_responses SELECT * FROM framework_nomis_mappings_responses_temp ON CONFLICT (id) DO UPDATE - SET - framework_response_id = EXCLUDED.framework_response_id, - framework_nomis_mapping_id = EXCLUDED.framework_nomis_mapping_id; - - - INSERT INTO categories SELECT * FROM categories_temp ON CONFLICT (id) DO UPDATE - SET - key = EXCLUDED.key, - title = EXCLUDED.title, - move_supported = EXCLUDED.move_supported, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at; - -- - INSERT INTO ethnicities SELECT * FROM ethnicities_temp ON CONFLICT (id) DO UPDATE - SET - key = EXCLUDED.key, - title = EXCLUDED.title, - description = EXCLUDED.description, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, - nomis_code = EXCLUDED.nomis_code, - disabled_at = EXCLUDED.disabled_at; - -- - - INSERT INTO genders SELECT * FROM genders_temp ON CONFLICT (id) DO UPDATE - SET - title = EXCLUDED.title, - description = EXCLUDED.description, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, - key = EXCLUDED.key, - nomis_code = EXCLUDED.nomis_code, - disabled_at = EXCLUDED.disabled_at; - -- - - INSERT INTO locations SELECT * FROM locations_temp ON CONFLICT (id) DO UPDATE - SET - title = EXCLUDED.title, - location_type = EXCLUDED.location_type, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, - nomis_agency_id = EXCLUDED.nomis_agency_id, - key = EXCLUDED.key, - disabled_at = EXCLUDED.disabled_at, - can_upload_documents = EXCLUDED.can_upload_documents, - category_id = EXCLUDED.category_id, - young_offender_institution = EXCLUDED.young_offender_institution, - premise = EXCLUDED.premise, - locality = EXCLUDED.locality, - city = EXCLUDED.city, - country = EXCLUDED.country, - postcode = EXCLUDED.postcode, - latitude = EXCLUDED.latitude, - longitude = EXCLUDED.longitude; - -- - INSERT INTO regions SELECT * FROM regions_temp ON CONFLICT (id) DO UPDATE - SET - name = EXCLUDED.name, - key = EXCLUDED.key, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at; - -- - - INSERT INTO suppliers SELECT * FROM suppliers_temp ON CONFLICT (id) DO UPDATE - SET - name = EXCLUDED.name, - key = EXCLUDED.key, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at; - -- - - INSERT INTO supplier_locations SELECT * FROM supplier_locations_temp ON CONFLICT (id) DO UPDATE - SET - supplier_id = EXCLUDED.supplier_id, - location_id = EXCLUDED.location_id, - effective_from = EXCLUDED.effective_from, - effective_to = EXCLUDED.effective_to, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at; - -- - - INSERT INTO locations_regions SELECT * FROM locations_regions_temp ON CONFLICT DO NOTHING; - -- - - INSERT INTO assessment_questions SELECT * FROM assessment_questions_temp ON CONFLICT (id) DO UPDATE - SET - title = EXCLUDED.title, - category = EXCLUDED.category, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, - key = EXCLUDED.key, - disabled_at = EXCLUDED.disabled_at; - -- - - INSERT INTO nationalities SELECT * FROM nationalities_temp ON CONFLICT (id) DO UPDATE - SET - title = EXCLUDED.title, - description = EXCLUDED.description, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, - key = EXCLUDED.key, - disabled_at = EXCLUDED.disabled_at; - -- - - INSERT INTO nomis_alerts SELECT * FROM nomis_alerts_temp ON CONFLICT (id) DO UPDATE - SET - type_code = EXCLUDED.type_code, - code = EXCLUDED.code, - description = EXCLUDED.description, - type_description = EXCLUDED.type_description, - assessment_question_id = EXCLUDED.assessment_question_id, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at; - -- - INSERT INTO populations SELECT * FROM populations_temp ON CONFLICT (id) DO UPDATE - SET - location_id = EXCLUDED.location_id, - date = EXCLUDED.date, - operational_capacity = EXCLUDED.operational_capacity, - usable_capacity = EXCLUDED.usable_capacity, - unlock = EXCLUDED.unlock, - bedwatch = EXCLUDED.bedwatch, - overnights_in = EXCLUDED.overnights_in, - overnights_out = EXCLUDED.overnights_out, - out_of_area_courts = EXCLUDED.out_of_area_courts, - discharges = EXCLUDED.discharges, - updated_by = EXCLUDED.updated_by, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at; - -- - - INSERT INTO moves SELECT * FROM moves_temp ON CONFLICT (id) DO UPDATE - SET - date = EXCLUDED.date, - from_location_id = EXCLUDED.from_location_id, - to_location_id = EXCLUDED.to_location_id, - status = EXCLUDED.status, - updated_at = EXCLUDED.updated_at, - reference = EXCLUDED.reference, - move_type = EXCLUDED.move_type, - additional_information = EXCLUDED.additional_information, - nomis_event_id = EXCLUDED.nomis_event_id, - time_due = EXCLUDED.time_due, - cancellation_reason = EXCLUDED.cancellation_reason, - cancellation_reason_comment = EXCLUDED.cancellation_reason_comment, - profile_id = EXCLUDED.profile_id, - prison_transfer_reason_id = EXCLUDED.prison_transfer_reason_id, - reason_comment = EXCLUDED.reason_comment, - move_agreed = EXCLUDED.move_agreed, - move_agreed_by = EXCLUDED.move_agreed_by, - date_from = EXCLUDED.date_from, - date_to = EXCLUDED.date_to, - allocation_id = EXCLUDED.allocation_id, - rejection_reason = EXCLUDED.rejection_reason, - original_move_id = EXCLUDED.original_move_id, - supplier_id = EXCLUDED.supplier_id, - is_lockout = EXCLUDED.is_lockout; - -- - - INSERT INTO journeys SELECT * FROM journeys_temp ON CONFLICT (id) DO UPDATE - SET - move_id = EXCLUDED.move_id, - supplier_id = EXCLUDED.supplier_id, - from_location_id = EXCLUDED.from_location_id, - to_location_id = EXCLUDED.to_location_id, - billable = EXCLUDED.billable, - state = EXCLUDED.state, - vehicle = EXCLUDED.vehicle, - client_timestamp = EXCLUDED.client_timestamp, - updated_at = EXCLUDED.updated_at, - date = EXCLUDED.date; - -- - INSERT INTO people SELECT * FROM people_temp ON CONFLICT (id) DO UPDATE - SET - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, - nomis_prison_number = EXCLUDED.nomis_prison_number, - prison_number = EXCLUDED.prison_number, - criminal_records_office = EXCLUDED.criminal_records_office, - police_national_computer = EXCLUDED.police_national_computer, - first_names = EXCLUDED.first_names, - last_name = EXCLUDED.last_name, - date_of_birth = EXCLUDED.date_of_birth, - gender_additional_information = EXCLUDED.gender_additional_information, - ethnicity_id = EXCLUDED.ethnicity_id, - gender_id = EXCLUDED.gender_id, - last_synced_with_nomis = EXCLUDED.last_synced_with_nomis, - latest_nomis_booking_id = EXCLUDED.latest_nomis_booking_id; - -- - - INSERT INTO profiles SELECT * FROM profiles_temp ON CONFLICT (id) DO UPDATE - SET - person_id = EXCLUDED.person_id, - last_name = EXCLUDED.last_name, - first_names = EXCLUDED.first_names, - date_of_birth = EXCLUDED.date_of_birth, - aliases = EXCLUDED.aliases, - gender_id = EXCLUDED.gender_id, - ethnicity_id = EXCLUDED.ethnicity_id, - nationality_id = EXCLUDED.nationality_id, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, - assessment_answers = EXCLUDED.assessment_answers, - profile_identifiers = EXCLUDED.profile_identifiers, - gender_additional_information = EXCLUDED.gender_additional_information, - latest_nomis_booking_id = EXCLUDED.latest_nomis_booking_id, - category = EXCLUDED.category, - category_code = EXCLUDED.category_code, - category_id = EXCLUDED.category_id, - requires_youth_risk_assessment = EXCLUDED.requires_youth_risk_assessment, - csra = EXCLUDED.csra; - -- - commit; - begin; - INSERT INTO person_escort_records SELECT * FROM person_escort_records_temp ON CONFLICT (id) DO UPDATE - SET - framework_id = EXCLUDED.framework_id, - profile_id = EXCLUDED.profile_id, - status = EXCLUDED.status, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, - confirmed_at = EXCLUDED.confirmed_at, - move_id = EXCLUDED.move_id, - nomis_sync_status = EXCLUDED.nomis_sync_status, - prefill_source_id = EXCLUDED.prefill_source_id, - completed_at = EXCLUDED.completed_at, - section_progress = EXCLUDED.section_progress, - amended_at = EXCLUDED.amended_at, - handover_details = EXCLUDED.handover_details, - handover_occurred_at = EXCLUDED.handover_occurred_at; - -- - - INSERT INTO allocation_complex_cases SELECT * FROM allocation_complex_cases_temp ON CONFLICT (id) DO UPDATE - SET - key = EXCLUDED.key, - title = EXCLUDED.title, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at; - -- - - - - INSERT INTO documents SELECT * FROM documents_temp ON CONFLICT (id) DO UPDATE - SET - move_id = EXCLUDED.move_id, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, - discarded_at = EXCLUDED.discarded_at, - documentable_type = EXCLUDED.documentable_type, - documentable_id = EXCLUDED.documentable_id; - -- - - INSERT INTO subscriptions SELECT * FROM subscriptions_temp ON CONFLICT DO NOTHING; - -- - - INSERT INTO notifications SELECT * FROM notifications_temp ON CONFLICT (id) DO UPDATE - SET - subscription_id = EXCLUDED.subscription_id, - event_type = EXCLUDED.event_type, - topic_id = EXCLUDED.topic_id, - topic_type = EXCLUDED.topic_type, - delivery_attempts = EXCLUDED.delivery_attempts, - delivery_attempted_at = EXCLUDED.delivery_attempted_at, - delivered_at = EXCLUDED.delivered_at, - discarded_at = EXCLUDED.discarded_at, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, - response_id = EXCLUDED.response_id, - notification_type_id = EXCLUDED.notification_type_id; - -- - - - INSERT INTO allocations SELECT * FROM allocations_temp ON CONFLICT (id) DO UPDATE - SET - from_location_id = EXCLUDED.from_location_id, - to_location_id = EXCLUDED.to_location_id, - date = EXCLUDED.date, - prisoner_category = EXCLUDED.prisoner_category, - sentence_length = EXCLUDED.sentence_length, - complex_cases = EXCLUDED.complex_cases, - moves_count = EXCLUDED.moves_count, - complete_in_full = EXCLUDED.complete_in_full, - other_criteria = EXCLUDED.other_criteria, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, - status = EXCLUDED.status, - cancellation_reason = EXCLUDED.cancellation_reason, - cancellation_reason_comment = EXCLUDED.cancellation_reason_comment, - requested_by = EXCLUDED.requested_by, - estate = EXCLUDED.estate, - sentence_length_comment = EXCLUDED.sentence_length_comment, - estate_comment = EXCLUDED.estate_comment; - -- - - - - INSERT INTO court_hearings SELECT * FROM court_hearings_temp ON CONFLICT (id) DO UPDATE - SET - move_id = EXCLUDED.move_id, - start_time = EXCLUDED.start_time, - case_start_date = EXCLUDED.case_start_date, - case_type = EXCLUDED.case_type, - comments = EXCLUDED.comments, - case_number = EXCLUDED.case_number, - nomis_case_id = EXCLUDED.nomis_case_id, - nomis_hearing_id = EXCLUDED.nomis_hearing_id, - saved_to_nomis = EXCLUDED.saved_to_nomis, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at; - -- - - INSERT INTO youth_risk_assessments SELECT * FROM youth_risk_assessments_temp ON CONFLICT (id) DO UPDATE - SET - framework_id = youth_risk_assessments.framework_id, - profile_id = EXCLUDED.profile_id, - move_id = EXCLUDED.move_id, - prefill_source_id = EXCLUDED.prefill_source_id, - status = EXCLUDED.status, - nomis_sync_status = EXCLUDED.nomis_sync_status, - confirmed_at = EXCLUDED.confirmed_at, - completed_at = EXCLUDED.completed_at, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, - section_progress = EXCLUDED.section_progress; - -- - COMMIT; - -- - BEGIN; - INSERT INTO generic_events SELECT * FROM generic_events_temp ON CONFLICT (id) DO UPDATE - SET - eventable_id = EXCLUDED.eventable_id, - eventable_type = EXCLUDED.eventable_type, - type = EXCLUDED.type, - notes = EXCLUDED.notes, - created_by = EXCLUDED.created_by, - details = EXCLUDED.details, - occurred_at = EXCLUDED.occurred_at, - recorded_at = EXCLUDED.recorded_at, - created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, - supplier_id = EXCLUDED.supplier_id, - classification = EXCLUDED.classification; - -- - - -- INSERT INTO versions (id, item_type, item_id, event, whodunnit, object, created_at, supplier_id) SELECT nextval('versions_id_seq'::regclass), item_type, item_id, event, whodunnit, object, created_at, supplier_id FROM versions_temp; - COMMIT; - diff --git a/preprod-refresh/04-drop-temp-tables.yaml b/preprod-refresh/04-drop-temp-tables.yaml deleted file mode 100644 index 99f6c40b1..000000000 --- a/preprod-refresh/04-drop-temp-tables.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: drop-temp-tables -data: - dropTables.sql: |- - BEGIN; - DROP TABLE IF EXISTS framework_flags_temp CASCADE; - DROP TABLE IF EXISTS framework_nomis_codes_temp CASCADE; - DROP TABLE IF EXISTS framework_nomis_mappings_temp CASCADE; - DROP TABLE IF EXISTS framework_questions_temp CASCADE; - DROP TABLE IF EXISTS framework_responses_temp CASCADE; - DROP TABLE IF EXISTS frameworks_temp CASCADE; - DROP TABLE IF EXISTS framework_flags_responses_temp CASCADE; - DROP TABLE IF EXISTS framework_nomis_codes_questions_temp CASCADE; - DROP TABLE IF EXISTS framework_nomis_mappings_responses_temp CASCADE; - DROP TABLE IF EXISTS locations_temp CASCADE; - DROP TABLE IF EXISTS moves_temp CASCADE; - DROP TABLE IF EXISTS journeys_temp CASCADE; - DROP TABLE IF EXISTS people_temp CASCADE; - DROP TABLE IF EXISTS profiles_temp CASCADE; - DROP TABLE IF EXISTS person_escort_records_temp CASCADE; - DROP TABLE IF EXISTS court_hearings_temp CASCADE; - DROP TABLE IF EXISTS allocation_complex_cases_temp CASCADE; - DROP TABLE IF EXISTS allocations_temp CASCADE; - DROP TABLE IF EXISTS assessment_questions_temp CASCADE; - DROP TABLE IF EXISTS categories_temp CASCADE; - DROP TABLE IF EXISTS documents_temp CASCADE; - DROP TABLE IF EXISTS ethnicities_temp CASCADE; - DROP TABLE IF EXISTS genders_temp CASCADE; - DROP TABLE IF EXISTS locations_regions_temp CASCADE; - DROP TABLE IF EXISTS nationalities_temp CASCADE; - DROP TABLE IF EXISTS nomis_alerts_temp CASCADE; - DROP TABLE IF EXISTS notifications_temp CASCADE; - DROP TABLE IF EXISTS populations_temp CASCADE; - DROP TABLE IF EXISTS regions_temp CASCADE; - DROP TABLE IF EXISTS subscriptions_temp CASCADE; - DROP TABLE IF EXISTS supplier_locations_temp CASCADE; - DROP TABLE IF EXISTS suppliers_temp CASCADE; - DROP TABLE IF EXISTS youth_risk_assessments_temp CASCADE; - DROP TABLE IF EXISTS generic_events_temp CASCADE; - -- DROP TABLE IF EXISTS versions_temp CASCADE; - COMMIT; diff --git a/preprod-refresh/05-preprod-refresh-script.yaml b/preprod-refresh/05-preprod-refresh-script.yaml deleted file mode 100644 index ea3d86ed8..000000000 --- a/preprod-refresh/05-preprod-refresh-script.yaml +++ /dev/null @@ -1,114 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: preprod-refresh-script -data: - entrypoint.sh: |- - #!/bin/bash - from=$(date --date="6 hours ago" '+%Y-%m-%d %T') #Job runs every four hours - extra 2 hours should account for clock changes - to=$(date '+%Y-%m-%d %T') - echo "Refreshing data from ${from} to ${to}" - - createTemp='/tmp/sql/createTables.sql' - copyScript='/tmp/sql/copyFromTemp.sql' - dropScript='/tmp/sql/dropTables.sql' - - execDest() { - cmd=$1 - psql $DEST_DB_INSTANCE -c "${cmd}" - } - - execSource() { - cmd=$1 - psql $SOURCE_DB_INSTANCE -c "${cmd}" - } - - execFileDest() { - file=$1 - psql $DEST_DB_INSTANCE -f ${file} - } - - execFileSource() { - file=$1 - psql $SOURCE_DB_INSTANCE -f ${file} - } - - temp () { - table=$1 - echo "Copying ${table} to source temp table" - cmd="INSERT INTO ${table}_temp SELECT * FROM ${table} where (created_at >= '${from}' and created_at < '${to}') OR (updated_at >= '${from}' and updated_at < '${to}');" - execSource "${cmd}" - } - - tempjoin () { - table=$1 - join=$2 - echo "Copying ${table} to temporary table - using explicit select statement" - cmd_two="INSERT INTO ${table}_temp ${join};" - execSource "${cmd_two}" - } - - execFileSource ${dropScript} - execFileDest ${dropScript} - execFileDest ${createTemp} - execFileSource ${createTemp} - - temp framework_flags - temp framework_nomis_codes - temp framework_nomis_mappings - temp framework_questions - temp framework_responses - temp frameworks - - #JOIN TABLES - tempjoin framework_flags_responses "SELECT framework_response_id, framework_flag_id FROM public.framework_flags_responses JOIN framework_responses_temp ON framework_responses_temp.id = framework_flags_responses.framework_response_id" - tempjoin framework_nomis_codes_questions "SELECT framework_question_id, framework_nomis_code_id FROM framework_nomis_codes_questions JOIN framework_questions_temp ON framework_questions_temp.id = framework_nomis_codes_questions.framework_question_id" - tempjoin framework_nomis_mappings_responses "SELECT framework_nomis_mappings_responses.id, framework_response_id, framework_nomis_mapping_id FROM framework_nomis_mappings_responses JOIN framework_responses_temp ON framework_responses_temp.id = framework_nomis_mappings_responses.framework_response_id" - - temp locations - temp moves - temp journeys - temp people - temp profiles - temp person_escort_records - temp court_hearings - - #tempall active_storage_attachments #Purposefully omitted - #tempall active_storage_blobs #Purposefully omitted - #tempall active_storage_variant_records #Purposefully omitted - temp allocation_complex_cases - temp allocations - #temp ar_internal_metadata #Purposefully omitted - temp assessment_questions - temp categories - temp documents - temp ethnicities - - temp genders - - temp locations_regions - temp nationalities - temp nomis_alerts - temp notifications - #temp oauth_access_grants #Purposefully omitted - #temp oauth_access_tokens #Purposefully omitted - #temp oauth_applications #Purposefully omitted - temp populations - temp regions - #temp schema_migrations #Purposefully omitted - temp subscriptions - temp supplier_locations - temp suppliers - temp youth_risk_assessments - temp generic_events - - echo "Starting dump of data to preprod" - pg_dump $SOURCE_DB_INSTANCE -t "*_temp" -O --section=data | psql $DEST_DB_INSTANCE - echo "Finished dumping data" - - echo "Copying data from destination temp tables to actual tables" - execFileDest ${copyScript} - echo "Dropping temporary tables" - execFileSource ${dropScript} - execFileDest ${dropScript} - echo "All done!" \ No newline at end of file diff --git a/preprod-refresh/06-refresh-cronjob.yaml b/preprod-refresh/06-refresh-cronjob.yaml deleted file mode 100644 index b4450a213..000000000 --- a/preprod-refresh/06-refresh-cronjob.yaml +++ /dev/null @@ -1,71 +0,0 @@ ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: preprod-refresh-job -spec: - schedule: "15 */4 * * *" - concurrencyPolicy: "Forbid" - successfulJobsHistoryLimit: 5 - failedJobsHistoryLimit: 3 - jobTemplate: - spec: - ttlSecondsAfterFinished: 345600 - backoffLimit: 0 - activeDeadlineSeconds: 7200 - template: - spec: - serviceAccountName: "book-a-secure-move-api" - containers: - - name: preprod-refresh - image: "ghcr.io/ministryofjustice/hmpps-devops-tools:latest" - command: - - /bin/entrypoint.sh - volumeMounts: - - name: create-temp-tables - mountPath: /tmp/sql/createTables.sql - readOnly: true - subPath: createTables.sql - - name: copy-from-temp-sql - mountPath: /tmp/sql/copyFromTemp.sql - readOnly: true - subPath: copyFromTemp.sql - - name: drop-temp-tables - mountPath: /tmp/sql/dropTables.sql - readOnly: true - subPath: dropTables.sql - - name: preprod-refresh-script - mountPath: /bin/entrypoint.sh - readOnly: true - subPath: entrypoint.sh - env: - - name: DEST_DB_INSTANCE - valueFrom: - secretKeyRef: - name: preprod-rds-creds - key: url - - name: SOURCE_DB_INSTANCE - valueFrom: - secretKeyRef: - name: rds-instance-hmpps-book-secure-move-api-production - key: url - - name: AWS_DEFAULT_REGION - value: "eu-west-2" - restartPolicy: "Never" - volumes: - - name: create-temp-tables - configMap: - name: create-temp-tables - defaultMode: 0755 - - name: copy-from-temp-sql - configMap: - name: copy-from-temp-sql - defaultMode: 0755 - - name: drop-temp-tables - configMap: - name: drop-temp-tables - defaultMode: 0755 - - name: preprod-refresh-script - configMap: - name: preprod-refresh-script - defaultMode: 0755 \ No newline at end of file