Fix missing value in upload model executions - add else clause to for…

… if loops (#386) * add else clause to for if loops * Refactor logic for timestamp values Refactor logic to evaluate compile_started_at and query_completed_at values. * Refactor logic for timestamp values for all macros * Update macros/upload_individual_datasets/upload_model_executions.sql * Fix issue with `.get` usage --------- Co-authored-by: Jonathan Sellar <jonathan.sellar@health.govt.nz> Co-authored-by: Gemma Down <52132406+glsdown@users.noreply.github.com> Co-authored-by: Gemma Down <gemma.down@brooklyndata.co>
brooklyn-data · Sep 28, 2023 · 3cf1473 · 3cf1473
1 parent 3bde046
commit 3cf1473
Showing 1 changed file with 12 additions and 60 deletions.
diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql
@@ -39,26 +39,10 @@
                 '{{ model.thread_id }}', {# thread_id #}
                 '{{ model.status }}', {# status #}
 
-                {% if model.timing != [] %}
-                    {% for stage in model.timing if stage.name == "compile" %}
-                        {% if loop.length == 0 %}
-                            null, {# compile_started_at #}
-                        {% else %}
-                            '{{ stage.started_at }}', {# compile_started_at #}
-                        {% endif %}
-                    {% endfor %}
-
-                    {% for stage in model.timing if stage.name == "execute" %}
-                        {% if loop.length == 0 %}
-                            null, {# query_completed_at #}
-                        {% else %}
-                            '{{ stage.completed_at }}', {# query_completed_at #}
-                        {% endif %}
-                    {% endfor %}
-                {% else %}
-                    null, {# compile_started_at #}
-                    null, {# query_completed_at #}
-                {% endif %}
+                {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %}
+                {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #}
+                {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %}
+                {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #}
 
                 {{ model.execution_time }}, {# total_node_runtime #}
                 null, -- rows_affected not available {# Only available in Snowflake & BigQuery #}
@@ -96,26 +80,10 @@
             '{{ model.thread_id }}', {# thread_id #}
             '{{ model.status }}', {# status #}
 
-            {% if model.timing != [] %}
-                {% for stage in model.timing if stage.name == "compile" %}
-                    {% if loop.length == 0 %}
-                        null, {# compile_started_at #}
-                    {% else %}
-                        '{{ stage.started_at }}', {# compile_started_at #}
-                    {% endif %}
-                {% endfor %}
-
-                {% for stage in model.timing if stage.name == "execute" %}
-                    {% if loop.length == 0 %}
-                        null, {# query_completed_at #}
-                    {% else %}
-                        '{{ stage.completed_at }}', {# query_completed_at #}
-                    {% endif %}
-                {% endfor %}
-            {% else %}
-                null, {# compile_started_at #}
-                null, {# query_completed_at #}
-            {% endif %}
+            {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %}
+            {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #}
+            {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %}
+            {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #}
 
             {{ model.execution_time }}, {# total_node_runtime #}
             safe_cast('{{ model.adapter_response.rows_affected }}' as int64),
@@ -172,26 +140,10 @@
                 '{{ model.thread_id }}', {# thread_id #}
                 '{{ model.status }}', {# status #}
 
-                {% if model.timing != [] %}
-                    {% for stage in model.timing if stage.name == "compile" %}
-                        {% if loop.length == 0 %}
-                            null, {# compile_started_at #}
-                        {% else %}
-                            '{{ stage.started_at }}', {# compile_started_at #}
-                        {% endif %}
-                    {% endfor %}
-
-                    {% for stage in model.timing if stage.name == "execute" %}
-                        {% if loop.length == 0 %}
-                            null, {# query_completed_at #}
-                        {% else %}
-                            '{{ stage.completed_at }}', {# query_completed_at #}
-                        {% endif %}
-                    {% endfor %}
-                {% else %}
-                    null, {# compile_started_at #}
-                    null, {# query_completed_at #}
-                {% endif %}
+                {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %}
+                {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #}
+                {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %}
+                {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #}
 
                 {{ model.execution_time }}, {# total_node_runtime #}
                 try_cast('{{ model.adapter_response.rows_affected }}' as int), {# rows_affected #}